add optional MemoryType to the SparseMatrix copy-ctor

Cherry-picking changes on top of our current hash.
There are things that hurt performance in laster mfem@master commits, not sure what yet. This includes: - feature/artv3/cusparse-Spmv - feature/tomstitt/temp-mem-type - patches to tmop.cpp and tmop_tools.cpp to address memory issues - patch to mem_manager/device to use mfem's default allcator instead of the host umpire one
2020-07-24 09:13:21 -07:00 · 2020-07-22 15:51:04 -07:00 · 2020-06-23 12:11:33 -07:00 · 2020-06-22 14:42:48 -07:00 · 2020-06-22 14:29:28 -07:00 · 2020-06-18 13:35:11 -07:00
180 changed files with 3633 additions and 16889 deletions
@@ -16,7 +16,7 @@ install:
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

 # Install METIS
- ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
+- ps: Start-FileDownload 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
@@ -26,17 +26,17 @@ install:
 - cd ..

 # Install hypre
- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
- 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
- cd hypre-2.19.0/src
- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
+- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2-10-0b
+- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
- cd ../..
+- cd ..

 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -50,7 +50,6 @@ examples/ex1[04-9]
 examples/ex1[0-9]p
 examples/ex2[0-9]
 examples/ex2[0-9]p
-examples/ex25-gpu

 examples/refined.mesh
 examples/displaced.mesh
@@ -123,7 +122,7 @@ examples/sundials/ex16-final.*
 examples/sundials/Example16*

 examples/petsc/ex[1-69]p
-examples/petsc/ex1[0-1]p
+examples/petsc/ex10p

 examples/petsc/mesh.*
 examples/petsc/sol.*
@@ -138,7 +137,6 @@ examples/petsc/Example9*
 examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
-examples/petsc/mode_*

 examples/pumi/ex1
 examples/pumi/ex[126]p
@@ -245,9 +243,6 @@ miniapps/navier/navier_3dfoc
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output

-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -16,12 +16,6 @@ stages:
  - tests
  - optional

-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
 jobs:
  include:

@@ -144,7 +138,8 @@ jobs:
           NPROCS=2
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -174,7 +169,8 @@ jobs:
           NPROCS=2
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -197,7 +193,7 @@ jobs:
        - cd ${TRAVIS_BUILD_DIR}/build
        - cmake ..
             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../hypre-2.10.0b/src/hypre
             -DMFEM_MPI_NP=$NPROCS
        - make -j3 mfem examples
        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
@@ -205,7 +201,8 @@ jobs:
        - ctest --output-on-failure
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -250,7 +247,8 @@ jobs:
           TMPDIR=/tmp
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -270,7 +268,8 @@ jobs:
           TMPDIR=/tmp
      cache:
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -336,18 +335,18 @@ install:

   # hypre
   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+        if [ ! -e hypre-2.10.0b/src/hypre/lib/libHYPRE.a ]; then
+           wget https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz --no-check-certificate;
+           rm -rf hypre-2.10.0b;
+           tar xvzf hypre-2.10.0b.tar.gz;
+           cd hypre-2.10.0b/src;
+           ./configure --disable-fortran --without-fei CC=mpicc CXX=mpic++;
           make -j3;
           cd ../..;
        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
+            echo "Reusing cached hypre-2.10.0b/";
        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
+        ln -s hypre-2.10.0b hypre;
     else
        echo "Serial build, not using hypre";
     fi
@@ -355,7 +354,7 @@ install:
   # METIS
   - if [ $MPI == "YES" ]; then
        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz;
           tar xvzf metis-4.0.3.tar.gz;
           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
           rm -rf metis-4.0;
@@ -33,9 +33,6 @@ Meshing improvements
  for example the periodic-annulus-sector and periodic-torus-sector files in
  the data directory.

- Added complete action of the TMOP Integrator to account for the spatial
-  derivatives of discrete and analytic targets.
-
 Performance improvements
 ------------------------
 - Added support for explicit vectorization in the high-performance templated
@@ -51,19 +48,6 @@ Improved GPU capabilities
 -------------------------
 - Added support for Chebyshev accelerated polynomial smoother on GPU.

- Optimized AMD/HIP kernel support.
-
- Added a Full Assembly mode compatible with Device kernel execution. This
-  assembly level builds on top of the current Element Assembly kernels to
-  compute a global sparse matrix. All integrators supported by element assembly
-  are also supported by full assembly. See the '-fa' option in Example 9.
-
- Added support for BlockOperator on GPU. See the updated Example 5.
-
- Added partial assembly and GPU support for ComplexOperator,
-  [Par]ComplexGridFunction, [Par]ComplexLinearForm, and [Par]SesquilinearForm.
-  See the updated Example 22.
-
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -87,7 +71,7 @@ Discretization improvements
  and, in the continuous field case, arbitrary mesh edges and faces.

 - Added new coefficient and vector coefficient classes for QuadratureFunctions.
-  Additionally, new LinearForm integrators were also added which make use of
+  Additionaly, new LinearForm integrators were also added which make use of
  these new QuadratureFunction coefficient classes.

 - Added support face integrals on the boundaries of NURBS meshes.
@@ -104,10 +88,6 @@ Linear and nonlinear solvers
  and solution during the solving process of an IterativeSolver after every
  iteration.

- Added support for the CVODES package in SUNDIALS which provides ODE
-  solvers with sensitivity analysis capabilities. See the CVODESSolver
-  class and the new adjoint miniapps below.
-
 - Block arrays of parallel matrices can now be merged into a single parallel
  matrix with the function HypreParMatrixFromBlocks. This could be useful for
  solving block systems with parallel direct solvers such as STRUMPACK.
@@ -115,8 +95,6 @@ Linear and nonlinear solvers
 - In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
  solver can work with non-SPD preconditioner B.

- Added support for the SLEPc eigensolver package.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -131,21 +109,6 @@ New and updated examples and miniapps
  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
  and periodic boundary conditions with either H1 or DG discretizations.

- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
-  equations of incompressible fluid dynamics. See the miniapps/navier directory
-  for more details.
-
- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
-  solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
-  these miniapps require the MFEM_USE_SUNDIALS configuration option.
-  * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
-    system of ODEs, evaluating both forward and adjoint quadratures in serial.
-  * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
-    for an advection diffusion PDE, evaluating adjoint quadratures in parallel.
-
- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
-  equation with the shift-and-invert spectral transformation method.
-
 - Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
  stitching together opposite surfaces of a mesh to create a topologically
  periodic mesh.
@@ -153,10 +116,11 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
  the Dirichlet problem for the minimal surface equation.

- Added full assembly support in Example 9/9p.
+- Added partial assembly support to examples 4/4p and 5/5p, with diagonal
+  preconditioning.

- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
-  form for H1, H(curl), H(div) and L_2, with partial assembly support.
+- Added a new test problem in example 24/24p, demonstrating a mixed bilinear
+  form for H(div) and L_2, with partial assembly support.

 - Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

@@ -164,9 +128,6 @@ New and updated examples and miniapps
  mesh based on element attributes. Any newly exposed boundary elements are
  assigned attribute numbers related to the trimmed element attributes.

- Added partial assembly and device support to Example 4/4p, Example 5/5p,
-  Example 22/22p, and Example 25/25p, with diagonal preconditioning.
-
 Improved testing
 ----------------
 - Added a GitLab pipeline that automates PR testing on supercomputing systems
@@ -149,13 +149,9 @@ if (MFEM_USE_MPI)
      message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
    endif()
    set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
-    if (MFEM_USE_SLEPC)
-      find_package(SLEPc REQUIRED config)
-      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
-    endif()
  endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
  foreach(PKG IN LISTS PKGS_NEED_MPI)
    if (MFEM_USE_${PKG})
      message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -211,10 +207,10 @@ endif()
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
  if (NOT MFEM_USE_MPI)
-    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODE ARKODE KINSOL)
  else()
    find_package(SUNDIALS REQUIRED
-      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODE ARKODE KINSOL)
  endif()
 endif()

@@ -356,7 +352,7 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -109,7 +109,6 @@ The MFEM source code has the following structure:
  ├── linalg
  ├── mesh
  ├── miniapps
-  │   ├── adjoint
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
@@ -383,10 +383,6 @@ MFEM_USE_PETSC = YES/NO
   and other features based on the PETSc package. When enabled, this option uses
   the PETSC_* library options, see below.

-MFEM_USE_SLEPC = YES/NO
-   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
-   option uses the SLEPC_* library options, see below.
-
 MFEM_USE_MPFR = YES/NO
   MPFR is a library for multiple-precision floating-point computations. This
   option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -601,12 +597,6 @@ The specific libraries and their options are:
  Options: PETSC_OPT, PETSC_LIB.
  Versions: PETSc >= 3.8.0.

- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
-  uses some of the PETSc options when compiled.
-  URL: https://slepc.upv.es/
-  Options: SLEPC_OPT, SLEPC_LIB.
-  Versions: SLEPc >= 3.8.0.
-
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
  URL: https://github.com/LLNL/axom
@@ -659,11 +649,12 @@ The specific libraries and their options are:
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
  Versions: OCCA >= 1.0.9.

- libCEED (optional), used when MFEM_USE_CEED = YES.
+- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
+  or later version, specifically, git-hash 3d05795 or later.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.6, git-hash a970f63.
+  Versions: libCEED >= 0.6.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -244,10 +244,6 @@ IF (DEFINED TPL_ENABLE_PETSC)
  SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_SLEPC)
-  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_MPFR)
  SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()
@@ -38,7 +38,6 @@ set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
-set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
@@ -104,9 +104,6 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC

-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE

@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - SLEPC_FOUND
-#   - SLEPC_INCLUDE_DIRS
-#   - SLEPC_LIBRARIES
-
-set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
-  "Additional packages required by SLEPc")
-
-include(MfemCmakeUtilities)
-mfem_find_package(SLEPc SLEPC SLEPC_DIR
-  "include" "slepceps.h"
-  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
-  "Paths to headers required by SLEPc."
-  "Libraries required by SLEPc."
-  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
-  CHECK_BUILD SLEPC_VERSION_OK TRUE
-"
-#include \"petsc.h\"
-#include \"slepceps.h\"
-int main()
-{
-  PetscErrorCode ierr;
-  int argc = 0;
-  char** argv = NULL;
-  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
-  EPS eps;
-  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
-  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
-  ierr = SlepcFinalize(); CHKERRQ(ierr);
-  return 0;
-}
-"
-  )
@@ -25,6 +25,5 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  ADD_COMPONENT NVector_ParHyp
    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
-  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
@@ -731,7 +731,7 @@ function(mfem_export_mk_files)
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
@@ -48,9 +48,6 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_SLEPC
-#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -118,9 +118,6 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC

-// Enable functionality based on the SLEPc library
-// #define MFEM_USE_SLEPC
-
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR

@@ -37,7 +37,6 @@ MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
-MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
@@ -39,7 +39,6 @@ option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
-option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -88,8 +87,6 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
-# and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
@@ -158,10 +155,6 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
    "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")

-set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
-    "Path to the SLEPc main directory.")
-set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
-
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
@@ -189,12 +189,10 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
-# and modify cmake variables for hypre for sundials
 SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
 SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
 SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
- -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+ -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
@@ -341,9 +339,9 @@ GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
 GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs

-# CUDA library configuration (currently not needed)
+# CUDA library configuration
 CUDA_OPT =
-CUDA_LIB =
+CUDA_LIB = -lcusparse

 # HIP library configuration (currently not needed)
 HIP_OPT =
@@ -770,7 +770,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/examples/pumi \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
-                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
@@ -88,8 +88,8 @@ namespace mfem {
 * - <a class="el" href="ex24p_8cpp_source.html">Example 24p</a>: parallel mixed finite element spaces and interpolators
 * - <a class="el" href="ex25_8cpp_source.html">Example 25</a>: simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
 * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
- * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
+ * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM 
+ * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM 
 *
 * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
@@ -101,9 +101,6 @@ namespace mfem {
 *   and
 *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
 *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
- * - CVODES adjoint examples:
- *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
- *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
 *
 * <H4>PETSc Examples</H4>
 * - Variants of Examples
@@ -143,9 +140,7 @@ namespace mfem {
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
 * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
 * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
-
-* - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
+ * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
 * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
 * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
 * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
@@ -162,6 +157,7 @@ namespace mfem {
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
 * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
+ * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -91,7 +91,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
    add_test(NAME ${TEST_NAME}_ser
      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+    add_test(NAME ${TEST_NAME}_np=4
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,22 +101,13 @@ endforeach()

 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+  add_test(NAME ex11p_strumpack_np=4
    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
    ${MPIEXEC_PREFLAGS}
    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
    ${MPIEXEC_POSTFLAGS})
 endif()

-# If SuperLU_DIST is enabled, add a test run that uses it.
-if (MFEM_USE_SUPERLU)
-  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-    ${MPIEXEC_POSTFLAGS})
-endif()
-
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
@@ -34,8 +34,7 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -103,8 +102,8 @@ int main(int argc, char *argv[])
   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -112,10 +111,10 @@ int main(int argc, char *argv[])
   //    elements.
   {
      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

@@ -123,70 +122,66 @@ int main(int argc, char *argv[])
   //    Lagrange finite elements of the specified order. If order < 1, we
   //    instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (mesh.GetNodes())
+   else if (mesh->GetNodes())
   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = mesh->GetNodes()->OwnFEC();
      cout << "Using isoparametric FEs: " << fec->Name() << endl;
   }
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   FiniteElementSpace fespace(&mesh, fec);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
+        << fespace->GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
+   if (mesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
+   LinearForm *b = new LinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero,
   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
+   GridFunction x(fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   BilinearForm *a = new BilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
   //     conditions, applying conforming constraints for non-conforming AMR,
   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   cout << "Size of linear system: " << A->Height() << endl;

@@ -207,9 +202,9 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
+         OperatorJacobiSmoother M(*a, ess_tdof_list);
         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
      }
      else
@@ -219,13 +214,13 @@ int main(int argc, char *argv[])
   }

   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 13. Save the refined mesh and the solution. This output can be viewed later
   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);
@@ -237,14 +232,15 @@ int main(int argc, char *argv[])
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
   }

   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;

   return 0;
 }
@@ -88,6 +88,8 @@ private:
   Vector funval2;
   Vector nor;
   Vector fluxN;
+   IntegrationPoint eip1;
+   IntegrationPoint eip2;

 public:
   FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -422,16 +424,19 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
   {
      const IntegrationPoint &ip = ir->IntPoint(i);

-      Tr.SetAllIntPoints(&ip); // set face and element int. points
+      Tr.Loc1.Transform(ip, eip1);
+      Tr.Loc2.Transform(ip, eip2);

      // Calculate basis functions on both elements at the face
-      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+      el1.CalcShape(eip1, shape1);
+      el2.CalcShape(eip2, shape2);

      // Interpolate elfun at the point
      elfun1_mat.MultTranspose(shape1, funval1);
      elfun2_mat.MultTranspose(shape2, funval2);

+      Tr.SetIntPoint(&ip);
+
      // Get the normal vector and the flux on the face
      CalcOrtho(Tr.Jacobian(), nor);
      const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);
@@ -32,8 +32,7 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -112,8 +111,8 @@ int main(int argc, char *argv[])
   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -121,23 +120,23 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
   {
      int par_ref_levels = 2;
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh.UniformRefinement();
+         pmesh->UniformRefinement();
      }
   }

@@ -145,16 +144,13 @@ int main(int argc, char *argv[])
   //    use continuous Lagrange finite elements of the specified order. If
   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (pmesh.GetNodes())
+   else if (pmesh->GetNodes())
   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = pmesh->GetNodes()->OwnFEC();
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -163,10 +159,9 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -177,44 +172,44 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
+   if (pmesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 9. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
+   ParLinearForm *b = new ParLinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 10. Define the solution vector x as a parallel finite element grid function
   //     corresponding to fespace. Initialize x with initial guess of zero,
   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
+   ParGridFunction x(fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
@@ -222,9 +217,9 @@ int main(int argc, char *argv[])
   Solver *prec = NULL;
   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
      }
   }
   else
@@ -242,7 +237,7 @@ int main(int argc, char *argv[])

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
@@ -253,7 +248,7 @@ int main(int argc, char *argv[])

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
+      pmesh->Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
@@ -268,14 +263,16 @@ int main(int argc, char *argv[])
      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
+      sol_sock << "solution\n" << *pmesh << x << flush;
   }

   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
   MPI_Finalize();

   return 0;
@@ -13,11 +13,6 @@
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -81,8 +76,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -113,10 +106,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -146,18 +135,13 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase resolution. In this example we do
+   // 3. Refine the mesh to increase resolution. In this example we do
   //    'ref_levels' of uniform refinement where the user specifies
   //    the number of levels with the '-r' option.
   for (int l = 0; l < ref_levels; l++)
@@ -165,7 +149,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
   //    order.
   if (dim == 1 && prob != 0 )
@@ -187,7 +171,7 @@ int main(int argc, char *argv[])
   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
        << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined based on the type
   //    of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -199,12 +183,12 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
   ComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 8. Define the solution vector u as a complex finite element grid function
+   // 7. Define the solution vector u as a complex finite element grid function
   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
   //    the exact solution if it is known.
   ComplexGridFunction u(fespace);
@@ -226,6 +210,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -278,7 +263,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 9. Set up the sesquilinear form a(.,.) on the finite element space
+   // 8. Set up the sesquilinear form a(.,.) on the finite element space
   //    corresponding to the damped harmonic oscillator operator of the
   //    appropriate type:
   //
@@ -297,7 +282,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -321,7 +305,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 9a. Set up the bilinear form for the preconditioner corresponding to the
+   // 8a. Set up the bilinear form for the preconditioner corresponding to the
   //     appropriate operator
   //
   //      0) A scalar H1 field
@@ -334,8 +318,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   BilinearForm *pcOp = new BilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
   switch (prob)
   {
      case 0:
@@ -356,9 +338,9 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 10. Assemble the form and the corresponding linear system, applying any
-   //     necessary transformations such as: assembly, eliminating boundary
-   //     conditions, conforming constraints for non-conforming AMR, etc.
+   // 9. Assemble the form and the corresponding linear system, applying any
+   //    necessary transformations such as: assembly, eliminating boundary
+   //    conditions, conforming constraints for non-conforming AMR, etc.
   a->Assemble();
   pcOp->Assemble();

@@ -366,17 +348,28 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;

-   cout << "Size of linear system: " << A->Width() << endl << endl;
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

-   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
+   {
+      ComplexSparseMatrix * Asp =
+         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
+
+      cout << "Size of linear system: "
+           << 2 * Asp->real().Width() << endl << endl;
+   }
+
+   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
   //     preconditioner based on the appropriate sparse smoother.
   {
      Array<int> blockOffsets;
      blockOffsets.SetSize(3);
      blockOffsets[0] = 0;
-      blockOffsets[1] = A->Height() / 2;
-      blockOffsets[2] = A->Height() / 2;
+      blockOffsets[1] = PCOp.Ptr()->Height();
+      blockOffsets[2] = PCOp.Ptr()->Height();
      blockOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockOffsets);
@@ -384,31 +377,22 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      double s = 1.0;
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
+         case 0:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+         case 1:
+            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
+            s = -1.0;
+            break;
+         case 2:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+
+         default: break; // This should be unreachable
      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 1:
-               pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 2:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            default:
-               break; // This should be unreachable
-         }
-      }
-      double s = (prob != 1) ? 1.0 : -1.0;
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
                                s:-s);
@@ -426,11 +410,9 @@ int main(int argc, char *argv[])
      gmres.Mult(B, U);
   }

-   // 12. Recover the solution as a finite element grid function and compute the
+   // 11. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a->RecoverFEMSolution(U, b, u);
-   u.real().SyncMemory(u);
-   u.imag().SyncMemory(u);

   if (exact_sol)
   {
@@ -460,7 +442,7 @@ int main(int argc, char *argv[])
      cout << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -475,7 +457,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -534,7 +516,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -13,11 +13,6 @@
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -46,6 +41,7 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.

+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -88,8 +84,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -122,10 +116,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -162,24 +152,19 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ser_ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -189,7 +174,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
   //    the specified order.
   if (dim == 1 && prob != 0 )
@@ -217,7 +202,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    based on the type of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -229,14 +214,14 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
   ParComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 10. Define the solution vector u as a parallel complex finite element grid
-   //     function corresponding to fespace. Initialize u with initial guess of
-   //     1+0i or the exact solution if it is known.
+   // 9. Define the solution vector u as a parallel complex finite element grid
+   //    function corresponding to fespace. Initialize u with initial guess of
+   //    1+0i or the exact solution if it is known.
   ParComplexGridFunction u(fespace);
   ParComplexGridFunction * u_exact = NULL;
   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -256,6 +241,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -310,7 +296,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
   //     space corresponding to the damped harmonic oscillator operator of the
   //     appropriate type:
   //
@@ -329,7 +315,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -353,7 +338,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 11a. Set up the parallel bilinear form for the preconditioner
+   // 10a. Set up the parallel bilinear form for the preconditioner
   //      corresponding to the appropriate operator
   //
   //      0) A scalar H1 field
@@ -366,7 +351,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   ParBilinearForm *pcOp = new ParBilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -387,7 +371,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
@@ -398,22 +382,30 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;
+
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

   if (myid == 0)
   {
+      ComplexHypreParMatrix * Ahyp =
+         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
+
      cout << "Size of linear system: "
-           << 2 * fespace->GlobalTrueVSize() << endl << endl;
+           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
   }

-   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
   //     diagonal preconditioner based on the appropriate multigrid
   //     preconditioner from hypre.
   {
      Array<int> blockTrueOffsets;
      blockTrueOffsets.SetSize(3);
      blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = A->Height() / 2;
-      blockTrueOffsets[2] = A->Height() / 2;
+      blockTrueOffsets[1] = PCOp.Ptr()->Height();
+      blockTrueOffsets[2] = PCOp.Ptr()->Height();
      blockTrueOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
@@ -421,34 +413,25 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
-      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-               break;
-            case 1:
+         case 0:
+            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+            break;
+         case 1:
+            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
+            break;
+         case 2:
+            if (dim == 2 )
+            {
               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               break;
-            case 2:
-               if (dim == 2 )
-               {
-                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               else
-               {
-                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               break;
-            default: break; // This should be unreachable
-         }
+            }
+            else
+            {
+               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
+            }
+            break;
+         default: break; // This should be unreachable
      }
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
@@ -466,11 +449,9 @@ int main(int argc, char *argv[])
      fgmres.SetPrintLevel(1);
      fgmres.Mult(B, U);
   }
-   // 14. Recover the parallel grid function corresponding to U. This is the
+   // 13. Recover the parallel grid function corresponding to U. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(U, b, u);
-   u.real().SyncMemory(u);
-   u.imag().SyncMemory(u);

   if (exact_sol)
   {
@@ -503,7 +484,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -523,7 +504,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -588,7 +569,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -7,7 +7,6 @@
 //               ex24 -m ../data/beam-tet.mesh
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               ex24 -m ../data/escher.mesh
 //               ex24 -m ../data/escher.mesh -o 2
 //               ex24 -m ../data/fichera.mesh
@@ -25,13 +24,12 @@
 //               ex24 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces, with two variants:
 //
 //               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
+//               2) (div v, q) for v in H(div) tested against q in L_2
 //
-//               Using different approaches, we project the gradient, curl, or
+//               Using different approaches, we project the gradient or
 //               divergence to the appropriate space.
 //
 //               We recommend viewing examples 1, 3, and 5 before viewing this
@@ -47,11 +45,8 @@ using namespace mfem;
 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
 double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -70,7 +65,7 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
+                  "Choose between 0: H(Curl) or 1: H(Div)");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -88,7 +83,6 @@ int main(int argc, char *argv[])
      return 1;
   }
   args.PrintOptions(cout);
-   kappa = freq * M_PI;

   // 2. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -125,15 +119,10 @@ int main(int argc, char *argv[])
      trial_fec = new H1_FECollection(order, dim);
      test_fec = new ND_FECollection(order, dim);
   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
   else
   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
+      trial_fec = new RT_FECollection(order - 1, dim);
+      test_fec = new L2_FECollection(order - 1, dim);
   }

   FiniteElementSpace trial_fes(mesh, trial_fec);
@@ -147,12 +136,6 @@ int main(int argc, char *argv[])
      cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
      cout << "Number of H1 finite element unknowns: " << trial_size << endl;
   }
-   else if (prob == 1)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-      cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-           endl;
-   }
   else
   {
      cout << "Number of Raviart-Thomas finite element unknowns: "
@@ -167,18 +150,12 @@ int main(int argc, char *argv[])
   GridFunction x(&test_fes);
   FunctionCoefficient p_coef(p_exact);
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
   FunctionCoefficient divgradp_coef(div_gradp_exact);

   if (prob == 0)
   {
      gftrial.ProjectCoefficient(p_coef);
   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
   else
   {
      gftrial.ProjectCoefficient(gradp_coef);
@@ -202,11 +179,6 @@ int main(int argc, char *argv[])
      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
   else
   {
      a.AddDomainIntegrator(new MassIntegrator(one));
@@ -272,10 +244,6 @@ int main(int argc, char *argv[])
   {
      dlo.AddDomainInterpolator(new GradientInterpolator());
   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
   else
   {
      dlo.AddDomainInterpolator(new DivergenceInterpolator());
@@ -290,10 +258,6 @@ int main(int argc, char *argv[])
   {
      exact_proj.ProjectCoefficient(gradp_coef);
   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
   else
   {
      exact_proj.ProjectCoefficient(divgradp_coef);
@@ -312,21 +276,8 @@ int main(int argc, char *argv[])
      cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
           "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
      cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-           " ||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-           "||_{L_2} = " << errProj << '\n' << endl;
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in H(div): "
-           "|| E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
           "||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
+      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
           "||_{L_2} = " << errProj << '\n' << endl;
   }
   else
@@ -344,7 +295,7 @@ int main(int argc, char *argv[])

      cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
           "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v "
+      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
           "||_{L_2} = " << errInterp << '\n' << endl;
      cout << " Projection f_h of exact div v in L_2: || f_h - div v "
           "||_{L_2} = " << errProj << '\n' << endl;
@@ -420,33 +371,3 @@ double div_gradp_exact(const Vector &x)

   return 0.0;
 }
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -6,8 +6,7 @@
 //               mpirun -np 4 ex24p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 2
+//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -p 1 -pa
 //               mpirun -np 4 ex24p -m ../data/escher.mesh
 //               mpirun -np 4 ex24p -m ../data/escher.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/fichera.mesh
@@ -25,13 +24,12 @@
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces, with two variants:
 //
 //               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
+//               2) (div v, q) for v in H(div) tested against q in L_2
 //
-//               Using different approaches, we project the gradient, curl, or
+//               Using different approaches, we project the gradient or
 //               divergence to the appropriate space.
 //
 //               We recommend viewing examples 1, 3, and 5 before viewing this
@@ -47,11 +45,8 @@ using namespace mfem;
 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
 double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -76,7 +71,7 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
+                  "Choose between 0: H(Curl) or 1: H(Div)");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -101,7 +96,6 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   kappa = freq * M_PI;

   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -153,15 +147,10 @@ int main(int argc, char *argv[])
      trial_fec = new H1_FECollection(order, dim);
      test_fec = new ND_FECollection(order, dim);
   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
   else
   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
+      trial_fec = new RT_FECollection(order - 1, dim);
+      test_fec = new L2_FECollection(order - 1, dim);
   }

   ParFiniteElementSpace trial_fes(pmesh, trial_fec);
@@ -177,12 +166,6 @@ int main(int argc, char *argv[])
         cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
         cout << "Number of H1 finite element unknowns: " << trial_size << endl;
      }
-      else if (prob == 1)
-      {
-         cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-         cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-              endl;
-      }
      else
      {
         cout << "Number of Raviart-Thomas finite element unknowns: "
@@ -198,18 +181,12 @@ int main(int argc, char *argv[])
   ParGridFunction x(&test_fes);
   FunctionCoefficient p_coef(p_exact);
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
   FunctionCoefficient divgradp_coef(div_gradp_exact);

   if (prob == 0)
   {
      gftrial.ProjectCoefficient(p_coef);
   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
   else
   {
      gftrial.ProjectCoefficient(gradp_coef);
@@ -233,11 +210,6 @@ int main(int argc, char *argv[])
      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
   else
   {
      a.AddDomainIntegrator(new MassIntegrator(one));
@@ -321,10 +293,6 @@ int main(int argc, char *argv[])
   {
      dlo.AddDomainInterpolator(new GradientInterpolator());
   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
   else
   {
      dlo.AddDomainInterpolator(new DivergenceInterpolator());
@@ -339,10 +307,6 @@ int main(int argc, char *argv[])
   {
      exact_proj.ProjectCoefficient(gradp_coef);
   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
   else
   {
      exact_proj.ProjectCoefficient(divgradp_coef);
@@ -360,27 +324,11 @@ int main(int argc, char *argv[])

      if (myid == 0)
      {
-         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl)"
-              ": || E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad"
-              " p ||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-              "||_{L_2} = " << errProj << '\n' << endl;
-      }
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in "
-              "H(div): || E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
+         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
+              "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
+         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
              "||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
+         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
              "||_{L_2} = " << errProj << '\n' << endl;
      }
   }
@@ -402,7 +350,7 @@ int main(int argc, char *argv[])
         cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
              "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
         cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
-              " ||_{L_2} = " << errInterp << '\n' << endl;
+              "||_{L_2} = " << errInterp << '\n' << endl;
         cout << " Projection f_h of exact div v in L_2: || f_h - div v "
              "||_{L_2} = " << errProj << '\n' << endl;
      }
@@ -488,33 +436,3 @@ double div_gradp_exact(const Vector &x)

   return 0.0;
 }
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -10,10 +10,6 @@
 //               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh
 //               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh
 //
-// Device sample runs:
-//               ex25 -o 2 -f 8.0 -ref 3 -prob 4 -m ../data/inline-quad.mesh -pa -d cuda
-//               ex25 -o 2 -f 2.0 -ref 1 -prob 4 -m ../data/inline-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -86,24 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -120,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -157,8 +153,6 @@ int main(int argc, char *argv[])
   double freq = 5.0;
   bool herm_conv = true;
   bool visualization = 1;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -180,21 +174,12 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();

   if (iprob > 4) { iprob = 4; }
   prob = (prob_type)iprob;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Setup the mesh
+   // 2. Setup the mesh
   if (!mesh_file)
   {
      exact_known = true;
@@ -235,7 +220,7 @@ int main(int argc, char *argv[])
   // Setup PML length
   Array2D<double> length(dim, 2); length = 0.0;

-   // 4. Setup the Cartesian PML region.
+   // 3. Setup the Cartesian PML region.
   switch (prob)
   {
      case disc:
@@ -261,19 +246,19 @@ int main(int argc, char *argv[])
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

-   // 5. Refine the mesh to increase the resolution.
+   // 4. Refine the mesh to increase the resolution.
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Reorient mesh in case of a tet mesh
+   // 5. Reorient mesh in case of a tet mesh
   mesh->ReorientTetMesh();

   // Set element attributes in order to distinguish elements in the PML region
   pml->SetAttributes(mesh);

-   // 7. Define a finite element space on the mesh. Here we use the Nedelec
+   // 6. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
@@ -281,7 +266,7 @@ int main(int argc, char *argv[])

   cout << "Number of finite element unknowns: " << size << endl;

-   // 8. Determine the list of true essential boundary dofs. In this example,
+   // 7. Determine the list of true essential boundary dofs. In this example,
   //    the boundary conditions are defined based on the specific mesh and the
   //    problem type.
   Array<int> ess_tdof_list;
@@ -323,12 +308,12 @@ int main(int argc, char *argv[])
   }
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 9. Setup Complex Operator convention
+   // 8. Setup Complex Operator convention
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 10. Set up the linear form b(.) which corresponds to the right-hand side of
-   //     the FEM linear system.
+   // 9. Set up the linear form b(.) which corresponds to the right-hand side of
+   //    the FEM linear system.
   VectorFunctionCoefficient f(dim, source);
   ComplexLinearForm b(fespace, conv);
   if (prob == load_src)
@@ -338,7 +323,7 @@ int main(int argc, char *argv[])
   b.Vector::operator=(0.0);
   b.Assemble();

-   // 11. Define the solution vector x as a complex finite element grid function
+   // 10. Define the solution vector x as a complex finite element grid function
   //     corresponding to fespace.
   ComplexGridFunction x(fespace);
   x = 0.0;
@@ -346,7 +331,7 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);

-   // 12. Set up the sesquilinear form a(.,.)
+   // 11. Set up the sesquilinear form a(.,.)
   //
   //     In Comp
   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -380,19 +365,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -400,29 +385,30 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                         new VectorFEMassIntegrator(restr_c2_Im));

-   // 13. Assemble the bilinear form and the corresponding linear system,
+   // 12. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: assembly, eliminating
   //     boundary conditions, applying conforming constraints for
   //     non-conforming AMR, etc.
-#ifndef MFEM_USE_SUITESPARSE
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-#endif
-   a.Assemble(0);
+   a.Assemble();

-   OperatorPtr A;
+   OperatorHandle Ah;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);
+
+   // 13. Transform to monolithic SparseMatrix
+   SparseMatrix *A = Ah.As<ComplexSparseMatrix>()->GetSystemMatrix();
+
+   cout << "Size of linear system: " << A->Height() << endl;

   // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
   {
-      if (pa) { cout << "PA not available with MFEM_USE_SUITESPARSE" << endl; }
-      ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
-      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      csolver.SetPrintLevel(1);
-      csolver.Mult(B, X);
+      UMFPackSolver  solver(*A);
+      solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      solver.Mult(B, X);
   }
 #else
+
   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
@@ -438,64 +424,45 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

-      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      prec.Assemble();

+      OperatorHandle PCOpAh;
+      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
-      //      preconditioner based on the Gauss-Seidel or Jacobi sparse smoother.
+      //      preconditioner based on the Gauss-Seidel sparse smoother.
      Array<int> offsets(3);
      offsets[0] = 0;
      offsets[1] = fespace->GetTrueVSize();
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
-      if (pa)
-      {
-         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
-      }
-      else
-      {
-         OperatorPtr PCOpAh;
-         prec.SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-         // Gauss-Seidel Smoother
-         GSSmoother *gs00 = new GSSmoother(*PCOpAh.As<SparseMatrix>());
-         ScaledOperator *gs11 = new ScaledOperator(gs00, s);
-         pc_r = gs00;
-         pc_i = gs11;
-      }
-
-      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      GSSmoother gs00(*PCOpAh.As<SparseMatrix>());
+      BlockDiagonalPreconditioner BlockGS(offsets);
+      ScaledOperator gs11(&gs00,
+                          (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
+      BlockGS.SetDiagonalBlock(0,&gs00);
+      BlockGS.SetDiagonalBlock(1,&gs11);

      GMRESSolver gmres;
      gmres.SetPrintLevel(1);
      gmres.SetKDim(200);
-      gmres.SetMaxIter(pa ? 5000 : 2000);
+      gmres.SetMaxIter(2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
      gmres.SetOperator(*A);
-      gmres.SetPreconditioner(BlockDP);
+      gmres.SetPreconditioner(BlockGS);
      gmres.Mult(B, X);
   }
 #endif
@@ -507,8 +474,10 @@ int main(int argc, char *argv[])
   // If exact is known compute the error
   if (exact_known)
   {
+      ComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -604,6 +573,7 @@ int main(int argc, char *argv[])
   }

   // 18. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -801,7 +771,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -812,13 +782,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -829,13 +800,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -846,13 +818,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -866,18 +839,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -890,18 +864,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -914,13 +889,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -10,10 +10,6 @@
 //               mpirun -np 4 ex25p -o 2 -f 8.0 -rs 2 -rp 2 -prob 4 -m ../data/inline-quad.mesh
 //               mpirun -np 4 ex25p -o 2 -f 2.0 -rs 1 -rp 1 -prob 4 -m ../data/inline-hex.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex25p -o 1 -f 3.0 -rs 3 -rp 1 -prob 2 -pa -d cuda
-//               mpirun -np 4 ex25p -o 2 -f 1.0 -rs 1 -rp 1 -prob 3 -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic wave
 //               propagation problem corresponding to the second order
 //               indefinite Maxwell equation
@@ -86,24 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -120,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -164,8 +160,6 @@ int main(int argc, char *argv[])
   double freq = 5.0;
   bool herm_conv = true;
   bool visualization = 1;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -189,21 +183,12 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();

   if (iprob > 4) { iprob = 4; }
   prob = (prob_type)iprob;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 4. Setup the (serial) mesh on all processors.
+   // 3. Setup the (serial) mesh on all processors.
   if (!mesh_file)
   {
      exact_known = true;
@@ -251,7 +236,7 @@ int main(int argc, char *argv[])
   // Setup PML length
   Array2D<double> length(dim, 2); length = 0.0;

-   // 5. Setup the Cartesian PML region.
+   // 4. Setup the Cartesian PML region.
   switch (prob)
   {
      case disc:
@@ -277,13 +262,13 @@ int main(int argc, char *argv[])
   comp_domain_bdr = pml->GetCompDomainBdr();
   domain_bdr = pml->GetDomainBdr();

-   // 6. Refine the serial mesh on all processors to increase the resolution.
+   // 5. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 7. Define a parallel mesh by a partitioning of the serial mesh.
+   // 6. Define a parallel mesh by a partitioning of the serial mesh.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
   {
@@ -293,13 +278,13 @@ int main(int argc, char *argv[])
      }
   }

-   // 7a. Reorient mesh in case of a tet mesh
+   // 6a. Reorient mesh in case of a tet mesh
   pmesh->ReorientTetMesh();

-   // 8. Set element attributes in order to distinguish elements in the PML
+   // 7. Set element attributes in order to distinguish elements in the PML
   pml->SetAttributes(pmesh);

-   // 9. Define a parallel finite element space on the parallel mesh. Here we
+   // 8. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -309,9 +294,9 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 10. Determine the list of true (i.e. parallel conforming) essential
-   //     boundary dofs. In this example, the boundary conditions are defined
-   //     based on the specific mesh and the problem type.
+   // 9. Determine the list of true (i.e. parallel conforming) essential
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    based on the specific mesh and the problem type.
   Array<int> ess_tdof_list;
   Array<int> ess_bdr;
   if (pmesh->bdr_attributes.Size())
@@ -351,11 +336,11 @@ int main(int argc, char *argv[])
   }
   fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-   // 11. Setup Complex Operator convention
+   // 10. Setup Complex Operator convention
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 12. Set up the parallel linear form b(.) which corresponds to the
+   // 11. Set up the parallel linear form b(.) which corresponds to the
   //     right-hand side of the FEM linear system.
   VectorFunctionCoefficient f(dim, source);
   ParComplexLinearForm b(fespace, conv);
@@ -366,7 +351,7 @@ int main(int argc, char *argv[])
   b.Vector::operator=(0.0);
   b.Assemble();

-   // 13. Define the solution vector x as a parallel complex finite element grid
+   // 12. Define the solution vector x as a parallel complex finite element grid
   //     function corresponding to fespace.
   ParComplexGridFunction x(fespace);
   x = 0.0;
@@ -374,7 +359,7 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E_Im(dim, E_bdr_data_Im);
   x.ProjectBdrCoefficientTangent(E_Re, E_Im, ess_bdr);

-   // 14. Set up the parallel sesquilinear form a(.,.)
+   // 13. Set up the parallel sesquilinear form a(.,.)
   //
   //     In Comp
   //     Domain:   1/mu (Curl E, Curl F) - omega^2 * epsilon (E,F)
@@ -408,19 +393,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -428,25 +413,27 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_Re),
                         new VectorFEMassIntegrator(restr_c2_Im));

-   // 15. Assemble the parallel bilinear form and the corresponding linear
+   // 14. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
-#ifndef MFEM_USE_SUPERLU
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-#endif
   a.Assemble();

-   OperatorPtr Ah;
+   OperatorHandle Ah;
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

+   // 15. Transform to monolithic HypreParMatrix
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A->GetGlobalNumRows() << endl;
+   }
+
   // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
   {
-      if (pa) { cout << "PA not available with MFEM_USE_SUPERLU" << endl; }
-      // Transform to monolithic HypreParMatrix
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
      SuperLURowLocMatrix SA(*A);
      SuperLUSolver superlu(MPI_COMM_WORLD);
      superlu.SetPrintStatistics(false);
@@ -454,9 +441,9 @@ int main(int argc, char *argv[])
      superlu.SetColumnPermutation(superlu::PARMETIS);
      superlu.SetOperator(SA);
      superlu.Mult(B, X);
-      delete A;
   }
 #else
+
   // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
@@ -472,20 +459,22 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

-      if (pa) { prec.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      prec.Assemble();

+      OperatorHandle PCOpAh;
+      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
+
      // 16b. Define and apply a parallel GMRES solver for AU=B with a block
      //      diagonal preconditioner based on hypre's AMS preconditioner.
      Array<int> offsets(3);
@@ -494,41 +483,21 @@ int main(int argc, char *argv[])
      offsets[2] = fespace->GetTrueVSize();
      offsets.PartialSum();

-      Operator *pc_r = nullptr;
-      Operator *pc_i = nullptr;
-      int s = (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0;
-      if (pa)
-      {
-         // Jacobi Smoother
-         OperatorJacobiSmoother *d00 = new OperatorJacobiSmoother(prec, ess_tdof_list);
-         ScaledOperator *d11 = new ScaledOperator(d00, s);
-         pc_r = d00;
-         pc_i = d11;
-      }
-      else
-      {
-         OperatorPtr PCOpAh;
-         prec.FormSystemMatrix(ess_tdof_list, PCOpAh);
-
-         // Hypre AMS
-         HypreAMS *ams00 = new HypreAMS(*PCOpAh.As<HypreParMatrix>(), fespace);
-         ScaledOperator *ams11 = new ScaledOperator(ams00, s);
-         pc_r = ams00;
-         pc_i = ams11;
-      }
-
-      BlockDiagonalPreconditioner BlockDP(offsets);
-      BlockDP.SetDiagonalBlock(0, pc_r);
-      BlockDP.SetDiagonalBlock(1, pc_i);
+      HypreAMS ams00(*PCOpAh.As<HypreParMatrix>(),fespace);
+      BlockDiagonalPreconditioner BlockAMS(offsets);
+      ScaledOperator ams11(&ams00,
+                           (conv == ComplexOperator::HERMITIAN) ? -1.0 : 1.0);
+      BlockAMS.SetDiagonalBlock(0,&ams00);
+      BlockAMS.SetDiagonalBlock(1,&ams11);

      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetPrintLevel(1);
      gmres.SetKDim(200);
-      gmres.SetMaxIter(pa ? 5000 : 2000);
+      gmres.SetMaxIter(2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
-      gmres.SetOperator(*Ah);
-      gmres.SetPreconditioner(BlockDP);
+      gmres.SetOperator(*A);
+      gmres.SetPreconditioner(BlockAMS);
      gmres.Mult(B, X);
   }
 #endif
@@ -540,8 +509,10 @@ int main(int argc, char *argv[])
   // If exact is known compute the error
   if (exact_known)
   {
+      ParComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -658,6 +629,7 @@ int main(int argc, char *argv[])
   }

   // 20. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -856,7 +828,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -867,13 +839,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -884,13 +857,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -901,13 +875,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -921,18 +896,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -945,18 +921,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -969,13 +946,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -11,12 +11,6 @@
 //               ex5 -m ../data/escher.mesh
 //               ex5 -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               ex5 -m ../data/star.mesh -pa -d cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-omp
-//               ex5 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -56,7 +50,6 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -66,8 +59,6 @@ int main(int argc, char *argv[])
                  "Finite element order (polynomial degree).");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -79,18 +70,13 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
@@ -103,7 +89,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use the
+   // 4. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -111,7 +97,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, hdiv_coll);
   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, l2_coll);

-   // 6. Define the BlockStructure of the problem, i.e. define the array of
+   // 5. Define the BlockStructure of the problem, i.e. define the array of
   //    offsets for each variable. The last component of the Array is the sum
   //    of the dimensions of each block.
   Array<int> block_offsets(3); // number of variables + 1
@@ -126,7 +112,7 @@ int main(int argc, char *argv[])
   std::cout << "dim(R+W) = " << block_offsets.Last() << "\n";
   std::cout << "***********************************************************\n";

-   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 6. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -136,28 +122,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
+   // 7. Allocate memory (x, rhs) for the analytical solution and the right hand
   //    side.  Define the GridFunction u,p for the finite element solution and
   //    linear forms fform and gform for the right hand side.  The data
   //    allocated by x and rhs are passed as a reference to the grid functions
   //    (u,p) and the linear forms (fform, gform).
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
+   BlockVector x(block_offsets), rhs(block_offsets);

   LinearForm *fform(new LinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);

   LinearForm *gform(new LinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);

-   // 9. Assemble the finite element matrices for the Darcy operator
+   // 8. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -202,7 +185,7 @@ int main(int argc, char *argv[])
      darcyOp.SetBlock(1,0, &B);
   }

-   // 10. Construct the operators for preconditioner
+   // 9. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
@@ -219,11 +202,10 @@ int main(int argc, char *argv[])
   if (pa)
   {
      mVarf->AssembleDiagonal(Md);
-      auto Md_host = Md.HostRead();
      Vector invMd(mVarf->Height());
      for (int i=0; i<mVarf->Height(); ++i)
      {
-         invMd(i) = 1.0 / Md_host[i];
+         invMd(i) = 1.0 / Md(i);
      }

      Vector BMBt_diag(bVarf->Height());
@@ -264,7 +246,7 @@ int main(int argc, char *argv[])
   darcyPrec.SetDiagonalBlock(0, invM);
   darcyPrec.SetDiagonalBlock(1, invS);

-   // 11. Solve the linear system with MINRES.
+   // 10. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(1000);
   double rtol(1.e-6);
@@ -281,7 +263,6 @@ int main(int argc, char *argv[])
   solver.SetPrintLevel(1);
   x = 0.0;
   solver.Mult(rhs, x);
-   if (device.IsEnabled()) { x.HostRead(); }
   chrono.Stop();

   if (solver.GetConverged())
@@ -292,7 +273,7 @@ int main(int argc, char *argv[])
                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

-   // 12. Create the grid functions u and p. Compute the L2 error norms.
+   // 11. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
   u.MakeRef(R_space, x.GetBlock(0), 0);
   p.MakeRef(W_space, x.GetBlock(1), 0);
@@ -312,7 +293,7 @@ int main(int argc, char *argv[])
   std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
   std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";

-   // 13. Save the mesh and the solution. This output can be viewed later using
+   // 12. Save the mesh and the solution. This output can be viewed later using
   //     GLVis: "glvis -m ex5.mesh -g sol_u.gf" or "glvis -m ex5.mesh -g
   //     sol_p.gf".
   {
@@ -329,13 +310,13 @@ int main(int argc, char *argv[])
      p.Save(p_ofs);
   }

-   // 14. Save data in the VisIt format
+   // 13. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5", mesh);
   visit_dc.RegisterField("velocity", &u);
   visit_dc.RegisterField("pressure", &p);
   visit_dc.Save();

-   // 15. Save data in the ParaView format
+   // 14. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5", mesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -347,7 +328,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",&p);
   paraview_dc.Save();

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -360,7 +341,7 @@ int main(int argc, char *argv[])
      p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete fform;
   delete gform;
   delete invM;
@@ -11,12 +11,6 @@
 //               mpirun -np 4 ex5p -m ../data/escher.mesh
 //               mpirun -np 4 ex5p -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-omp
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -66,7 +60,6 @@ int main(int argc, char *argv[])
   int order = 1;
   bool par_format = false;
   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;
   bool adios2 = false;

@@ -82,8 +75,6 @@ int main(int argc, char *argv[])
                  "Format to use when saving the results for VisIt.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -105,18 +96,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 10,000 elements, unless the user specifies it as input.
@@ -132,7 +118,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -145,7 +131,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -165,7 +151,7 @@ int main(int argc, char *argv[])
      std::cout << "***********************************************************\n";
   }

-   // 8. Define the two BlockStructure of the problem.  block_offsets is used
+   // 7. Define the two BlockStructure of the problem.  block_offsets is used
   //    for Vector based on dof (like ParGridFunction or ParLinearForm),
   //    block_trueOffstes is used for Vector based on trueDof (HypreParVector
   //    for the rhs and solution of the linear system).  The offsets computed
@@ -182,7 +168,7 @@ int main(int argc, char *argv[])
   block_trueOffsets[2] = W_space->TrueVSize();
   block_trueOffsets.PartialSum();

-   // 9. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 8. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -192,30 +178,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 10. Define the parallel grid function and parallel linear forms, solution
-   //     vector and rhs.
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
+   // 9. Define the parallel grid function and parallel linear forms, solution
+   //    vector and rhs.
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);

   ParLinearForm *fform(new ParLinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);
   fform->ParallelAssemble(trueRhs.GetBlock(0));
-   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);

   ParLinearForm *gform(new ParLinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);
   gform->ParallelAssemble(trueRhs.GetBlock(1));
-   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);

-   // 11. Assemble the finite element matrices for the Darcy operator
+   // 10. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -268,7 +249,7 @@ int main(int argc, char *argv[])
      darcyOp->SetBlock(1,0, B);
   }

-   // 12. Construct the operators for preconditioner
+   // 11. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
@@ -285,11 +266,10 @@ int main(int argc, char *argv[])
   {
      Md_PA.SetSize(R_space->GetTrueVSize());
      mVarf->AssembleDiagonal(Md_PA);
-      auto Md_host = Md_PA.HostRead();
      Vector invMd(Md_PA.Size());
      for (int i=0; i<Md_PA.Size(); ++i)
      {
-         invMd(i) = 1.0 / Md_host[i];
+         invMd(i) = 1.0 / Md_PA(i);
      }

      Vector BMBt_diag(W_space->GetTrueVSize());
@@ -322,7 +302,7 @@ int main(int argc, char *argv[])
   darcyPr->SetDiagonalBlock(0, invM);
   darcyPr->SetDiagonalBlock(1, invS);

-   // 13. Solve the linear system with MINRES.
+   // 12. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(pa ? 1000 : 500);
   double rtol(1.e-6);
@@ -339,7 +319,6 @@ int main(int argc, char *argv[])
   solver.SetPrintLevel(verbose);
   trueX = 0.0;
   solver.Mult(trueRhs, trueX);
-   if (device.IsEnabled()) { trueX.HostRead(); }
   chrono.Stop();

   if (verbose)
@@ -353,7 +332,7 @@ int main(int argc, char *argv[])
      std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
   }

-   // 14. Extract the parallel grid function corresponding to the finite element
+   // 13. Extract the parallel grid function corresponding to the finite element
   //     approximation X. This is the local solution on each processor. Compute
   //     L2 error norms.
   ParGridFunction *u(new ParGridFunction);
@@ -381,7 +360,7 @@ int main(int argc, char *argv[])
      std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_*".
   {
      ostringstream mesh_name, u_name, p_name;
@@ -402,7 +381,7 @@ int main(int argc, char *argv[])
      p->Save(p_ofs);
   }

-   // 16. Save data in the VisIt format
+   // 15. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5-Parallel", pmesh);
   visit_dc.RegisterField("velocity", u);
   visit_dc.RegisterField("pressure", p);
@@ -411,7 +390,7 @@ int main(int argc, char *argv[])
                      DataCollection::PARALLEL_FORMAT);
   visit_dc.Save();

-   // 17. Save data in the ParaView format
+   // 16. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5P", pmesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -423,7 +402,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",p);
   paraview_dc.Save();

-   // 18. Optionally output a BP (binary pack) file using ADIOS2. This can be
+   // 17. Optionally output a BP (binary pack) file using ADIOS2. This can be
   //     visualized with the ParaView VTX reader.
 #ifdef MFEM_USE_ADIOS2
   if (adios2)
@@ -443,7 +422,7 @@ int main(int argc, char *argv[])
   }
 #endif

-   // 19. Send the solution by socket to a GLVis server.
+   // 18. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -463,7 +442,7 @@ int main(int argc, char *argv[])
             << endl;
   }

-   // 20. Free the used memory.
+   // 19. Free the used memory.
   delete fform;
   delete gform;
   delete u;
@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//             * ex6 -pa -d ceed-cuda
+//               * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//             * mpirun -np 4 ex6p -pa -d ceed-cuda
+//               * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -20,11 +20,8 @@
 // Device sample runs:
 //    ex9 -pa
 //    ex9 -ea
-//    ex9 -fa
 //    ex9 -pa -m ../data/periodic-cube.mesh
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -147,7 +144,6 @@ int main(int argc, char *argv[])
   int order = 3;
   bool pa = false;
   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -174,8 +170,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -284,11 +278,6 @@ int main(int argc, char *argv[])
      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
@@ -448,19 +437,21 @@ int main(int argc, char *argv[])
 FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
+   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+   bool ea = M.GetAssemblyLevel() == AssemblyLevel::ELEMENT;
   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
+   if (pa || ea)
   {
      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
      M_solver.SetOperator(M);
      dg_solver = NULL;
   }
+   else
+   {
+      M_prec = new DSmoother(M.SpMat());
+      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
+      M_solver.SetOperator(M.SpMat());
+   }
   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
@@ -21,11 +21,8 @@
 // Device sample runs:
 //    mpirun -np 4 ex9p -pa
 //    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -167,7 +164,6 @@ int main(int argc, char *argv[])
   int order = 3;
   bool pa = false;
   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -197,8 +193,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -335,12 +329,6 @@ int main(int argc, char *argv[])
      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
   m->AddDomainIntegrator(new MassIntegrator);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
@@ -577,21 +565,29 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
+   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
+   bool ea = _M.GetAssemblyLevel()==AssemblyLevel::ELEMENT;
+
+   if (pa || ea)
   {
      M.Reset(&_M, false);
      K.Reset(&_K, false);
   }
+   else
+   {
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
+   }

   M_solver.SetOperator(*M);

   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
+   if (pa || ea)
+   {
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
+      dg_solver = NULL;
+   }
+   else
   {
      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -600,11 +596,6 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,

      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }

   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
@@ -114,11 +114,6 @@ ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-ex11p-test-superlu: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
-test-par-YES: ex11p-test-superlu
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -34,15 +34,6 @@ if (MFEM_USE_MPI)
    )
 endif()

-if (MFEM_USE_SLEPC)
-  list(APPEND PETSC_EXAMPLES_SRCS
-    ex11p.cpp
-    )
-  list(APPEND PETSC_RC_FILES
-    rc_ex11p_lobpcg rc_ex11p_gd
-    )
-endif()
-
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})

@@ -87,22 +78,12 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
-if (MFEM_USE_SLEPC)
-  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
-  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
-  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
-endif()

 # Add the tests: one test per command-line-variable.
-set(TEST_OPTIONS_VARS
-    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-endif()
-
-foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+foreach(TEST_OPTIONS_VAR
+        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
@@ -1,440 +0,0 @@
-//                       MFEM Example 11 - Parallel Version
-//                              PETSc Modification
-//
-// Compile with: make ex11p
-//
-// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
-//
-// Description:  This example code demonstrates the use of MFEM to solve the
-//               eigenvalue problem -Delta u = lambda u with homogeneous
-//               Dirichlet boundary conditions.
-//
-//               We compute a number of the lowest eigenmodes by discretizing
-//               the Laplacian and Mass operators using a FE space of the
-//               specified order, or an isoparametric/isogeometric space if
-//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example demonstrates the use of the SLEPc eigensolver as an
-//               alternative to the LOBPCG eigenvalue solver. The shift and
-//               invert spectral transformation is used to help the convergence
-//               to the smaller eigenvalues. Alternative solver parameters can
-//               be passed in a file with "-slepcopts".
-//
-//               Reusing a single GLVis visualization window for multiple
-//               eigenfunctions is also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_SLEPC
-#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   int nev = 5;
-   int seed = 75;
-   bool slu_solver  = false;
-   bool sp_solver = false;
-   bool visualization = 1;
-   bool use_slepc = true;
-   const char *slepcrc_file = "";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-   args.AddOption(&seed, "-s", "--seed",
-                  "Random seed used to initialize LOBPCG.");
-#ifdef MFEM_USE_SUPERLU
-   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
-                  "--no-superlu", "Use the SuperLU Solver.");
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
-                  "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
-                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
-   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
-                  "SlepcOptions file to use.");
-   args.Parse();
-   if (slu_solver && sp_solver)
-   {
-      if (myid == 0)
-         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
-              << " please choose either one." << endl
-              << "         Defaulting to SuperLU." << endl;
-      sp_solver = false;
-   }
-   // The command line options are also passed to the STRUMPACK
-   // solver. So do not exit if some options are not recognized.
-   if (!sp_solver)
-   {
-      if (!args.Good())
-      {
-         if (myid == 0)
-         {
-            args.PrintUsage(cout);
-         }
-         MPI_Finalize();
-         return 1;
-      }
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
-   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
-   //    specified on the command line with -rs).
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution (1 time by
-   //    default, or specified on the command line with -rp). Once the parallel
-   //    mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << size << endl;
-   }
-
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
-   //    element space. The first corresponds to the Laplacian operator -Delta,
-   //    while the second is a simple mass matrix needed on the right hand side
-   //    of the generalized eigenvalue problem below. The boundary conditions
-   //    are implemented by elimination with special values on the diagonal to
-   //    shift the Dirichlet eigenvalues out of the computational range. After
-   //    serial and parallel assembly we extract the corresponding parallel
-   //    matrices A and M.
-   ConstantCoefficient one(1.0);
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   if (pmesh->bdr_attributes.Size() == 0)
-   {
-      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
-      // closed surface.
-      a->AddDomainIntegrator(new MassIntegrator(one));
-   }
-   a->Assemble();
-   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
-   a->Finalize();
-
-   ParBilinearForm *m = new ParBilinearForm(fespace);
-   m->AddDomainIntegrator(new MassIntegrator(one));
-   m->Assemble();
-   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
-   m->Finalize();
-
-   PetscParMatrix *pA = NULL, *pM = NULL;
-   HypreParMatrix *A = NULL, *M = NULL;
-   Operator::Type tid =
-      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
-   OperatorHandle Ah(tid), Mh(tid);
-
-   a->ParallelAssemble(Ah);
-   if (!use_slepc) { Ah.Get(A); }
-   else { Ah.Get(pA); }
-   Ah.SetOperatorOwner(false);
-
-   m->ParallelAssemble(Mh);
-   if (!use_slepc) {Mh.Get(M); }
-   else {Mh.Get(pM); }
-   Mh.SetOperatorOwner(false);
-
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   Operator * Arow = NULL;
-#ifdef MFEM_USE_SUPERLU
-   if (slu_solver)
-   {
-      Arow = new SuperLURowLocMatrix(*A);
-   }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   if (sp_solver)
-   {
-      Arow = new STRUMPACKRowLocMatrix(*A);
-   }
-#endif
-#endif
-
-   delete a;
-   delete m;
-
-   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
-   //    preconditioner for A to be used within the solver. Set the matrices
-   //    which define the generalized eigenproblem A x = lambda M x.
-   Solver * precond = NULL;
-   if (!use_slepc)
-   {
-      if (!slu_solver && !sp_solver)
-      {
-         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
-         amg->SetPrintLevel(0);
-         precond = amg;
-      }
-      else
-      {
-#ifdef MFEM_USE_SUPERLU
-         if (slu_solver)
-         {
-            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-            superlu->SetPrintStatistics(false);
-            superlu->SetSymmetricPattern(true);
-            superlu->SetColumnPermutation(superlu::PARMETIS);
-            superlu->SetOperator(*Arow);
-            precond = superlu;
-         }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-         if (sp_solver)
-         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpack->SetPrintFactorStatistics(true);
-            strumpack->SetPrintSolveStatistics(false);
-            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
-            strumpack->SetOperator(*Arow);
-            strumpack->SetFromCommandLine();
-            precond = strumpack;
-         }
-#endif
-      }
-   }
-
-   HypreLOBPCG * lobpcg = NULL;
-   SlepcEigenSolver * slepc = NULL;
-   if (!use_slepc)
-   {
-
-      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-      lobpcg->SetNumModes(nev);
-      lobpcg->SetRandomSeed(seed);
-      lobpcg->SetPreconditioner(*precond);
-      lobpcg->SetMaxIter(200);
-      lobpcg->SetTol(1e-8);
-      lobpcg->SetPrecondUsageMode(1);
-      lobpcg->SetPrintLevel(1);
-      lobpcg->SetMassMatrix(*M);
-      lobpcg->SetOperator(*A);
-   }
-   else
-   {
-      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
-      slepc->SetNumModes(nev);
-      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
-      slepc->SetTarget(0.0);
-      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
-      slepc->SetOperators(*pA,*pM);
-   }
-
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
-   Array<double> eigenvalues;
-   if (!use_slepc)
-   {
-      lobpcg->Solve();
-      lobpcg->GetEigenvalues(eigenvalues);
-   }
-   else
-   {
-      slepc->Solve();
-      eigenvalues.SetSize(nev);
-      for (int i=0; i<nev; i++)
-      {
-         slepc->GetEigenvalue(i,eigenvalues[i]);
-      }
-   }
-   Vector temp(fespace->GetTrueVSize());
-   ParGridFunction x(fespace);
-
-   // 10. Save the refined mesh and the modes in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
-   {
-      ostringstream mesh_name, mode_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      for (int i=0; i<nev; i++)
-      {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-
-         }
-
-         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
-                   << setfill('0') << setw(6) << myid;
-
-         ofstream mode_ofs(mode_name.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         mode_name.str("");
-      }
-   }
-
-   // 11. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<nev; i++)
-      {
-         if ( myid == 0 )
-         {
-            cout << "Eigenmode " << i+1 << '/' << nev
-                 << ", Lambda = " << eigenvalues[i] << endl;
-         }
-
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-         }
-
-         mode_sock << "parallel " << num_procs << " " << myid << "\n"
-                   << "solution\n" << *pmesh << x << flush
-                   << "window_title 'Eigenmode " << i+1 << '/' << nev
-                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
-
-         char c;
-         if (myid == 0)
-         {
-            cout << "press (q)uit or (c)ontinue --> " << flush;
-            cin >> c;
-         }
-         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 12. Free the used memory.
-   if (!use_slepc)
-   {
-      delete lobpcg;
-   }
-   else
-   {
-      delete slepc;
-   }
-   delete precond;
-   delete M;
-   delete A;
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   delete Arow;
-#endif
-
-   delete fespace;
-   if (order > 0)
-   {
-      delete fec;
-   }
-   delete pmesh;
-
-   // We finalize SLEPc
-   MFEMFinalizeSlepc();
-   MPI_Finalize();
-
-   return 0;
-}
@@ -23,9 +23,6 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
-ifeq ($(MFEM_USE_SLEPC),YES)
-   PAR_EXAMPLES += ex11p
-endif
 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -90,9 +87,6 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
-EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
-EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd

 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -120,12 +114,6 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
-ifeq ($(MFEM_USE_SLEPC),YES)
-ex11p-test-par: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -1,6 +0,0 @@
-# Options for the eigenvalue solver
-eps_view
-eps_converged_reason
-eps_type gd
-# Options for the spectral transform
-st_type precond
@@ -1,11 +0,0 @@
-# Options for the eigenvalue solver
-eps_monitor
-eps_converged_reason
-eps_view_values
-eps_type lobpcg
-eps_gen_hermitian
-eps_smallest_real
-eps_lobpcg_blocksize 5
-# Options for the spectral transform
-st_type precond
-st_pc_type gamg
@@ -76,7 +76,7 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
   precompute_sparsity = 0;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;
 }
@@ -94,7 +94,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   precompute_sparsity = ps;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;

@@ -121,10 +121,9 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
-         ext = new FABilinearFormExtension(this);
+         // ext = new FABilinearFormExtension(this);
+         // Use the original BilinearForm implementation for now
         break;
      case AssemblyLevel::ELEMENT:
         ext = new EABilinearFormExtension(this);
@@ -144,7 +143,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 void BilinearForm::EnableStaticCondensation()
 {
   delete static_cond;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      static_cond = NULL;
      MFEM_WARNING("Static condensation not supported for this assembly level");
@@ -169,7 +168,7 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                       const Array<int> &ess_tdof_list)
 {
   delete hybridization;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      delete constr_integ;
      hybridization = NULL;
@@ -224,7 +223,7 @@ MatrixInverse * BilinearForm::Inverse() const

 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      if (!static_cond) { mat->Finalize(skip_zeros); }
      if (mat_e) { mat_e->Finalize(skip_zeros); }
@@ -640,7 +639,8 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
   }
   else
   {
-      mat->GetDiag(diag);
+      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
+                 "matrix and use SparseMatrix::GetDiag?");
   }
 }

@@ -1083,7 +1083,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 0;
-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1108,7 +1108,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   bbfi_marker = mbf->bbfi_marker;
   btfbfi_marker = mbf->btfbfi_marker;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1121,8 +1121,6 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
         // ext = new FAMixedBilinearFormExtension(this);
         // Use the original BilinearForm implementation for now
@@ -1193,7 +1191,7 @@ void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
      return NULL;
@@ -1206,7 +1204,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      mat -> Finalize (skip_zeros);
   }
@@ -1483,7 +1481,7 @@ void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,

 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
      return;
@@ -29,11 +29,8 @@ namespace mfem
    form classes derived from Operator. */
 enum class AssemblyLevel
 {
-   /// Legacy fully assembled form, i.e. a global sparse matrix in MFEM, Hypre
-   /// or PETSC format. This assembly is ALWAYS performed on the host.
-   LEGACYFULL = 0,
-   /// Fully assembled form, i.e. a global sparse matrix in MFEM format. This
-   /// assembly is compatible with device execution.
+   /// Fully assembled form, i.e. a global sparse matrix in MFEM, Hypre or PETSC
+   /// format.
   FULL,
   /// Form assembled at element level, which computes and stores dense element
   /// matrices.
@@ -122,7 +119,7 @@ protected:
      static_cond = NULL; hybridization = NULL;
      precompute_sparsity = 0;
      diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::LEGACYFULL;
+      assembly = AssemblyLevel::FULL;
      batch = 1;
      ext = NULL;
   }
@@ -15,7 +15,6 @@
 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
 #include "libceed/ceed.hpp"
-#include "pgridfunc.hpp"

 namespace mfem
 {
@@ -293,8 +292,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const

 // Data and methods for element-assembled bilinear forms
 EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
-   : PABilinearFormExtension(form),
-     factorize_face_terms(form->FESpace()->IsDGSpace())
+   : PABilinearFormExtension(form)
 {
 }

@@ -349,17 +347,6 @@ void EABilinearFormExtension::Assemble()
   {
      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
   }
-
-   if (factorize_face_terms && int_face_restrict_lex)
-   {
-      auto restFint = dynamic_cast<const L2FaceRestriction&>(*int_face_restrict_lex);
-      restFint.AddFaceMatricesToElementMatrices(ea_data_int, ea_data);
-   }
-   if (factorize_face_terms && bdr_face_restrict_lex)
-   {
-      auto restFbdr = dynamic_cast<const L2FaceRestriction&>(*bdr_face_restrict_lex);
-      restFbdr.AddFaceMatricesToElementMatrices(ea_data_bdr, ea_data);
-   }
 }

 void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
@@ -412,27 +399,24 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         const int NDOFS = faceDofs;
         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
+         auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
+         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            const int f = glob_j/NDOFS;
+            const int j = glob_j%NDOFS;
+            double res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
+               res += A_int(i, j, 0, f)*X(i, 0, f);
+            }
+            Y(j, 0, f) += res;
+            res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
+            {
+               res += A_int(i, j, 1, f)*X(i, 1, f);
+            }
+            Y(j, 1, f) += res;
+         });
         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
@@ -459,7 +443,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   // Treatment of boundary faces
   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
+   if (bdr_face_restrict_lex && bFISz>0)
   {
      // Apply the Boundary Face Restriction
      bdr_face_restrict_lex->Mult(x, faceBdrX);
@@ -538,27 +522,24 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         const int NDOFS = faceDofs;
         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
+         auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
+         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            const int f = glob_j/NDOFS;
+            const int j = glob_j%NDOFS;
+            double res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
+               res += A_int(j, i, 0, f)*X(i, 0, f);
+            }
+            Y(j, 0, f) += res;
+            res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
+            {
+               res += A_int(j, i, 1, f)*X(i, 1, f);
+            }
+            Y(j, 1, f) += res;
+         });
         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
@@ -585,7 +566,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   // Treatment of boundary faces
   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
+   if (bdr_face_restrict_lex && bFISz>0)
   {
      // Apply the Boundary Face Restriction
      bdr_face_restrict_lex->Mult(x, faceBdrX);
@@ -614,139 +595,6 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   }
 }

-// Data and methods for fully-assembled bilinear forms
-FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
-   : EABilinearFormExtension(form),
-     mat(form->FESpace()->GetVSize(),form->FESpace()->GetVSize(),0),
-     face_mat(form->FESpace()->GetVSize(),0,0),
-     use_face_mat(false)
-{
-#ifdef MFEM_USE_MPI
-   if ( ParFiniteElementSpace* pfes =
-           dynamic_cast<ParFiniteElementSpace*>(form->FESpace()) )
-   {
-      if (pfes->IsDGSpace())
-      {
-         use_face_mat = true;
-         pfes->ExchangeFaceNbrData();
-         face_mat.SetWidth(pfes->GetFaceNbrVSize());
-      }
-   }
-#endif
-}
-
-void FABilinearFormExtension::Assemble()
-{
-   EABilinearFormExtension::Assemble();
-   FiniteElementSpace &fes = *a->FESpace();
-   if (fes.IsDGSpace())
-   {
-      const L2ElementRestriction *restE =
-         static_cast<const L2ElementRestriction*>(elem_restrict);
-      const L2FaceRestriction *restF =
-         static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
-      // 1. Fill I
-      //  1.1 Increment with restE
-      restE->FillI(mat);
-      //  1.2 Increment with restF
-      if (restF) { restF->FillI(mat, face_mat); }
-      //  1.3 Sum the non-zeros in I
-      auto h_I = mat.HostReadWriteI();
-      int cpt = 0;
-      const int vd = fes.GetVDim();
-      const int ndofs = ne*elemDofs*vd;
-      for (int i = 0; i < ndofs; i++)
-      {
-         const int nnz = h_I[i];
-         h_I[i] = cpt;
-         cpt += nnz;
-      }
-      const int nnz = cpt;
-      h_I[ndofs] = nnz;
-      mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
-      mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
-      if (use_face_mat && restF)
-      {
-         auto h_I_face = face_mat.HostReadWriteI();
-         int cpt = 0;
-         for (int i = 0; i < ndofs; i++)
-         {
-            const int nnz = h_I_face[i];
-            h_I_face[i] = cpt;
-            cpt += nnz;
-         }
-         const int nnz_face = cpt;
-         h_I_face[ndofs] = nnz_face;
-         face_mat.GetMemoryJ().New(nnz_face,
-                                   face_mat.GetMemoryJ().GetMemoryType());
-         face_mat.GetMemoryData().New(nnz_face,
-                                      face_mat.GetMemoryData().GetMemoryType());
-      }
-      // 2. Fill J and Data
-      // 2.1 Fill J and Data with Elem ea_data
-      restE->FillJAndData(ea_data, mat);
-      // 2.2 Fill J and Data with Face ea_data_ext
-      if (restF) { restF->FillJAndData(ea_data_ext, mat, face_mat); }
-      // 2.3 Shift indirections in I back to original
-      auto I = mat.HostReadWriteI();
-      for (int i = ndofs; i > 0; i--)
-      {
-         I[i] = I[i-1];
-      }
-      I[0] = 0;
-      if (use_face_mat && restF)
-      {
-         auto I_face = face_mat.HostReadWriteI();
-         for (int i = ndofs; i > 0; i--)
-         {
-            I_face[i] = I_face[i-1];
-         }
-         I_face[0] = 0;
-      }
-   }
-   else // continuous Galerkin case
-   {
-      const ElementRestriction &rest =
-         static_cast<const ElementRestriction&>(*elem_restrict);
-      rest.FillSparseMatrix(ea_data, mat);
-   }
-}
-
-void FABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   mat.Mult(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMult(shared_x, y); }
-   }
-#endif
-}
-
-void FABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   mat.MultTranspose(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMultTranspose(shared_x, y); }
-   }
-#endif
-}
-
-
 MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
   : Operator(form->Height(), form->Width()), a(form)
 {
@@ -62,6 +62,27 @@ public:
   virtual void Update() = 0;
 };

+/** @brief Data and methods for fully-assembled bilinear forms.
+    Not yet implemented!  Use the BilinearForm Class instead. */
+class FABilinearFormExtension : public BilinearFormExtension
+{
+public:
+   FABilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~FABilinearFormExtension() {}
+};
+
 /// Data and methods for partially-assembled bilinear forms
 class PABilinearFormExtension : public BilinearFormExtension
 {
@@ -98,12 +119,10 @@ class EABilinearFormExtension : public PABilinearFormExtension
 protected:
   int ne;
   int elemDofs;
-   // The element matrices are stored row major
   Vector ea_data;
   int nf_int, nf_bdr;
   int faceDofs;
   Vector ea_data_int, ea_data_ext, ea_data_bdr;
-   bool factorize_face_terms;

 public:
   EABilinearFormExtension(BilinearForm *form);
@@ -113,23 +132,6 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
 };

-/// Data and methods for fully-assembled bilinear forms
-class FABilinearFormExtension : public EABilinearFormExtension
-{
-private:
-   SparseMatrix mat;
-   /// face_mat handles parallelism for DG face terms.
-   SparseMatrix face_mat;
-   bool use_face_mat;
-
-public:
-   FABilinearFormExtension(BilinearForm *form);
-
-   void Assemble();
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
 /// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
 class MFBilinearFormExtension : public BilinearFormExtension
 {
@@ -926,14 +926,11 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      // Set the integration point in the face and the neighboring element
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
+      IntegrationPoint eip;
+      Trans.Loc1.Transform(ip, eip);
      el1.CalcShape(eip, shape);

+      Trans.SetIntPoint(&ip);
      w = Trans.Weight() * ip.weight;
      if (Q)
      {
@@ -1522,7 +1519,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
   double w;

 #ifdef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
   curlshape.SetSize(nd,dimc);
@@ -1530,7 +1526,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
   elmat.SetSize(nd);
   if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }

   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
@@ -1574,12 +1569,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
         Mult(curlshape_dFt, M, curlshape);
         AddMultABt(curlshape, curlshape_dFt, elmat);
      }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(curlshape_dFt, D, elmat);
-      }
      else if (Q)
      {
         w *= Q->Eval(Trans, ip);
@@ -2582,17 +2571,16 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
+      Trans.Loc1.Transform(ip, eip1);
+      if (ndof2)
+      {
+         Trans.Loc2.Transform(ip, eip2);
+      }
      el1.CalcShape(eip1, shape1);

+      Trans.SetIntPoint(&ip);
+
      u->Eval(vu, *Trans.Elem1, eip1);

      if (dim == 1)
@@ -2739,15 +2727,10 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip1, eip2;

-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.SetIntPoint(&ip);
      if (dim == 1)
      {
         nor(0) = 2*eip1.x - 1.0;
@@ -2804,6 +2787,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(

      if (ndof2)
      {
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
         w = ip.weight/2/Trans.Elem2->Weight();
@@ -3021,14 +3005,9 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
   for (int pind = 0; pind < ir->GetNPoints(); ++pind)
   {
      const IntegrationPoint &ip = ir->IntPoint(pind);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+      IntegrationPoint eip1, eip2; // integration point in the reference space
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.SetIntPoint(&ip);

      el1.CalcShape(eip1, shape1);
      el1.CalcDShape(eip1, dshape1);
@@ -3048,6 +3027,7 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
      double w, wLM;
      if (ndofs2)
      {
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
         CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3185,22 +3165,17 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
      // Trace finite element shape function
+      Trans.SetIntPoint(&ip);
      trial_face_fe.CalcShape(ip, face_shape);
      // Side 1 finite element shape function
+      Trans.Loc1.Transform(ip, eip1);
      test_fe1.CalcShape(eip1, shape1);
      if (ndof2)
      {
         // Side 2 finite element shape function
+         Trans.Loc2.Transform(ip, eip2);
         test_fe2.CalcShape(eip2, shape2);
      }
      w = ip.weight;
@@ -20,13 +20,6 @@
 namespace mfem
 {

-// Local maximum size of dofs and quads in 1D
-constexpr int HCURL_MAX_D1D = 5;
-constexpr int HCURL_MAX_Q1D = 6;
-
-constexpr int HDIV_MAX_D1D = 5;
-constexpr int HDIV_MAX_Q1D = 6;
-
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -1692,22 +1685,6 @@ protected:
   {
      trial_fe.CalcPhysCurlShape(Trans, shape);
   }
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-private:
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, dofs1Dtest,quad1D, testType, trialType, coeffDim;
 };

 /** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 3D and
@@ -1747,20 +1724,6 @@ protected:
   {
      test_fe.CalcPhysCurlShape(Trans, shape);
   }
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-private:
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, quad1D, testType, trialType, coeffDim;
 };

 /** Class for integrating the bilinear form a(u,v) := - (Q u, grad v) in either
@@ -2300,14 +2263,12 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
   Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape, curlshape_dFt, M;
   DenseMatrix vshape, projcurl;
 #endif

 protected:
   Coefficient *Q;
-   VectorCoefficient *DQ;
   MatrixCoefficient *MQ;

   // PA extension
@@ -2316,17 +2277,12 @@ protected:
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
-   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
+   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2404,11 +2360,8 @@ protected:
   Vector pa_data;
   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
+   int dim, ne, nq, dofs1D, quad1D, fetype;

 public:
   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
@@ -2429,8 +2382,6 @@ public:

   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
 };
@@ -32,7 +32,7 @@ static void EAConvectionAssemble1D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -54,7 +54,7 @@ static void EAConvectionAssemble1D(const int NE,
            {
               val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
            }
-            A(i1, j1, e) += val;
+            A(i1, j1, e) = val;
         }
      }
   });
@@ -76,7 +76,7 @@ static void EAConvectionAssemble2D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -121,7 +121,7 @@ static void EAConvectionAssemble2D(const int NE,
                               * r_B[k1][j1]* r_B[k2][j2];
                     }
                  }
-                  A(i1, i2, j1, j2, e) += val;
+                  A(i1, i2, j1, j2, e) = val;
               }
            }
         }
@@ -145,7 +145,7 @@ static void EAConvectionAssemble3D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -191,7 +191,7 @@ static void EAConvectionAssemble3D(const int NE,
                              }
                           }
                        }
-                        A(i1, i2, i3, j1, j2, j3, e) += val;
+                        A(i1, i2, i3, j1, j2, j3, e) = val;
                     }
                  }
               }
@@ -788,20 +788,6 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   {
      vel = cQ->GetVec();
   }
-   else if (VectorQuadratureFunctionCoefficient* cQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      vel.SetSize(dim * nq * ne);
@@ -167,19 +167,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      r.SetSize(1);
      r(0) = c_rho->constant;
   }
-   else if (QuadratureFunctionCoefficient* c_rho =
-               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
-   {
-      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      r.SetSize(nq * nf);
@@ -213,20 +200,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   {
      vel = c_u->GetVec();
   }
-   else if (VectorQuadratureFunctionCoefficient* c_u =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
-   {
-      // Assumed to be in lexicographical ordering
-      const QuadratureFunction &qFun = c_u->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      vel.SetSize(dim * nq * nf);
@@ -31,7 +31,7 @@ static void EADiffusionAssemble1D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -53,7 +53,7 @@ static void EADiffusionAssemble1D(const int NE,
            {
               val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
            }
-            A(i1, j1, e) += val;
+            A(i1, j1, e) = val;
         }
      }
   });
@@ -75,7 +75,7 @@ static void EADiffusionAssemble2D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -120,7 +120,7 @@ static void EADiffusionAssemble2D(const int NE,
                               + gbi * D11 * gbj;
                     }
                  }
-                  A(i1, i2, j1, j2, e) += val;
+                  A(i1, i2, j1, j2, e) = val;
               }
            }
         }
@@ -144,7 +144,7 @@ static void EADiffusionAssemble3D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -208,7 +208,7 @@ static void EADiffusionAssemble3D(const int NE,
                              }
                           }
                        }
-                        A(i1, i2, i3, j1, j2, j3, e) += val;
+                        A(i1, i2, i3, j1, j2, j3, e) = val;
                     }
                  }
               }
@@ -176,42 +176,41 @@ static void PADiffusionSetup3D(const int Q1D,
   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
   auto D = Reshape(d.Write(), NQ, 6, NE);
-   MFEM_FORALL(e, NE,
+   MFEM_FORALL(eq, NE*NQ,
   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double coeff = const_c ? C(0,0) : C(q,e);
-         const double c_detJ = W[q] * coeff / detJ;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-         D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-         D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-         D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-         D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-         D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-         D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
-      }
+      const int e = eq / NQ;
+      const int q = eq % NQ;
+      const double J11 = J(q,0,0,e);
+      const double J21 = J(q,1,0,e);
+      const double J31 = J(q,2,0,e);
+      const double J12 = J(q,0,1,e);
+      const double J22 = J(q,1,1,e);
+      const double J32 = J(q,2,1,e);
+      const double J13 = J(q,0,2,e);
+      const double J23 = J(q,1,2,e);
+      const double J33 = J(q,2,2,e);
+      const double detJ = J11 * (J22 * J33 - J32 * J23) -
+      /* */               J21 * (J12 * J33 - J32 * J13) +
+      /* */               J31 * (J12 * J23 - J22 * J13);
+      const double coeff = const_c ? C(0,0) : C(q,e);
+      const double c_detJ = W[q] * coeff / detJ;
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J32 * J13) - (J12 * J33);
+      const double A13 = (J12 * J23) - (J22 * J13);
+      const double A21 = (J31 * J23) - (J21 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J21 * J13) - (J11 * J23);
+      const double A31 = (J21 * J32) - (J31 * J22);
+      const double A32 = (J31 * J12) - (J11 * J32);
+      const double A33 = (J11 * J22) - (J12 * J21);
+      // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+      D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+      D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+      D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+      D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+      D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+      D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
   });
 }

@@ -296,19 +295,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
      coeff.SetSize(1);
      coeff(0) = cQ->constant;
   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      coeff.SetSize(nq * ne);
@@ -1320,33 +1306,7 @@ static void PADiffusionApply3D(const int NE,
   });
 }

-// Half of B and G are stored in shared to get B, Bt, G and Gt.
-// Indices computation for SmemPADiffusionApply3D.
-static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
-{
-   return (q<=d) ? q : Q-1-q;
-}
-
-static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
-{
-   return (q<=d) ? d : D-1-d;
-}
-
-static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
-{
-   return (q<=d) ? Q-1-q : q;
-}
-
-static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
-{
-   return (q<=d) ? D-1-d : d;
-}
-
-static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
-{
-   return (q<=d) ? -1.0 : 1.0;
-}
-
+// Shared memory PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
                                   const Array<double> &b_,
@@ -1359,27 +1319,28 @@ static void SmemPADiffusionApply3D(const int NE,
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto d = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
+      const int tidz = MFEM_THREAD_ID(z);
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sBG[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sBG;
-      double (*G)[MD1] = (double (*)[MD1]) sBG;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sBG;
-      double (*Gt)[MQ1] = (double (*)[MQ1]) sBG;
+      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+      MFEM_SHARED double sBG[2][MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) (sBG+0);
+      double (*G)[MD1] = (double (*)[MD1]) (sBG+1);
+      double (*Bt)[MQ1] = (double (*)[MQ1]) (sBG+0);
+      double (*Gt)[MQ1] = (double (*)[MQ1]) (sBG+1);
      MFEM_SHARED double sm0[3][MDQ*MDQ*MDQ];
      MFEM_SHARED double sm1[3][MDQ*MDQ*MDQ];
      double (*X)[MD1][MD1]    = (double (*)[MD1][MD1]) (sm0+2);
@@ -1397,127 +1358,108 @@ static void SmemPADiffusionApply3D(const int NE,
      double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
      double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
      double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
         }
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(qx,dy,Q1D);
-            const int j = dj(qx,dy,D1D);
-            const int k = qk(qx,dy,Q1D);
-            const int l = dl(qx,dy,D1D);
-            B[i][j] = b(qx,dy);
-            G[k][l] = g(qx,dy) * sign(qx,dy);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+               G[q][d] = g(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            double u[D1D], v[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
+               double u = 0.0;
+               double v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
               {
                  const double coords = X[dz][dy][dx];
-                  u[dz] += coords * B[i][j];
-                  v[dz] += coords * G[k][l] * s;
+                  u += coords * B[qx][dx];
+                  v += coords * G[qx][dx];
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ0[dz][dy][qx] = u[dz];
-               DDQ1[dz][dy][qx] = v[dz];
+               DDQ0[dz][dy][qx] = u;
+               DDQ1[dz][dy][qx] = v;
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
               {
-                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
-                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
-                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
+                  u += DDQ1[dz][dy][qx] * B[qy][dy];
+                  v += DDQ0[dz][dy][qx] * G[qy][dy];
+                  w += DDQ0[dz][dy][qx] * B[qy][dy];
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ0[dz][qy][qx] = u[dz];
-               DQQ1[dz][qy][qx] = v[dz];
-               DQQ2[dz][qy][qx] = w[dz];
+               DQQ0[dz][qy][qx] = u;
+               DQQ1[dz][qy][qx] = v;
+               DQQ2[dz][qy][qx] = w;
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int dz = 0; dz < D1D; ++dz)
               {
-                  const int i = qi(qz,dz,Q1D);
-                  const int j = dj(qz,dz,D1D);
-                  const int k = qk(qz,dz,Q1D);
-                  const int l = dl(qz,dz,D1D);
-                  const double s = sign(qz,dz);
-                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
-                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
-                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
+                  u += DQQ0[dz][qy][qx] * B[qz][dz];
+                  v += DQQ1[dz][qy][qx] * B[qz][dz];
+                  w += DQQ2[dz][qy][qx] * G[qz][dz];
               }
+               QQQ0[qz][qy][qx] = u;
+               QQQ1[qz][qy][qx] = v;
+               QQQ2[qz][qy][qx] = w;
            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const double O11 = d(qx,qy,qz,0,e);
-               const double O12 = d(qx,qy,qz,1,e);
-               const double O13 = d(qx,qy,qz,2,e);
-               const double O22 = d(qx,qy,qz,3,e);
-               const double O23 = d(qx,qy,qz,4,e);
-               const double O33 = d(qx,qy,qz,5,e);
-               const double gX = u[qz];
-               const double gY = v[qz];
-               const double gZ = w[qz];
+               const int q = qx + ((qy*Q1D) + (qz*Q1D*Q1D));
+               const double O11 = d(q,0,e);
+               const double O12 = d(q,1,e);
+               const double O13 = d(q,2,e);
+               const double O22 = d(q,3,e);
+               const double O23 = d(q,4,e);
+               const double O33 = d(q,5,e);
+               const double gX = QQQ0[qz][qy][qx];
+               const double gY = QQQ1[qz][qy][qx];
+               const double gZ = QQQ2[qz][qy][qx];
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
               QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
               QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
@@ -1525,112 +1467,78 @@ static void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      if (tidz == 0)
      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(q,d,Q1D);
-            const int j = dj(q,d,D1D);
-            const int k = qk(q,d,Q1D);
-            const int l = dl(q,d,D1D);
-            Bt[j][i] = b(q,d);
-            Gt[l][k] = g(q,d) * sign(q,d);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bt[d][q] = b(q,d);
+               Gt[d][q] = g(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               MFEM_UNROLL(MQ1)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += QQQ0[qz][qy][qx] * Gt[dx][qx];
+                  v += QQQ1[qz][qy][qx] * Bt[dx][qx];
+                  w += QQQ2[qz][qy][qx] * Bt[dx][qx];
+               }
+               QQD0[qz][qy][dx] = u;
+               QQD1[qz][qy][dx] = v;
+               QQD2[qz][qy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  v += QQD1[qz][qy][dx] * Gt[dy][qy];
+                  w += QQD2[qz][qy][dx] * Bt[dy][qy];
+               }
+               QDD0[qz][dy][dx] = u;
+               QDD1[qz][dy][dx] = v;
+               QDD2[qz][dy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
               for (int qz = 0; qz < Q1D; ++qz)
               {
-                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
-                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
-                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
+                  u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  v += QDD1[qz][dy][dx] * Bt[dz][qz];
+                  w += QDD2[qz][dy][dx] * Gt[dz][qz];
               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD0[qz][qy][dx] = u[qz];
-               QQD1[qz][qy][dx] = v[qz];
-               QQD2[qz][qy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
-                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
-                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD0[qz][dy][dx] = u[qz];
-               QDD1[qz][dy][dx] = v[qz];
-               QDD2[qz][dy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int i = qi(qz,dz,Q1D);
-                  const int j = dj(qz,dz,D1D);
-                  const int k = qk(qz,dz,Q1D);
-                  const int l = dl(qz,dz,D1D);
-                  const double s = sign(qz,dz);
-                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
-                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
-                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
+               y(dx,dy,dz,e) += (u + v + w);
            }
         }
      }
@@ -1665,11 +1573,9 @@ static void PADiffusionApply(const int dim,
      MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
   }
 #endif // MFEM_USE_OCCA
-   const int ID = (D1D << 4 ) | Q1D;
-
   if (dim == 2)
   {
-      switch (ID)
+      switch ((D1D << 4 ) | Q1D)
      {
         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
@@ -1682,10 +1588,9 @@ static void PADiffusionApply(const int dim,
         default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
-
-   if (dim == 3)
+   else if (dim == 3)
   {
-      switch (ID)
+      switch ((D1D << 4 ) | Q1D)
      {
         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
@@ -23,6 +23,11 @@ using namespace std;
 namespace mfem
 {

+// Local maximum size of dofs and quads in 1D
+constexpr int HDIV_MAX_D1D = 5;
+constexpr int HDIV_MAX_Q1D = 6;
+
+
 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
                   const int NE,
@@ -109,8 +114,6 @@ void PAHdivMassApply2D(const int D1D,
                       Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
@@ -235,7 +238,6 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  Vector &_diag)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
@@ -612,8 +614,6 @@ static void PADivDivApply2D(const int D1D,
                            Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Bot = Reshape(_Bot.Read(), D1D-1, Q1D);
@@ -977,7 +977,6 @@ static void PADivDivAssembleDiagonal2D(const int D1D,
                                       Vector &_diag)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
@@ -1401,8 +1400,6 @@ static void PAHdivL2Apply2D(const int D1D,
                            Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
@@ -1669,8 +1666,6 @@ static void PAHdivL2ApplyTranspose2D(const int D1D,
                                     Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto L2Bo = Reshape(_L2Bo.Read(), Q1D, L2D1D);
   auto Gct = Reshape(_Gct.Read(), D1D, Q1D);
@@ -1729,7 +1724,7 @@ static void PAHdivL2ApplyTranspose2D(const int D1D,

      for (int qy = 0; qy < Q1D; ++qy)
      {
-         double aX[MAX_D1D];
+         double aX[HDIV_MAX_D1D];

         int osc = 0;
         for (int c = 0; c < VDIM; ++c)  // loop over x, y components
@@ -30,7 +30,7 @@ static void EAMassAssemble1D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(basis.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
+   auto M = Reshape(eadata.Write(), D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -52,7 +52,7 @@ static void EAMassAssemble1D(const int NE,
            {
               val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
            }
-            M(i1, j1, e) += val;
+            M(i1, j1, e) = val;
         }
      }
   });
@@ -72,7 +72,7 @@ static void EAMassAssemble2D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(basis.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
+   auto M = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -114,7 +114,7 @@ static void EAMassAssemble2D(const int NE,
                               * s_D[k1][k2];
                     }
                  }
-                  M(i1, i2, j1, j2, e) += val;
+                  M(i1, i2, j1, j2, e) = val;
               }
            }
         }
@@ -136,7 +136,7 @@ static void EAMassAssemble3D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(basis.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto M = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -189,7 +189,7 @@ static void EAMassAssemble3D(const int NE,
                              }
                           }
                        }
-                        M(i1, i2, i3, j1, j2, j3, e) += val;
+                        M(i1, i2, i3, j1, j2, j3, e) = val;
                     }
                  }
               }
@@ -25,6 +25,7 @@ namespace mfem

 void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
 {
+
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
@@ -41,8 +42,6 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
      InitCeedCoeff(Q, ptr);
      return CeedPAMassAssemble(fes, *ir, *ptr);
   }
-#else
-   MFEM_CONTRACT_VAR(force);
 #endif
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
@@ -53,34 +52,30 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
-   Vector coeff;
+   Vector *coeff{nullptr};
+   bool own_coeff{true};
   if (Q == nullptr)
   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
+      coeff = new Vector;
+      coeff->SetSize(1);
+      (*coeff)(0) = 1.0;
   }
   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
+      coeff = new Vector;
+      coeff->SetSize(1);
+      (*coeff)(0) = 1.0;
   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   else if (QuadratureCoefficient* cQ = dynamic_cast<QuadratureCoefficient*>(Q))
   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      coeff = cQ->Data();
+      own_coeff = false;
   }
   else
   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      coeff = new Vector;
+      coeff->SetSize(nq * ne);
+      auto C = Reshape(coeff->HostWrite(), nq, ne);
      for (int e = 0; e < ne; ++e)
      {
         ElementTransformation& T = *fes.GetElementTransformation(e);
@@ -95,11 +90,11 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
   {
      const int NE = ne;
      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
+      const bool const_c = coeff->Size() == 1;
      auto w = ir->GetWeights().Read();
      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
+         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
      auto v = Reshape(pa_data.Write(), NQ, NE);
      MFEM_FORALL(e, NE,
      {
@@ -119,11 +114,11 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
   {
      const int NE = ne;
      const int NQ = nq;
-      const bool const_c = coeff.Size() == 1;
+      const bool const_c = coeff->Size() == 1;
      auto W = ir->GetWeights().Read();
      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
      auto C =
-         const_c ? Reshape(coeff.Read(), 1,1) : Reshape(coeff.Read(), NQ,NE);
+         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
      auto v = Reshape(pa_data.Write(), NQ,NE);
      MFEM_FORALL(e, NE,
      {
@@ -140,6 +135,8 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
         }
      });
   }
+
+   if (own_coeff) { delete coeff; }
 }

 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
@@ -654,7 +651,6 @@ static void SmemPAMassApply2D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
-   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
@@ -918,7 +914,6 @@ static void SmemPAMassApply3D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
-   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -25,7 +25,7 @@ void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
   if (ne == 0) { return; }
   const int dofs = fes.GetFE(0)->GetDof();
   auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
-   auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
+   auto AT = Reshape(ea_data.Write(), dofs, dofs, ne);
   MFEM_FORALL(e, ne,
   {
      for (int i = 0; i < dofs; i++)
@@ -9,14 +9,12 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#include "../general/forall.hpp"
 #include "bilininteg.hpp"

 namespace mfem
 {

 void PAHcurlSetup2D(const int Q1D,
-                    const int coeffDim,
                    const int NE,
                    const Array<double> &w,
                    const Vector &j,
@@ -24,7 +22,6 @@ void PAHcurlSetup2D(const int Q1D,
                    Vector &op);

 void PAHcurlSetup3D(const int Q1D,
-                    const int coeffDim,
                    const int NE,
                    const Array<double> &w,
                    const Vector &j,
@@ -34,7 +31,6 @@ void PAHcurlSetup3D(const int Q1D,
 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
                                   const int NE,
-                                   const bool symmetric,
                                   const Array<double> &_Bo,
                                   const Array<double> &_Bc,
                                   const Vector &_op,
@@ -43,7 +39,6 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                   const int Q1D,
                                   const int NE,
-                                   const bool symmetric,
                                   const Array<double> &_Bo,
                                   const Array<double> &_Bc,
                                   const Vector &_op,
@@ -52,7 +47,6 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 void PAHcurlMassApply2D(const int D1D,
                        const int Q1D,
                        const int NE,
-                        const bool symmetric,
                        const Array<double> &_Bo,
                        const Array<double> &_Bc,
                        const Array<double> &_Bot,
@@ -64,7 +58,6 @@ void PAHcurlMassApply2D(const int D1D,
 void PAHcurlMassApply3D(const int D1D,
                        const int Q1D,
                        const int NE,
-                        const bool symmetric,
                        const Array<double> &_Bo,
                        const Array<double> &_Bc,
                        const Array<double> &_Bot,
@@ -147,573 +140,20 @@ void PAHdivMassApply3D(const int D1D,
                       const Vector &_x,
                       Vector &_y);

-void PAHcurlL2Setup(const int NQ,
-                    const int coeffDim,
-                    const int NE,
-                    const Array<double> &w,
-                    Vector &_coeff,
-                    Vector &op);
-
-// PA H(curl) x H(div) mass assemble 3D kernel, with factor
-// dF^{-1} C dF for a vector or matrix coefficient C.
-// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
-void PAHcurlHdivSetup3D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const bool transpose,
-                        const Array<double> &_w,
-                        const Vector &j,
-                        Vector &_coeff,
-                        Vector &op)
-{
-   const int NQ = Q1D*Q1D*Q1D;
-   const bool symmetric = (coeffDim != 9);
-   auto W = _w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), 9, NQ, NE);
-
-   const int i11 = 0;
-   const int i12 = transpose ? 3 : 1;
-   const int i13 = transpose ? 6 : 2;
-   const int i21 = transpose ? 1 : 3;
-   const int i22 = 4;
-   const int i23 = transpose ? 7 : 5;
-   const int i31 = transpose ? 2 : 6;
-   const int i32 = transpose ? 5 : 7;
-   const int i33 = 8;
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double w_detJ = W[q] / detJ;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-
-         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-         {
-            // First compute entries of R = MJ
-            const double M11 = (!symmetric) ? coeff(i11, q, e) : coeff(0, q, e);
-            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
-            const double M13 = (!symmetric) ? coeff(i13, q, e) : coeff(2, q, e);
-            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
-            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(3, q, e);
-            const double M23 = (!symmetric) ? coeff(i23, q, e) : coeff(4, q, e);
-            const double M31 = (!symmetric) ? coeff(i31, q, e) : M13;
-            const double M32 = (!symmetric) ? coeff(i32, q, e) : M23;
-            const double M33 = (!symmetric) ? coeff(i33, q, e) : coeff(5, q, e);
-
-            const double R11 = M11*J11 + M12*J12 + M13*J13;
-            const double R12 = M11*J21 + M12*J22 + M13*J23;
-            const double R13 = M11*J31 + M12*J32 + M13*J33;
-            const double R21 = M21*J11 + M22*J12 + M23*J13;
-            const double R22 = M21*J21 + M22*J22 + M23*J23;
-            const double R23 = M21*J31 + M22*J32 + M23*J33;
-            const double R31 = M31*J11 + M32*J12 + M33*J13;
-            const double R32 = M31*J21 + M32*J22 + M33*J23;
-            const double R33 = M31*J31 + M32*J32 + M33*J33;
-
-            // Now set y to detJ J^{-1} R = adj(J) R
-            y(i11,q,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-            y(i12,q,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
-            y(i13,q,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-            y(i21,q,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
-            y(i22,q,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
-            y(i23,q,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
-            y(i31,q,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-            y(i32,q,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-            y(i33,q,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
-         }
-         else if (coeffDim == 3)  // Vector coefficient version
-         {
-            const double D1 = coeff(0, q, e);
-            const double D2 = coeff(1, q, e);
-            const double D3 = coeff(2, q, e);
-            // detJ J^{-1} DJ = adj(J) DJ
-            y(i11,q,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
-            y(i12,q,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
-            y(i13,q,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
-            y(i21,q,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
-            y(i22,q,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
-            y(i23,q,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
-            y(i31,q,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
-            y(i32,q,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
-            y(i33,q,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
-         }
-      }
-   });
-}
-
-// PA H(curl) x H(div) mass assemble 2D kernel, with factor
-// dF^{-1} C dF for a vector or matrix coefficient C.
-// If transpose, use dF^T C dF^{-T} for H(div) x H(curl).
-void PAHcurlHdivSetup2D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const bool transpose,
-                        const Array<double> &_w,
-                        const Vector &j,
-                        Vector &_coeff,
-                        Vector &op)
-{
-   const int NQ = Q1D*Q1D;
-   const bool symmetric = (coeffDim != 4);
-   auto W = _w.Read();
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), 4, NQ, NE);
-
-   const int i11 = 0;
-   const int i12 = transpose ? 2 : 1;
-   const int i21 = transpose ? 1 : 2;
-   const int i22 = 3;
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double w_detJ = W[q] / (J11*J22) - (J21*J12);
-
-         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
-         {
-            // First compute entries of R = MJ
-            const double M11 = coeff(i11, q, e);
-            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
-            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
-            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(2, q, e);
-
-            const double R11 = M11*J11 + M12*J21;
-            const double R12 = M11*J12 + M12*J22;
-            const double R21 = M21*J11 + M22*J21;
-            const double R22 = M21*J12 + M22*J22;
-
-            // Now set y to J^{-1} R
-            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-         }
-         else if (coeffDim == 2) // Vector coefficient version
-         {
-            const double D1 = coeff(0, q, e);
-            const double D2 = coeff(1, q, e);
-            const double R11 = D1*J11;
-            const double R12 = D1*J12;
-            const double R21 = D2*J21;
-            const double R22 = D2*J22;
-            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-         }
-      }
-   });
-}
-
-// Mass operator for H(curl) and H(div) functions, using Piola transformations
-// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
-void PAHcurlHdivMassApply3D(const int D1D,
-                            const int D1Dtest,
-                            const int Q1D,
-                            const int NE,
-                            const bool scalarCoeff,
-                            const bool trialHcurl,
-                            const Array<double> &_Bo,
-                            const Array<double> &_Bc,
-                            const Array<double> &_Bot,
-                            const Array<double> &_Bct,
-                            const Vector &_op,
-                            const Vector &_x,
-                            Vector &_y)
-{
-   constexpr static int MAX_D1D = HCURL_MAX_D1D;
-   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
-
-   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
-   constexpr static int VDIM = 3;
-
-   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
-   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
-   auto Bot = Reshape(_Bot.Read(), D1Dtest-1, Q1D);
-   auto Bct = Reshape(_Bct.Read(), D1Dtest, Q1D);
-   auto op = Reshape(_op.Read(), scalarCoeff ? 1 : 9, Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(_x.Read(), 3*(D1D-1)*D1D*(trialHcurl ? D1D : D1D-1), NE);
-   auto y = Reshape(_y.ReadWrite(), 3*(D1Dtest-1)*D1Dtest*
-                    (trialHcurl ? D1Dtest-1 : D1Dtest), NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
-
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               for (int c = 0; c < VDIM; ++c)
-               {
-                  mass[qz][qy][qx][c] = 0.0;
-               }
-            }
-         }
-      }
-
-      int osc = 0;
-      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z trial components
-      {
-         const int D1Dz = trialHcurl ? ((c == 2) ? D1D - 1 : D1D) :
-                          ((c == 2) ? D1D : D1D - 1);
-         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
-                          ((c == 1) ? D1D : D1D - 1);
-         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
-                          ((c == 0) ? D1D : D1D - 1);
-
-         for (int dz = 0; dz < D1Dz; ++dz)
-         {
-            double massXY[MAX_Q1D][MAX_Q1D];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  massXY[qy][qx] = 0.0;
-               }
-            }
-
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               double massX[MAX_Q1D];
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  massX[qx] = 0.0;
-               }
-
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  const double t = x(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
-                                       ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
-                  }
-               }
-
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
-                                    ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const double wx = massX[qx];
-                     massXY[qy][qx] += wx * wy;
-                  }
-               }
-            }
-
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double wz = trialHcurl ? ((c == 2) ? Bo(qz,dz) : Bc(qz,dz)) :
-                                 ((c == 2) ? Bc(qz,dz) : Bo(qz,dz));
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
-                  }
-               }
-            }
-         }
-
-         osc += D1Dx * D1Dy * D1Dz;
-      }  // loop (c) over components
-
-      // Apply D operator.
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double O11 = op(0,qx,qy,qz,e);
-               const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,qz,e);
-               const double O13 = scalarCoeff ? 0.0 : op(2,qx,qy,qz,e);
-               const double O21 = scalarCoeff ? 0.0 : op(3,qx,qy,qz,e);
-               const double O22 = scalarCoeff ? O11 : op(4,qx,qy,qz,e);
-               const double O23 = scalarCoeff ? 0.0 : op(5,qx,qy,qz,e);
-               const double O31 = scalarCoeff ? 0.0 : op(6,qx,qy,qz,e);
-               const double O32 = scalarCoeff ? 0.0 : op(7,qx,qy,qz,e);
-               const double O33 = scalarCoeff ? O11 : op(8,qx,qy,qz,e);
-               const double massX = mass[qz][qy][qx][0];
-               const double massY = mass[qz][qy][qx][1];
-               const double massZ = mass[qz][qy][qx][2];
-               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
-               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
-               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
-            }
-         }
-      }
-
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double massXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
-
-         osc = 0;
-         for (int c = 0; c < VDIM; ++c)  // loop over x, y, z test components
-         {
-            const int D1Dz = trialHcurl ? ((c == 2) ? D1Dtest : D1Dtest - 1) :
-                             ((c == 2) ? D1Dtest - 1 : D1Dtest);
-            const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
-                             ((c == 1) ? D1Dtest - 1 : D1Dtest);
-            const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
-                             ((c == 0) ? D1Dtest - 1 : D1Dtest);
-
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  massXY[dy][dx] = 0.0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               double massX[HDIV_MAX_D1D];
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  massX[dx] = 0.0;
-               }
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     massX[dx] += mass[qz][qy][qx][c] * (trialHcurl ?
-                                                         ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
-                                                         ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
-                  }
-               }
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
-                                    ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     massXY[dy][dx] += massX[dx] * wy;
-                  }
-               }
-            }
-
-            for (int dz = 0; dz < D1Dz; ++dz)
-            {
-               const double wz = trialHcurl ? ((c == 2) ? Bct(dz,qz) : Bot(dz,qz)) :
-                                 ((c == 2) ? Bot(dz,qz) : Bct(dz,qz));
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e) +=
-                        massXY[dy][dx] * wz;
-                  }
-               }
-            }
-
-            osc += D1Dx * D1Dy * D1Dz;
-         }  // loop c
-      }  // loop qz
-   }); // end of element loop
-}
-
-// Mass operator for H(curl) and H(div) functions, using Piola transformations
-// u = dF^{-T} \hat{u} in H(curl), v = (1 / det dF) dF \hat{v} in H(div).
-void PAHcurlHdivMassApply2D(const int D1D,
-                            const int D1Dtest,
-                            const int Q1D,
-                            const int NE,
-                            const bool scalarCoeff,
-                            const bool trialHcurl,
-                            const Array<double> &_Bo,
-                            const Array<double> &_Bc,
-                            const Array<double> &_Bot,
-                            const Array<double> &_Bct,
-                            const Vector &_op,
-                            const Vector &_x,
-                            Vector &_y)
-{
-   constexpr static int MAX_D1D = HCURL_MAX_D1D;
-   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
-
-   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
-   constexpr static int VDIM = 2;
-
-   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
-   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
-   auto Bot = Reshape(_Bot.Read(), D1Dtest-1, Q1D);
-   auto Bct = Reshape(_Bct.Read(), D1Dtest, Q1D);
-   auto op = Reshape(_op.Read(), scalarCoeff ? 1 : 4, Q1D, Q1D, NE);
-   auto x = Reshape(_x.Read(), 2*(D1D-1)*D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), 2*(D1Dtest-1)*D1Dtest, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      double mass[MAX_Q1D][MAX_Q1D][VDIM];
-
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            for (int c = 0; c < VDIM; ++c)
-            {
-               mass[qy][qx][c] = 0.0;
-            }
-         }
-      }
-
-      int osc = 0;
-      for (int c = 0; c < VDIM; ++c)  // loop over x, y trial components
-      {
-         const int D1Dy = trialHcurl ? ((c == 1) ? D1D - 1 : D1D) :
-                          ((c == 1) ? D1D : D1D - 1);
-         const int D1Dx = trialHcurl ? ((c == 0) ? D1D - 1 : D1D) :
-                          ((c == 0) ? D1D : D1D - 1);
-
-         for (int dy = 0; dy < D1Dy; ++dy)
-         {
-            double massX[MAX_Q1D];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               massX[qx] = 0.0;
-            }
-
-            for (int dx = 0; dx < D1Dx; ++dx)
-            {
-               const double t = x(dx + (dy * D1Dx) + osc, e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  massX[qx] += t * (trialHcurl ? ((c == 0) ? Bo(qx,dx) : Bc(qx,dx)) :
-                                    ((c == 0) ? Bc(qx,dx) : Bo(qx,dx)));
-               }
-            }
-
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy = trialHcurl ? ((c == 1) ? Bo(qy,dy) : Bc(qy,dy)) :
-                                 ((c == 1) ? Bc(qy,dy) : Bo(qy,dy));
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  mass[qy][qx][c] += massX[qx] * wy;
-               }
-            }
-         }
-
-         osc += D1Dx * D1Dy;
-      }  // loop (c) over components
-
-      // Apply D operator.
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double O11 = op(0,qx,qy,e);
-            const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,e);
-            const double O21 = scalarCoeff ? 0.0 : op(2,qx,qy,e);
-            const double O22 = scalarCoeff ? O11 : op(3,qx,qy,e);
-            const double massX = mass[qy][qx][0];
-            const double massY = mass[qy][qx][1];
-            mass[qy][qx][0] = (O11*massX)+(O12*massY);
-            mass[qy][qx][1] = (O21*massX)+(O22*massY);
-         }
-      }
-
-      osc = 0;
-      for (int c = 0; c < VDIM; ++c)  // loop over x, y test components
-      {
-         const int D1Dy = trialHcurl ? ((c == 1) ? D1Dtest : D1Dtest - 1) :
-                          ((c == 1) ? D1Dtest - 1 : D1Dtest);
-         const int D1Dx = trialHcurl ? ((c == 0) ? D1Dtest : D1Dtest - 1) :
-                          ((c == 0) ? D1Dtest - 1 : D1Dtest);
-
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double massX[HDIV_MAX_D1D];
-            for (int dx = 0; dx < D1Dx; ++dx)
-            {
-               massX[dx] = 0.0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  massX[dx] += mass[qy][qx][c] * (trialHcurl ?
-                                                  ((c == 0) ? Bct(dx,qx) : Bot(dx,qx)) :
-                                                  ((c == 0) ? Bot(dx,qx) : Bct(dx,qx)));
-               }
-            }
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               const double wy = trialHcurl ? ((c == 1) ? Bct(dy,qy) : Bot(dy,qy)) :
-                                 ((c == 1) ? Bot(dy,qy) : Bct(dy,qy));
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  y(dx + (dy * D1Dx) + osc, e) += massX[dx] * wy;
-               }
-            }
-         }
-
-         osc += D1Dx * D1Dy;
-      }  // loop c
-   }); // end of element loop
-}
-
 void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   AssemblePA(fes, fes);
-}
-
-void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
-                                        const FiniteElementSpace &test_fes)
 {
   // Assumes tensor-product elements
-   Mesh *mesh = trial_fes.GetMesh();
+   Mesh *mesh = fes.GetMesh();
+   const FiniteElement *fel = fes.GetFE(0);

-   const FiniteElement *trial_fel = trial_fes.GetFE(0);
-   const VectorTensorFiniteElement *trial_el =
-      dynamic_cast<const VectorTensorFiniteElement*>(trial_fel);
-   MFEM_VERIFY(trial_el != NULL, "Only VectorTensorFiniteElement is supported!");
-
-   const FiniteElement *test_fel = test_fes.GetFE(0);
-   const VectorTensorFiniteElement *test_el =
-      dynamic_cast<const VectorTensorFiniteElement*>(test_fel);
-   MFEM_VERIFY(test_el != NULL, "Only VectorTensorFiniteElement is supported!");
+   const VectorTensorFiniteElement *el =
+      dynamic_cast<const VectorTensorFiniteElement*>(fel);
+   MFEM_VERIFY(el != NULL, "Only VectorTensorFiniteElement is supported!");

   const IntegrationRule *ir
-      = IntRule ? IntRule : &MassIntegrator::GetRule(*trial_el, *trial_el,
+      = IntRule ? IntRule : &MassIntegrator::GetRule(*el, *el,
                                                     *mesh->GetElementTransformation(0));
-   const int dims = trial_el->GetDim();
+   const int dims = el->GetDim();
   MFEM_VERIFY(dims == 2 || dims == 3, "");

   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -721,160 +161,53 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   dim = mesh->Dimension();
   MFEM_VERIFY(dim == 2 || dim == 3, "");

-   ne = trial_fes.GetNE();
-   MFEM_VERIFY(ne == test_fes.GetNE(),
-               "Different meshes for test and trial spaces");
+   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   mapsC = &trial_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   mapsO = &trial_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
+   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
+   mapsO = &el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
   dofs1D = mapsC->ndof;
   quad1D = mapsC->nqpt;

-   mapsCtest = &test_el->GetDofToQuad(*ir, DofToQuad::TENSOR);
-   mapsOtest = &test_el->GetDofToQuadOpen(*ir, DofToQuad::TENSOR);
-   dofs1Dtest = mapsCtest->ndof;
-
   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   trial_fetype = trial_el->GetDerivType();
-   test_fetype = test_el->GetDerivType();
+   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());

-   const int MQsymmDim = MQ ? (MQ->GetWidth() * (MQ->GetWidth() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = MQ ? (MQ->IsSymmetric() ? MQsymmDim : MQfullDim) : 0;
-   const int coeffDim = MQ ? MQdim : (VQ ? VQ->GetVDim() : 1);
-
-   symmetric = MQ ? MQ->IsSymmetric() : true;
-
-   if ((trial_fetype == mfem::FiniteElement::CURL &&
-        test_fetype == mfem::FiniteElement::DIV) ||
-       (trial_fetype == mfem::FiniteElement::DIV &&
-        test_fetype == mfem::FiniteElement::CURL))
-      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
-                      Device::GetMemoryType());
-   else
-      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
-                      Device::GetMemoryType());
-
-   Vector coeff(coeffDim * ne * nq);
+   Vector coeff(ne * nq);
   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || VQ || MQ)
+   if (Q)
   {
-      Vector D(VQ ? coeffDim : 0);
-      DenseMatrix M;
-      Vector Msymm;
-      if (MQ)
-      {
-         if (symmetric)
-         {
-            Msymm.SetSize(MQsymmDim);
-         }
-         else
-         {
-            M.SetSize(dim);
-         }
-      }
-
-      if (VQ)
-      {
-         MFEM_VERIFY(coeffDim == dim, "");
-      }
-      if (MQ)
-      {
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-      }
-
      for (int e=0; e<ne; ++e)
      {
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            if (MQ)
-            {
-               if (MQ->IsSymmetric())
-               {
-                  MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
-
-                  for (int i=0; i<MQsymmDim; ++i)
-                  {
-                     coeffh(i, p, e) = Msymm[i];
-                  }
-               }
-               else
-               {
-                  MQ->Eval(M, *tr, ir->IntPoint(p));
-
-                  for (int i=0; i<dim; ++i)
-                     for (int j=0; j<dim; ++j)
-                     {
-                        coeffh(j+(i*dim), p, e) = M(i,j);
-                     }
-               }
-            }
-            else if (VQ)
-            {
-               VQ->Eval(D, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = D[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
         }
      }
   }

-   if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype
-       && dim == 3)
+   fetype = el->GetDerivType();
+
+   if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
   {
-      PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
   }
-   else if (trial_fetype == mfem::FiniteElement::CURL
-            && test_fetype == trial_fetype && dim == 2)
+   else if (el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
   {
-      PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
   }
-   else if (trial_fetype == mfem::FiniteElement::DIV
-            && test_fetype == trial_fetype && dim == 3)
+   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
   {
      PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
-   else if (trial_fetype == mfem::FiniteElement::DIV
-            && test_fetype == trial_fetype && dim == 2)
+   else if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 2)
   {
      PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
-   else if (((trial_fetype == mfem::FiniteElement::CURL &&
-              test_fetype == mfem::FiniteElement::DIV) ||
-             (trial_fetype == mfem::FiniteElement::DIV &&
-              test_fetype == mfem::FiniteElement::CURL)) &&
-            test_fel->GetOrder() == trial_fel->GetOrder())
-   {
-      if (coeffDim == 1)
-      {
-         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
-      }
-      else
-      {
-         const bool tr = (trial_fetype == mfem::FiniteElement::DIV &&
-                          test_fetype == mfem::FiniteElement::CURL);
-         if (dim == 3)
-            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
-                               geom->J, coeff, pa_data);
-         else
-            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
-                               geom->J, coeff, pa_data);
-      }
-   }
   else
   {
      MFEM_ABORT("Unknown kernel.");
@@ -885,13 +218,12 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
 {
   if (dim == 3)
   {
-      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
+      if (fetype == mfem::FiniteElement::CURL)
      {
-         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
+         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
      }
-      else if (trial_fetype == mfem::FiniteElement::DIV &&
-               test_fetype == trial_fetype)
+      else if (fetype == mfem::FiniteElement::DIV)
      {
         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne,
                                      mapsO->B, mapsC->B, pa_data, diag);
@@ -903,13 +235,12 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
   }
   else
   {
-      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
+      if (fetype == mfem::FiniteElement::CURL)
      {
-         PAHcurlMassAssembleDiagonal2D(dofs1D, quad1D, ne, symmetric,
+         PAHcurlMassAssembleDiagonal2D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
      }
-      else if (trial_fetype == mfem::FiniteElement::DIV &&
-               test_fetype == trial_fetype)
+      else if (fetype == mfem::FiniteElement::DIV)
      {
         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne,
                                      mapsO->B, mapsC->B, pa_data, diag);
@@ -925,33 +256,16 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
   if (dim == 3)
   {
-      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
+      if (fetype == mfem::FiniteElement::CURL)
      {
-         PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
      }
-      else if (trial_fetype == mfem::FiniteElement::DIV &&
-               test_fetype == trial_fetype)
+      else if (fetype == mfem::FiniteElement::DIV)
      {
         PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                           mapsC->Bt, pa_data, x, y);
      }
-      else if (trial_fetype == mfem::FiniteElement::CURL &&
-               test_fetype == mfem::FiniteElement::DIV)
-      {
-         const bool scalarCoeff = !(VQ || MQ);
-         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                true, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                mapsCtest->Bt, pa_data, x, y);
-      }
-      else if (trial_fetype == mfem::FiniteElement::DIV &&
-               test_fetype == mfem::FiniteElement::CURL)
-      {
-         const bool scalarCoeff = !(VQ || MQ);
-         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                false, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                mapsCtest->Bt, pa_data, x, y);
-      }
      else
      {
         MFEM_ABORT("Unknown kernel.");
@@ -959,28 +273,16 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
   else
   {
-      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
+      if (fetype == mfem::FiniteElement::CURL)
      {
-         PAHcurlMassApply2D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
-                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
+                            mapsC->Bt, pa_data, x, y);
      }
-      else if (trial_fetype == mfem::FiniteElement::DIV &&
-               test_fetype == trial_fetype)
+      else if (fetype == mfem::FiniteElement::DIV)
      {
         PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
                           mapsC->Bt, pa_data, x, y);
      }
-      else if ((trial_fetype == mfem::FiniteElement::CURL &&
-                test_fetype == mfem::FiniteElement::DIV) ||
-               (trial_fetype == mfem::FiniteElement::DIV &&
-                test_fetype == mfem::FiniteElement::CURL))
-      {
-         const bool scalarCoeff = !(VQ || MQ);
-         const bool trialHcurl = (trial_fetype == mfem::FiniteElement::CURL);
-         PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                trialHcurl, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                mapsCtest->Bt, pa_data, x, y);
-      }
      else
      {
         MFEM_ABORT("Unknown kernel.");
@@ -1046,12 +348,12 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
   // Use the same setup functions as VectorFEMassIntegrator.
   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
   {
-      PAHcurlSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
   }
   else if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
   {
-      PAHcurlSetup2D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+      PAHcurlSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
   }
   else
@@ -12,6 +12,7 @@
 // Implementation of Coefficient class

 #include "fem.hpp"
+#include "../linalg/dtensor.hpp"

 #include <cmath>
 #include <limits>
@@ -21,6 +22,13 @@ namespace mfem

 using namespace std;

+double QuadratureCoefficient::Eval(ElementTransformation & T,
+                                   const IntegrationPoint & ip)
+{
+   auto coeff = mfem::Reshape(qData->HostRead(), nip, NE);
+   return coeff(ip.index, T.ElementNo);
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                const IntegrationPoint & ip)
 {
@@ -209,24 +217,18 @@ void GradientGridFunctionCoefficient::Eval(
   GridFunc->GetGradients(T, ir, M);
 }

-CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
+CurlGridFunctionCoefficient::CurlGridFunctionCoefficient (
   const GridFunction *gf)
-   : VectorCoefficient(0)
+   : VectorCoefficient ((gf) ?
+                        gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
 {
-   SetGridFunction(gf);
+   GridFunc = gf;
 }

 void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 {
-   if (gf)
-   {
-      int sdim = gf -> FESpace() -> GetMesh() -> SpaceDimension();
-      MFEM_VERIFY(sdim == 2 || sdim == 3,
-                  "CurlGridFunctionCoefficient "
-                  "only defind for spaces of dimension 2 or 3.");
-   }
-   GridFunc = gf;
-   vdim = (gf) ? (2 * gf -> FESpace() -> GetMesh() -> SpaceDimension() - 3) : 0;
+   GridFunc = gf; vdim = (gf) ?
+                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
 }

 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
@@ -319,31 +321,6 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
   }
 }

-void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
-                                              ElementTransformation &T,
-                                              const IntegrationPoint &ip)
-{
-   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
-               "MatrixFunctionCoefficient is not symmetric");
-
-   double x[3];
-   Vector transip(x, 3);
-
-   T.Transform(ip, transip);
-
-   K.SetSize((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
-
-   if (SymmFunction)
-   {
-      (*SymmFunction)(transip, K);
-   }
-
-   if (Q)
-   {
-      K *= Q->Eval(T, ip, GetTime());
-   }
-}
-
 MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
   : MatrixCoefficient (dim)
 {
@@ -30,10 +30,7 @@ class ParMesh;
 /** @brief Base class Coefficients that optionally depend on space and time.
    These are used by the BilinearFormIntegrator, LinearFormIntegrator, and
    NonlinearFormIntegrator classes to represent the physical coefficients in
-    the PDEs that are being discretized. This class can also be used in a more
-    general way to represent functions that don't necessarily belong to a FE
-    space, e.g., to project onto GridFunctions to use as initial conditions,
-    exact solutions, etc. See, e.g., ex4 or ex22 for these uses. */
+    the PDEs that are being discretized. */
 class Coefficient
 {
 protected:
@@ -87,6 +84,33 @@ public:
   { return (constant); }
 };

+
+/// class for quadrature coefficient
+class QuadratureCoefficient : public Coefficient
+{
+
+private:
+   const int nip;
+   const int NE;
+public:
+   Vector *qData{nullptr};
+
+   //Set external data
+   QuadratureCoefficient(Vector *Data, int in_nip, int in_NE)
+      : qData(Data), nip(in_nip), NE(in_NE)
+   { }
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+
+   Vector *Data()
+   {
+      return qData;
+   }
+
+};
+
+/// class for piecewise constant coefficient
 /** @brief A piecewise constant coefficient with the constants keyed
    off the element attribute numbers. */
 class PWConstCoefficient : public Coefficient
@@ -695,16 +719,13 @@ class MatrixCoefficient
 protected:
   int height, width;
   double time;
-   bool symmetric;

 public:
   /// Construct a dim x dim matrix coefficient.
-   explicit MatrixCoefficient(int dim, bool symm=false)
-   { height = width = dim; time = 0.; symmetric = symm; }
+   explicit MatrixCoefficient(int dim) { height = width = dim; time = 0.; }

   /// Construct a h x w matrix coefficient.
-   MatrixCoefficient(int h, int w, bool symm=false) :
-      height(h), width(w), time(0.), symmetric(symm) { }
+   MatrixCoefficient(int h, int w) : height(h), width(w), time(0.) { }

   /// Set the time for time dependent coefficients
   void SetTime(double t) { time = t; }
@@ -721,9 +742,6 @@ public:
   /// For backward compatibility get the width of the matrix.
   int GetVDim() const { return width; }

-   void SetSymmetric(bool s) { symmetric = s; }
-   bool IsSymmetric() const { return symmetric; }
-
   /** @brief Evaluate the matrix coefficient in the element described by @a T
       at the point @a ip, storing the result in @a K. */
   /** @note When this method is called, the caller must make sure that the
@@ -732,15 +750,6 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip) = 0;

-   /** @brief Evaluate the upper triangular entries of the matrix coefficient
-       in the symmetric case, similarly to Eval. Matrix entry (i,j) is stored
-       in K[j - i + os_i] for 0 <= i <= j < width, os_0 = 0,
-       os_{i+1} = os_i + width - i. That is, K = {M(0,0), ..., M(0,w-1),
-       M(1,1), ..., M(1,w-1), ..., M(w-1,w-1) with w = width. */
-   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
-                              const IntegrationPoint &ip)
-   { mfem_error("MatrixCoefficient::EvalSymmetric"); }
-
   virtual ~MatrixCoefficient() { }
 };

@@ -768,7 +777,6 @@ class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
   void (*Function)(const Vector &, DenseMatrix &);
-   void (*SymmFunction)(const Vector &, Vector &);
   void (*TDFunction)(const Vector &, double, DenseMatrix &);
   Coefficient *Q;
   DenseMatrix mat;
@@ -806,26 +814,10 @@ public:
      mat.SetSize(0);
   }

-   /// Construct a symmetric square matrix coefficient from a C-function
-   /// defining a vector function used by EvalSymmetric
-   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
-                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim, true), Q(q)
-   {
-      SymmFunction = F;
-      Function = NULL;
-      TDFunction = NULL;
-      mat.SetSize(0);
-   }
-
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Evaluate the symmetric matrix coefficient at @a ip.
-   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
-                              const IntegrationPoint &ip);
-
   virtual ~MatrixFunctionCoefficient() { }
 };

@@ -99,8 +99,8 @@ public:
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);

-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
-       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .

       The pointer @a fes is not owned by the newly constructed object.

@@ -195,8 +195,8 @@ private:
   BilinearForm *blfr;
   BilinearForm *blfi;

-   /* These methods check if the real/imag parts of the sesquilinear form are
-      not empty */
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
   bool RealInteg();
   bool ImagInteg();

@@ -204,7 +204,7 @@ public:
   SesquilinearForm(FiniteElementSpace *fes,
                    ComplexOperator::Convention
                    convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
       the same integrators as the BilinearForms @a bfr and @a bfi .

       The pointer @a fes is not owned by the newly constructed object.
@@ -219,21 +219,6 @@ public:
   void SetConvention(const ComplexOperator::Convention &
                      convention) { conv = convention; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      blfr->SetAssemblyLevel(assembly_level);
-      blfi->SetAssemblyLevel(assembly_level);
-   }
-
   BilinearForm & real() { return *blfr; }
   BilinearForm & imag() { return *blfi; }
   const BilinearForm & real() const { return *blfr; }
@@ -324,7 +309,7 @@ protected:
 public:

   /* @brief Construct a ParComplexGridFunction associated with the
-      ParFiniteElementSpace @a *pf. */
+      ParFiniteElementSpace @a *f. */
   ParComplexGridFunction(ParFiniteElementSpace *pf);

   void Update();
@@ -416,8 +401,8 @@ public:
                        convention = ComplexOperator::HERMITIAN);

   /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plf_r (real) and
-       @a plf_i (imag).
+       using the same integrators as the LinearForms @a plfr (real) and @a plfi
+       (imag) .

      The pointer @a fes is not owned by the newly constructed object.

@@ -493,7 +478,7 @@ public:
 /** Class for a parallel sesquilinear form

    A sesquilinear form is a generalization of a bilinear form to complex-valued
-    fields. Sesquilinear forms are linear in the second argument but the
+    fields. Sesquilinear forms are linear in the second argument but but the
    first argument involves a complex conjugate in the sense that:

                a(alpha u, beta v) = conj(alpha) beta a(u, v)
@@ -539,21 +524,6 @@ public:
   void SetConvention(const ComplexOperator::Convention &
                      convention) { conv = convention; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pblfr->SetAssemblyLevel(assembly_level);
-      pblfi->SetAssemblyLevel(assembly_level);
-   }
-
   ParBilinearForm & real() { return *pblfr; }
   ParBilinearForm & imag() { return *pblfi; }
   const ParBilinearForm & real() const { return *pblfr; }
@@ -415,6 +415,9 @@ void VisItDataCollection::SetMesh(MPI_Comm comm, Mesh *new_mesh)
 void VisItDataCollection::RegisterField(const std::string& name,
                                        GridFunction *gf)
 {
+   DataCollection::RegisterField(name, gf);
+   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim());
+
   int LOD = 1;
   if (gf->FESpace()->GetNURBSext())
   {
@@ -428,27 +431,6 @@ void VisItDataCollection::RegisterField(const std::string& name,
      }
   }

-   DataCollection::RegisterField(name, gf);
-   field_info_map[name] = VisItFieldInfo("nodes", gf->VectorDim(), LOD);
-   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
-}
-
-void VisItDataCollection::RegisterQField(const std::string& name,
-                                         QuadratureFunction *qf)
-{
-   int LOD = -1;
-   Mesh *mesh = qf->GetSpace()->GetMesh();
-   for (int e=0; e<qf->GetSpace()->GetNE(); e++)
-   {
-      int locLOD = GlobGeometryRefiner.GetRefinementLevelFromElems(
-                      mesh->GetElementBaseGeometry(e),
-                      qf->GetElementIntRule(e).GetNPoints());
-
-      LOD = std::max(LOD,locLOD);
-   }
-
-   DataCollection::RegisterQField(name, qf);
-   field_info_map[name] = VisItFieldInfo("elements", 1, LOD);
   visit_levels_of_detail = std::max(visit_levels_of_detail, LOD);
 }

@@ -616,28 +598,14 @@ void VisItDataCollection::LoadFields()
      // TODO: 1) load parallel GridFunction on one processor
      if (serial)
      {
-         if ((it->second).association == "nodes")
-         {
-            field_map.Register(it->first, new GridFunction(mesh, file), own_data);
-         }
-         else if ((it->second).association == "elements")
-         {
-            q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
-         }
+         field_map.Register(it->first, new GridFunction(mesh, file), own_data);
      }
      else
      {
 #ifdef MFEM_USE_MPI
-         if ((it->second).association == "nodes")
-         {
-            field_map.Register(
-               it->first,
-               new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
-         }
-         else if ((it->second).association == "elements")
-         {
-            q_field_map.Register(it->first, new QuadratureFunction(mesh, file), own_data);
-         }
+         field_map.Register(
+            it->first,
+            new ParGridFunction(dynamic_cast<ParMesh*>(mesh), file), own_data);
 #else
         error = READ_ERROR;
         MFEM_WARNING("Reading parallel format in serial is not supported");
@@ -672,7 +640,7 @@ std::string VisItDataCollection::GetVisItRootString()
   {
      ftags["assoc"] = picojson::value((it->second).association);
      ftags["comps"] = picojson::value(to_string((it->second).num_components));
-      ftags["lod"] = picojson::value(to_string((it->second).lod));
+      ftags["lod"] = picojson::value(to_string(visit_levels_of_detail));
      field["path"] = picojson::value(path_str + it->first + file_ext_format);
      field["tags"] = picojson::value(ftags);
      fields[it->first] = picojson::value(field);
@@ -391,10 +391,9 @@ class VisItFieldInfo
 public:
   std::string association;
   int num_components;
-   int lod;
-   VisItFieldInfo() { association = ""; num_components = 0; lod = 1;}
-   VisItFieldInfo(std::string _association, int _num_components, int _lod = 1)
-   { association = _association; num_components = _num_components; lod =_lod;}
+   VisItFieldInfo() { association = ""; num_components = 0; }
+   VisItFieldInfo(std::string _association, int _num_components)
+   { association = _association; num_components = _num_components; }
 };

 /// Data collection with VisIt I/O routines
@@ -446,12 +445,6 @@ public:
   /// Add a grid function to the collection and update the root file
   virtual void RegisterField(const std::string& field_name, GridFunction *gf);

-   /// Add a quadrature function to the collection and update the root file.
-   /** Visualization of quadrature function is not supported in VisIt(3.12).
-       A patch has been sent to VisIt developers in June 2020. */
-   virtual void RegisterQField(const std::string& q_field_name,
-                               QuadratureFunction *qf);
-
   /// Set VisIt parameter: default levels of detail for the MultiresControl
   void SetLevelsOfDetail(int levels_of_detail);

@@ -552,32 +552,26 @@ void IntegrationPointTransformation::Transform (const IntegrationRule &ir1,
   }
 }

-void FaceElementTransformations::SetIntPoint(const IntegrationPoint *face_ip)
+void FaceElementTransformations::SetIntPoint(const IntegrationPoint *ip)
 {
-   IsoparametricTransformation::SetIntPoint(face_ip);
+   IsoparametricTransformation::SetIntPoint(ip);

-   if (mask & 4)
+   if (Elem1)
   {
-      Loc1.Transform(*face_ip, eip1);
-      if (Elem1)
-      {
-         Elem1->SetIntPoint(&eip1);
-      }
+      Loc1.Transform(*ip, eip1);
+      Elem1->SetIntPoint(&eip1);
   }
-   if (mask & 8)
+   if (Elem2)
   {
-      Loc2.Transform(*face_ip, eip2);
-      if (Elem2)
-      {
-         Elem2->SetIntPoint(&eip2);
-      }
+      Loc2.Transform(*ip, eip2);
+      Elem2->SetIntPoint(&eip2);
   }
 }

 ElementTransformation &
 FaceElementTransformations::GetElement1Transformation()
 {
-   MFEM_VERIFY(mask & HAVE_ELEM1 && Elem1 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & 1 && Elem1 != NULL, "The ElementTransformation "
               "for the element has not been configured for side 1.");
   return *Elem1;
 }
@@ -585,7 +579,7 @@ FaceElementTransformations::GetElement1Transformation()
 ElementTransformation &
 FaceElementTransformations::GetElement2Transformation()
 {
-   MFEM_VERIFY(mask & HAVE_ELEM2 && Elem2 != NULL, "The ElementTransformation "
+   MFEM_VERIFY(mask & 2 && Elem2 != NULL, "The ElementTransformation "
               "for the element has not been configured for side 2.");
   return *Elem2;
 }
@@ -593,7 +587,7 @@ FaceElementTransformations::GetElement2Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint1Transformation()
 {
-   MFEM_VERIFY(mask & HAVE_LOC1, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & 4, "The IntegrationPointTransformation "
               "for the element has not been configured for side 1.");
   return Loc1;
 }
@@ -601,7 +595,7 @@ FaceElementTransformations::GetIntPoint1Transformation()
 IntegrationPointTransformation &
 FaceElementTransformations::GetIntPoint2Transformation()
 {
-   MFEM_VERIFY(mask & HAVE_LOC2, "The IntegrationPointTransformation "
+   MFEM_VERIFY(mask & 8, "The IntegrationPointTransformation "
               "for the element has not been configured for side 2.");
   return Loc2;
 }
@@ -609,7 +603,7 @@ FaceElementTransformations::GetIntPoint2Transformation()
 void FaceElementTransformations::Transform(const IntegrationPoint &ip,
                                           Vector &trans)
 {
-   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
+   MFEM_VERIFY(mask & 16, "The ElementTransformation "
               "for the face has not been configured.");
   IsoparametricTransformation::Transform(ip, trans);
 }
@@ -617,7 +611,7 @@ void FaceElementTransformations::Transform(const IntegrationPoint &ip,
 void FaceElementTransformations::Transform(const IntegrationRule &ir,
                                           DenseMatrix &tr)
 {
-   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
+   MFEM_VERIFY(mask & 16, "The ElementTransformation "
               "for the face has not been configured.");
   IsoparametricTransformation::Transform(ir, tr);
 }
@@ -625,7 +619,7 @@ void FaceElementTransformations::Transform(const IntegrationRule &ir,
 void FaceElementTransformations::Transform(const DenseMatrix &matrix,
                                           DenseMatrix &result)
 {
-   MFEM_VERIFY(mask & HAVE_FACE, "The ElementTransformation "
+   MFEM_VERIFY(mask & 16, "The ElementTransformation "
               "for the face has not been configured.");
   IsoparametricTransformation::Transform(matrix, result);
 }
@@ -439,57 +439,15 @@ public:
   void Transform (const IntegrationRule  &, IntegrationRule  &);
 };

-/** @brief A specialized ElementTransformation class representing a face and
-    its two neighboring elements.

-    This class can be used as a container for the element transformation data
-    needed for integrating discontinuous fields on element interfaces in a
-    Discontinuous Galerkin (DG) context.
-
-    The secondary purpose of this class is to enable the
-    GridFunction::GetValue function, and various related functions, to properly
-    evaluate fields with limited continuity on boundary elements.
-*/
 class FaceElementTransformations : public IsoparametricTransformation
 {
 private:
-
-   // Bitwise OR of ConfigMasks
   int mask;

   IntegrationPoint eip1, eip2;

-protected: // interface for Mesh to be able to configure this object.
-
-   friend class Mesh;
-#ifdef MFEM_USE_MPI
-   friend class ParMesh;
-#endif
-
-   /// Set the mask indicating which portions of the object have been setup
-   /** The argument @a m is a bitmask used in
-       Mesh::GetFaceElementTransformations to indicate which portions of the
-       FaceElementTransformations object have been configured.
-
-       mask &  1: Elem1 is configured
-       mask &  2: Elem2 is configured
-       mask &  4: Loc1 is configured
-       mask &  8: Loc2 is configured
-       mask & 16: The Face transformation itself is configured
-   */
-   void SetConfigurationMask(int m) { mask = m; }
-
 public:
-
-   enum ConfigMasks
-   {
-      HAVE_ELEM1 =  1, ///< Element on side 1 is configured
-      HAVE_ELEM2 =  2, ///< Element on side 2 is configured
-      HAVE_LOC1  =  4, ///< Point transformation for side 1 is configured
-      HAVE_LOC2  =  8, ///< Point transformation for side 2 is configured
-      HAVE_FACE  = 16  ///< Face transformation is configured
-   };
-
   int Elem1No, Elem2No;
   Geometry::Type &FaceGeom; ///< @deprecated Use GetGeometryType instead
   ElementTransformation *Elem1, *Elem2;
@@ -508,10 +466,10 @@ public:
   */
   void SetGeometryType(Geometry::Type g) { geom = g; }

-   /** @brief Return the mask defining the configuration state.
-
-       The mask value indicates which portions of FaceElementTransformations
-       object have been configured.
+   /// Set the mask indicating which portions of the object have been setup
+   /** The argument @a m is a bitmask used in
+       Mesh::GetFaceElementTransformations to indicate which portions of the
+       FaceElement Transformations object have been configured.

       mask &  1: Elem1 is configured
       mask &  2: Elem2 is configured
@@ -519,45 +477,12 @@ public:
       mask &  8: Loc2 is configured
       mask & 16: The Face transformation itself is configured
   */
-   int GetConfigurationMask() const { return mask; }
+   void SetConfigurationMask(int m) { mask = m; }
+   int  GetConfigurationMask() const { return mask; }

   /** @brief Set the integration point in the Face and the two neighboring
-       elements, if present.
-
-       The point @a face_ip must be in the reference coordinate system of the
-       face.
-   */
-   void SetIntPoint(const IntegrationPoint *face_ip);
-
-   /** @brief Set the integration point in the Face and the two neighboring
-       elements, if present.
-
-       This is a more expressive member function name than SetIntPoint, which
-       in this special case, does the same thing. This function can be used for
-       greater code clarity.
-   */
-   inline void SetAllIntPoints(const IntegrationPoint *face_ip)
-   { FaceElementTransformations::SetIntPoint(face_ip); }
-
-   /** @brief Get a const reference to the integration point in neighboring
-       element 1 corresponding to the currently set integration point on the
-       face.
-
-       This IntegrationPoint object will only contain up-to-date data if
-       SetIntPoint or SetAllIntPoints has been called with the latest
-       integration point for the face and the appropriate point transformation
-       has been configured. */
-   const IntegrationPoint &GetElement1IntPoint() { return eip1; }
-
-   /** @brief Get a const reference to the integration point in neighboring
-       element 2 corresponding to the currently set integration point on the
-       face.
-
-       This IntegrationPoint object will only contain up-to-date data if
-       SetIntPoint or SetAllIntPoints has been called with the latest
-       integration point for the face and the appropriate point transformation
-       has been configured. */
-   const IntegrationPoint &GetElement2IntPoint() { return eip2; }
+       elements, if present. */
+   void SetIntPoint(const IntegrationPoint *ip);

   virtual void Transform(const IntegrationPoint &, Vector &);
   virtual void Transform(const IntegrationRule &, DenseMatrix &);
@@ -37,8 +37,7 @@ public:
      ClosedUniform   = 4,  ///< Nodes: x_i = i/(n-1),     i=0,...,n-1
      OpenHalfUniform = 5,  ///< Nodes: x_i = (i+1/2)/n,   i=0,...,n-1
      Serendipity     = 6,  ///< Serendipity basis (squares / cubes)
-      ClosedGL        = 7,  ///< Closed GaussLegendre
-      NumBasisTypes   = 8   /**< Keep track of maximum types to prevent
+      NumBasisTypes   = 7   /**< Keep track of maximum types to prevent
                                 hard-coding */
   };
   /** @brief If the input does not represents a valid BasisType, abort with an
@@ -70,7 +69,6 @@ public:
         case ClosedUniform:   return Quadrature1D::ClosedUniform;
         case OpenHalfUniform: return Quadrature1D::OpenHalfUniform;
         case Serendipity:     return Quadrature1D::GaussLobatto;
-         case ClosedGL:        return Quadrature1D::ClosedGL;
      }
      return Quadrature1D::Invalid;
   }
@@ -84,7 +82,6 @@ public:
         case Quadrature1D::OpenUniform:     return OpenUniform;
         case Quadrature1D::ClosedUniform:   return ClosedUniform;
         case Quadrature1D::OpenHalfUniform: return OpenHalfUniform;
-         case Quadrature1D::ClosedGL:        return ClosedGL;
      }
      return Invalid;
   }
@@ -756,12 +756,6 @@ public:
   /// Return the total number of quadrature points.
   int GetSize() const { return size; }

-   /// Returns the mesh
-   inline Mesh *GetMesh() const { return mesh; }
-
-   /// Returns number of elements in the mesh.
-   inline int GetNE() const { return mesh->GetNE(); }
-
   /// Get the IntegrationRule associated with mesh element @a idx.
   const IntegrationRule &GetElementIntRule(int idx) const
   { return *int_rule[mesh->GetElementBaseGeometry(idx)]; }
@@ -1344,14 +1344,15 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
            return NULL;
         }
         ir = FindInIntPts(Geom, Times-1);
-         if (ir) { return ir; }
-
-         ir = new IntegrationRule(Times-1);
-         for (int i = 1; i < Times; i++)
+         if (ir == NULL)
         {
-            IntegrationPoint &ip = ir->IntPoint(i-1);
-            ip.x = double(i) / Times;
-            ip.y = ip.z = 0.0;
+            ir = new IntegrationRule(Times-1);
+            for (int i = 1; i < Times; i++)
+            {
+               IntegrationPoint &ip = ir->IntPoint(i-1);
+               ip.x = double(i) / Times;
+               ip.y = ip.z = 0.0;
+            }
         }
      }
      break;
@@ -1363,17 +1364,18 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
            return NULL;
         }
         ir = FindInIntPts(Geom, ((Times-1)*(Times-2))/2);
-         if (ir) { return ir; }
-
-         ir = new IntegrationRule(((Times-1)*(Times-2))/2);
-         for (int k = 0, j = 1; j < Times-1; j++)
-            for (int i = 1; i < Times-j; i++, k++)
-            {
-               IntegrationPoint &ip = ir->IntPoint(k);
-               ip.x = double(i) / Times;
-               ip.y = double(j) / Times;
-               ip.z = 0.0;
-            }
+         if (ir == NULL)
+         {
+            ir = new IntegrationRule(((Times-1)*(Times-2))/2);
+            for (int k = 0, j = 1; j < Times-1; j++)
+               for (int i = 1; i < Times-j; i++, k++)
+               {
+                  IntegrationPoint &ip = ir->IntPoint(k);
+                  ip.x = double(i) / Times;
+                  ip.y = double(j) / Times;
+                  ip.z = 0.0;
+               }
+         }
      }
      break;

@@ -1384,17 +1386,18 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
            return NULL;
         }
         ir = FindInIntPts(Geom, (Times-1)*(Times-1));
-         if (ir) { return ir; }
-
-         ir = new IntegrationRule((Times-1)*(Times-1));
-         for (int k = 0, j = 1; j < Times; j++)
-            for (int i = 1; i < Times; i++, k++)
-            {
-               IntegrationPoint &ip = ir->IntPoint(k);
-               ip.x = double(i) / Times;
-               ip.y = double(j) / Times;
-               ip.z = 0.0;
-            }
+         if (ir == NULL)
+         {
+            ir = new IntegrationRule((Times-1)*(Times-1));
+            for (int k = 0, j = 1; j < Times; j++)
+               for (int i = 1; i < Times; i++, k++)
+               {
+                  IntegrationPoint &ip = ir->IntPoint(k);
+                  ip.x = double(i) / Times;
+                  ip.y = double(j) / Times;
+                  ip.z = 0.0;
+               }
+         }
      }
      break;

@@ -1402,121 +1405,10 @@ const IntegrationRule *GeometryRefiner::RefineInterior(Geometry::Type Geom,
         mfem_error("GeometryRefiner::RefineInterior(...)");
   }

-   MFEM_ASSERT(ir != NULL, "Failed to construct the refined IntegrationRule.");
-   IntPts[Geom].Append(ir);
-
+   if (ir) { IntPts[Geom].Append(ir); }
   return ir;
 }

-
-int GeometryRefiner::GetRefinementLevelFromPoints(Geometry::Type geom, int Npts)
-{
-   switch (geom)
-   {
-      case Geometry::POINT:
-      {
-         return -1;
-      }
-      case Geometry::SEGMENT:
-      {
-         return Npts -1;
-      }
-      case Geometry::TRIANGLE:
-      {
-         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
-         {
-            np = (n+1)*(n+2)/2;
-            if (np == Npts) { return n; }
-         }
-         return -1;
-      }
-      case Geometry::SQUARE:
-      {
-         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
-         {
-            np = (n+1)*(n+1);
-            if (np == Npts) { return n; }
-         }
-         return -1;
-      }
-      case Geometry::CUBE:
-      {
-         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
-         {
-            np = (n+1)*(n+1)*(n+1);
-            if (np == Npts) { return n; }
-         }
-         return -1;
-      }
-      case Geometry::TETRAHEDRON:
-      {
-         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
-         {
-            np = (n+3)*(n+2)*(n+1)/6;
-            if (np == Npts) { return n; }
-         }
-         return -1;
-      }
-      case Geometry::PRISM:
-      {
-         for (int n = 0, np = 0; (n < 15) && (np < Npts) ; n++)
-         {
-            np = (n+1)*(n+1)*(n+2)/2;
-            if (np == Npts) { return n; }
-         }
-         return -1;
-      }
-      default:
-      {
-         mfem_error("Non existing Geometry.");
-      }
-   }
-
-   return -1;
-}
-
-
-int GeometryRefiner::GetRefinementLevelFromElems(Geometry::Type geom, int Nels)
-{
-   switch (geom)
-   {
-      case Geometry::POINT:
-      {
-         return -1;
-      }
-      case Geometry::SEGMENT:
-      {
-         return Nels;
-      }
-      case Geometry::TRIANGLE:
-      case Geometry::SQUARE:
-      {
-         for (int n = 0; (n < 15) && (n*n < Nels+1) ; n++)
-         {
-            if (n*n == Nels) { return n-1; }
-         }
-         return -1;
-      }
-      case Geometry::CUBE:
-      case Geometry::TETRAHEDRON:
-      case Geometry::PRISM:
-      {
-         for (int n = 0; (n < 15) && (n*n*n < Nels+1) ; n++)
-         {
-            if (n*n*n == Nels) { return n-1; }
-         }
-         return -1;
-      }
-      default:
-      {
-         mfem_error("Non existing Geometry.");
-      }
-   }
-
-   return -1;
-}
-
-
 GeometryRefiner GlobGeometryRefiner;

 }
@@ -273,12 +273,6 @@ public:
   /// @note This method always uses Quadrature1D::OpenUniform points.
   const IntegrationRule *RefineInterior(Geometry::Type Geom, int Times);

-   /// Get the Refinement level based on number of points
-   virtual int GetRefinementLevelFromPoints(Geometry::Type Geom, int Npts);
-
-   /// Get the Refinement level based on number of elements
-   virtual int GetRefinementLevelFromElems(Geometry::Type geom, int Npts);
-
   ~GeometryRefiner();
 };

@@ -199,7 +199,8 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
   if (f != fes) { Destroy(); }
   fes = f;
   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
+                    fes->GetVSize(), true);
   sequence = fes->GetSequence();
 }

@@ -396,16 +397,8 @@ const
   fes->DofsToVDofs(vdim-1, dofs);
   Vector DofVal(dofs.Size()), LocVec;
   const FiniteElement *fe = fes->GetFE(i);
-   if (fe->GetMapType() == FiniteElement::VALUE)
-   {
-      fe->CalcShape(ip, DofVal);
-   }
-   else
-   {
-      ElementTransformation *Tr = fes->GetElementTransformation(i);
-      Tr->SetIntPoint(&ip);
-      fe->CalcPhysShape(*Tr, DofVal);
-   }
+   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
+   fe->CalcShape(ip, DofVal);
   GetSubVector(dofs, LocVec);

   return (DofVal * LocVec);
@@ -422,17 +415,10 @@ void GridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
   GetSubVector(vdofs, loc_data);
   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
+      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
+                  "invalid FE map type");
      Vector shape(dof);
-      if (FElem->GetMapType() == FiniteElement::VALUE)
-      {
-         FElem->CalcShape(ip, shape);
-      }
-      else
-      {
-         ElementTransformation *Tr = fes->GetElementTransformation(i);
-         Tr->SetIntPoint(&ip);
-         FElem->CalcPhysShape(*Tr, shape);
-      }
+      FElem->CalcShape(ip, shape);
      int vdim = fes->GetVDim();
      val.SetSize(vdim);
      for (int k = 0; k < vdim; k++)
@@ -766,21 +752,19 @@ double GridFunction::GetValue(ElementTransformation &T,
            IntegrationPoint fip;
            be_to_bfe(FET->GetGeometryType(), o, ip, fip);

-            // Compute and set the point in element 1 from fip
-            FET->SetAllIntPoints(&fip);
+            FET->SetIntPoint(&fip);
            ElementTransformation & T1 = FET->GetElement1Transformation();
            return GetValue(T1, T1.GetIntPoint(), comp);
         }
+         break;
      }
-      break;
      case ElementTransformation::BDR_FACE:
      {
         FaceElementTransformations * FET =
            dynamic_cast<FaceElementTransformations *>(&T);

         // Evaluate in neighboring element for both continuous and
-         // discontinuous fields (the integration point in T1 should have
-         // already been set).
+         // discontinuous fields.
         ElementTransformation & T1 = FET->GetElement1Transformation();
         return GetValue(T1, T1.GetIntPoint(), comp);
      }
@@ -904,21 +888,19 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
            IntegrationPoint fip;
            be_to_bfe(FET->GetGeometryType(), o, ip, fip);

-            // Compute and set the point in element 1 from fip
-            FET->SetAllIntPoints(&fip);
+            FET->SetIntPoint(&fip);
            ElementTransformation & T1 = FET->GetElement1Transformation();
            return GetVectorValue(T1, T1.GetIntPoint(), val);
         }
+         break;
      }
-      break;
      case ElementTransformation::BDR_FACE:
      {
         FaceElementTransformations * FET =
            dynamic_cast<FaceElementTransformations *>(&T);

         // Evaluate in neighboring element for both continuous and
-         // discontinuous fields (the integration point in T1 should have
-         // already been set).
+         // discontinuous fields.
         ElementTransformation & T1 = FET->GetElement1Transformation();
         return GetVectorValue(T1, T1.GetIntPoint(), val);
      }
@@ -1356,262 +1338,107 @@ void GridFunction::GetVectorGradientHat(
   MultAtB(loc_data_mat, dshape, gh);
 }

-double GridFunction::GetDivergence(ElementTransformation &T) const
+double GridFunction::GetDivergence(ElementTransformation &tr) const
 {
-   switch (T.ElementType)
+   double div_v;
+   int elNo = tr.ElementNo;
+   const FiniteElement *FElem = fes->GetFE(elNo);
+   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
-      case ElementTransformation::ELEMENT:
+      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
+                  "invalid FE map type");
+      DenseMatrix grad_hat;
+      GetVectorGradientHat(tr, grad_hat);
+      const DenseMatrix &Jinv = tr.InverseJacobian();
+      div_v = 0.0;
+      for (int i = 0; i < Jinv.Width(); i++)
      {
-         int elNo = T.ElementNo;
-         const FiniteElement *fe = fes->GetFE(elNo);
-         if (fe->GetRangeType() == FiniteElement::SCALAR)
+         for (int j = 0; j < Jinv.Height(); j++)
         {
-            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
-                        "invalid FE map type");
-            DenseMatrix grad_hat;
-            GetVectorGradientHat(T, grad_hat);
-            const DenseMatrix &Jinv = T.InverseJacobian();
-            double div_v = 0.0;
-            for (int i = 0; i < Jinv.Width(); i++)
-            {
-               for (int j = 0; j < Jinv.Height(); j++)
-               {
-                  div_v += grad_hat(i, j) * Jinv(j, i);
-               }
-            }
-            return div_v;
+            div_v += grad_hat(i, j) * Jinv(j, i);
         }
-         else
-         {
-            // Assuming RT-type space
-            Array<int> dofs;
-            fes->GetElementDofs(elNo, dofs);
-            Vector loc_data, divshape(fe->GetDof());
-            GetSubVector(dofs, loc_data);
-            fe->CalcDivShape(T.GetIntPoint(), divshape);
-            return (loc_data * divshape) / T.Weight();
-         }
-      }
-      break;
-      case ElementTransformation::BDR_ELEMENT:
-      {
-         // In order to properly capture the derivative of the normal component
-         // of the field (as well as the transverse divergence of the
-         // tangential compoents) we must evaluate it in the neighboring
-         // element.
-         FaceElementTransformations * FET =
-            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
-
-         // Boundary elements and Boundary Faces may have different
-         // orientations so adjust the integration point if necessary.
-         int o = 0;
-         if (fes->GetMesh()->Dimension() == 3)
-         {
-            int f;
-            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         }
-
-         IntegrationPoint fip;
-         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
-
-         // Compute and set the point in element 1 from fip
-         FET->SetAllIntPoints(&fip);
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-
-         return GetDivergence(T1);
-      }
-      break;
-      case ElementTransformation::BDR_FACE:
-      {
-         // This must be a DG context so this dynamic cast must succeed.
-         FaceElementTransformations * FET =
-            dynamic_cast<FaceElementTransformations *>(&T);
-
-         // Evaluate in neighboring element (the integration point in T1 should
-         // have already been set).
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-         return GetDivergence(T1);
-      }
-      break;
-      default:
-      {
-         MFEM_ABORT("GridFunction::GetDivergence: Unsupported element type \""
-                    << T.ElementType << "\"");
      }
   }
-   return 0.0; // never reached
+   else
+   {
+      // Assuming RT-type space
+      Array<int> dofs;
+      fes->GetElementDofs(elNo, dofs);
+      Vector loc_data, divshape(FElem->GetDof());
+      GetSubVector(dofs, loc_data);
+      FElem->CalcDivShape(tr.GetIntPoint(), divshape);
+      div_v = (loc_data * divshape) / tr.Weight();
+   }
+   return div_v;
 }

-void GridFunction::GetCurl(ElementTransformation &T, Vector &curl) const
+void GridFunction::GetCurl(ElementTransformation &tr, Vector &curl) const
 {
-   switch (T.ElementType)
+   int elNo = tr.ElementNo;
+   const FiniteElement *FElem = fes->GetFE(elNo);
+   if (FElem->GetRangeType() == FiniteElement::SCALAR)
   {
-      case ElementTransformation::ELEMENT:
+      MFEM_ASSERT(FElem->GetMapType() == FiniteElement::VALUE,
+                  "invalid FE map type");
+      DenseMatrix grad_hat;
+      GetVectorGradientHat(tr, grad_hat);
+      const DenseMatrix &Jinv = tr.InverseJacobian();
+      DenseMatrix grad(grad_hat.Height(), Jinv.Width()); // vdim x FElem->Dim
+      Mult(grad_hat, Jinv, grad);
+      MFEM_ASSERT(grad.Height() == grad.Width(), "");
+      if (grad.Height() == 3)
      {
-         int elNo = T.ElementNo;
-         const FiniteElement *fe = fes->GetFE(elNo);
-         if (fe->GetRangeType() == FiniteElement::SCALAR)
-         {
-            MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
-                        "invalid FE map type");
-            DenseMatrix grad_hat;
-            GetVectorGradientHat(T, grad_hat);
-            const DenseMatrix &Jinv = T.InverseJacobian();
-            // Dimensions of grad are vdim x FElem->Dim
-            DenseMatrix grad(grad_hat.Height(), Jinv.Width());
-            Mult(grad_hat, Jinv, grad);
-            MFEM_ASSERT(grad.Height() == grad.Width(), "");
-            if (grad.Height() == 3)
-            {
-               curl.SetSize(3);
-               curl(0) = grad(2,1) - grad(1,2);
-               curl(1) = grad(0,2) - grad(2,0);
-               curl(2) = grad(1,0) - grad(0,1);
-            }
-            else if (grad.Height() == 2)
-            {
-               curl.SetSize(1);
-               curl(0) = grad(1,0) - grad(0,1);
-            }
-         }
-         else
-         {
-            // Assuming ND-type space
-            Array<int> dofs;
-            fes->GetElementDofs(elNo, dofs);
-            Vector loc_data;
-            GetSubVector(dofs, loc_data);
-            DenseMatrix curl_shape(fe->GetDof(), fe->GetDim() == 3 ? 3 : 1);
-            fe->CalcCurlShape(T.GetIntPoint(), curl_shape);
-            curl.SetSize(curl_shape.Width());
-            if (curl_shape.Width() == 3)
-            {
-               double curl_hat[3];
-               curl_shape.MultTranspose(loc_data, curl_hat);
-               T.Jacobian().Mult(curl_hat, curl);
-            }
-            else
-            {
-               curl_shape.MultTranspose(loc_data, curl);
-            }
-            curl /= T.Weight();
-         }
+         curl.SetSize(3);
+         curl(0) = grad(2,1) - grad(1,2);
+         curl(1) = grad(0,2) - grad(2,0);
+         curl(2) = grad(1,0) - grad(0,1);
      }
-      break;
-      case ElementTransformation::BDR_ELEMENT:
+      else if (grad.Height() == 2)
      {
-         // In order to capture the tangential components of the curl we
-         // must evaluate it in the neighboring element.
-         FaceElementTransformations * FET =
-            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
-
-         // Boundary elements and Boundary Faces may have different
-         // orientations so adjust the integration point if necessary.
-         int o = 0;
-         if (fes->GetMesh()->Dimension() == 3)
-         {
-            int f;
-            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         }
-
-         IntegrationPoint fip;
-         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
-
-         // Compute and set the point in element 1 from fip
-         FET->SetAllIntPoints(&fip);
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-
-         GetCurl(T1, curl);
+         curl.SetSize(1);
+         curl(0) = grad(1,0) - grad(0,1);
      }
-      break;
-      case ElementTransformation::BDR_FACE:
+   }
+   else
+   {
+      // Assuming ND-type space
+      Array<int> dofs;
+      fes->GetElementDofs(elNo, dofs);
+      Vector loc_data;
+      GetSubVector(dofs, loc_data);
+      DenseMatrix curl_shape(FElem->GetDof(), FElem->GetDim() == 3 ? 3 : 1);
+      FElem->CalcCurlShape(tr.GetIntPoint(), curl_shape);
+      curl.SetSize(curl_shape.Width());
+      if (curl_shape.Width() == 3)
      {
-         // This must be a DG context so this dynamic cast must succeed.
-         FaceElementTransformations * FET =
-            dynamic_cast<FaceElementTransformations *>(&T);
-
-         // Evaluate in neighboring element (the integration point in T1 should
-         // have already been set).
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-         GetCurl(T1, curl);
+         double curl_hat[3];
+         curl_shape.MultTranspose(loc_data, curl_hat);
+         tr.Jacobian().Mult(curl_hat, curl);
      }
-      break;
-      default:
+      else
      {
-         MFEM_ABORT("GridFunction::GetCurl: Unsupported element type \""
-                    << T.ElementType << "\"");
+         curl_shape.MultTranspose(loc_data, curl);
      }
+      curl /= tr.Weight();
   }
 }

-void GridFunction::GetGradient(ElementTransformation &T, Vector &grad) const
+void GridFunction::GetGradient(ElementTransformation &tr, Vector &grad) const
 {
-   switch (T.ElementType)
-   {
-      case ElementTransformation::ELEMENT:
-      {
-         const FiniteElement * fe = fes->GetFE(T.ElementNo);
-         MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE,
-                     "invalid FE map type");
-         int spaceDim = fes->GetMesh()->SpaceDimension();
-         int dim = fe->GetDim(), dof = fe->GetDof();
-         DenseMatrix dshape(dof, dim);
-         Vector lval, gh(dim);
-         Array<int> dofs;
+   int elNo = tr.ElementNo;
+   const FiniteElement *fe = fes->GetFE(elNo);
+   MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
+   int dim = fe->GetDim(), dof = fe->GetDof();
+   DenseMatrix dshape(dof, dim);
+   Vector lval, gh(dim);
+   Array<int> dofs;

-         grad.SetSize(spaceDim);
-         fes->GetElementDofs(T.ElementNo, dofs);
-         GetSubVector(dofs, lval);
-         fe->CalcDShape(T.GetIntPoint(), dshape);
-         dshape.MultTranspose(lval, gh);
-         T.InverseJacobian().MultTranspose(gh, grad);
-      }
-      break;
-      case ElementTransformation::BDR_ELEMENT:
-      {
-         // In order to properly capture the normal component of the gradient
-         // as well as its tangential components we must evaluate it in the
-         // neighboring element.
-         FaceElementTransformations * FET =
-            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
-
-         // Boundary elements and Boundary Faces may have different
-         // orientations so adjust the integration point if necessary.
-         int o = 0;
-         if (fes->GetMesh()->Dimension() == 3)
-         {
-            int f;
-            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         }
-
-         IntegrationPoint fip;
-         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
-
-         // Compute and set the point in element 1 from fip
-         FET->SetAllIntPoints(&fip);
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-
-         GetGradient(T1, grad);
-      }
-      break;
-      case ElementTransformation::BDR_FACE:
-      {
-         // This must be a DG context so this dynamic cast must succeed.
-         FaceElementTransformations * FET =
-            dynamic_cast<FaceElementTransformations *>(&T);
-
-         // Evaluate in neighboring element (the integration point in T1 should
-         // have already been set).
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-         GetGradient(T1, grad);
-      }
-      break;
-      default:
-      {
-         MFEM_ABORT("GridFunction::GetGradient: Unsupported element type \""
-                    << T.ElementType << "\"");
-      }
-   }
+   grad.SetSize(dim);
+   fes->GetElementDofs(elNo, dofs);
+   GetSubVector(dofs, lval);
+   fe->CalcDShape(tr.GetIntPoint(), dshape);
+   dshape.MultTranspose(lval, gh);
+   tr.InverseJacobian().MultTranspose(gh, grad);
 }

 void GridFunction::GetGradients(ElementTransformation &tr,
@@ -1640,65 +1467,15 @@ void GridFunction::GetGradients(ElementTransformation &tr,
 }

 void GridFunction::GetVectorGradient(
-   ElementTransformation &T, DenseMatrix &grad) const
+   ElementTransformation &tr, DenseMatrix &grad) const
 {
-   switch (T.ElementType)
-   {
-      case ElementTransformation::ELEMENT:
-      {
-         MFEM_ASSERT(fes->GetFE(T.ElementNo)->GetMapType() ==
-                     FiniteElement::VALUE, "invalid FE map type");
-         DenseMatrix grad_hat;
-         GetVectorGradientHat(T, grad_hat);
-         const DenseMatrix &Jinv = T.InverseJacobian();
-         grad.SetSize(grad_hat.Height(), Jinv.Width());
-         Mult(grad_hat, Jinv, grad);
-      }
-      break;
-      case ElementTransformation::BDR_ELEMENT:
-      {
-         // In order to capture the normal component of the gradient we
-         // must evaluate it in the neighboring element.
-         FaceElementTransformations * FET =
-            fes->GetMesh()->GetBdrFaceTransformations(T.ElementNo);
-
-         // Boundary elements and Boundary Faces may have different
-         // orientations so adjust the integration point if necessary.
-         int o = 0;
-         if (fes->GetMesh()->Dimension() == 3)
-         {
-            int f;
-            fes->GetMesh()->GetBdrElementFace(T.ElementNo, &f, &o);
-         }
-
-         IntegrationPoint fip;
-         be_to_bfe(FET->GetGeometryType(), o, T.GetIntPoint(), fip);
-
-         // Compute and set the point in element 1 from fip
-         FET->SetAllIntPoints(&fip);
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-
-         GetVectorGradient(T1, grad);
-      }
-      break;
-      case ElementTransformation::BDR_FACE:
-      {
-         // This must be a DG context so this dynamic cast must succeed.
-         FaceElementTransformations * FET =
-            dynamic_cast<FaceElementTransformations *>(&T);
-
-         // Evaluate in neighboring element (the integration point in T1 should
-         // have already been set).
-         ElementTransformation & T1 = FET->GetElement1Transformation();
-         GetVectorGradient(T1, grad);
-      }
-      break;
-      default:
-      {
-         MFEM_ABORT("GridFunction::GetVectorGradient: "
-                    "Unsupported element type \"" << T.ElementType << "\"");
-      }
-   }
+   MFEM_ASSERT(fes->GetFE(tr.ElementNo)->GetMapType() == FiniteElement::VALUE,
+               "invalid FE map type");
+   DenseMatrix grad_hat;
+   GetVectorGradientHat(tr, grad_hat);
+   const DenseMatrix &Jinv = tr.InverseJacobian();
+   grad.SetSize(grad_hat.Height(), Jinv.Width());
+   Mult(grad_hat, Jinv, grad);
 }

 void GridFunction::GetElementAverages(GridFunction &avgs) const
@@ -105,7 +105,7 @@ public:
       have the same size.

       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
+       assignemnt operator. */
   GridFunction &operator=(const GridFunction &rhs)
   { return operator=((const Vector &)rhs); }

@@ -162,8 +162,7 @@ public:
                           int vdim = 1) const;

   /** Return a vector value from within the given element. */
-   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
-                               Vector &val) const;
+   void GetVectorValue(int i, const IntegrationPoint &ip, Vector &val) const;
   ///@}

   /** @name Element Index Get Values Methods
@@ -209,14 +208,13 @@ public:
   ///@{
   /** Return a scalar value from within the element indicated by the
       ElementTransformation Object. */
-   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
-                           int comp = 0, Vector *tr = NULL) const;
+   double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
+                   int comp = 0, Vector *tr = NULL) const;

   /** Return a vector value from within the element indicated by the
       ElementTransformation Object. */
-   virtual void GetVectorValue(ElementTransformation &T,
-                               const IntegrationPoint &ip,
-                               Vector &val, Vector *tr = NULL) const;
+   void GetVectorValue(ElementTransformation &T, const IntegrationPoint &ip,
+                       Vector &val, Vector *tr = NULL) const;
   ///@}

   /** @name ElementTransformation Get Values Methods
@@ -714,7 +712,7 @@ public:
       the same size.

       @note Defining this method overwrites the implicitly defined copy
-       assignment operator. */
+       assignemnt operator. */
   QuadratureFunction &operator=(const QuadratureFunction &v);

   /// Get the IntegrationRule associated with mesh element @a idx.
@@ -192,7 +192,6 @@ void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
   const int ncomp      = field_in.FESpace()->GetVDim(),
             points_fld = field_in.Size() / ncomp,
             points_cnt = codes.Size();
-   field_out.SetSize(points_cnt*ncomp);

   for (int i = 0; i < ncomp; i++)
   {
@@ -618,26 +618,6 @@ void QuadratureFunctions1D::OpenHalfUniform(const int np, IntegrationRule* ir)
   CalculateUniformWeights(ir, Quadrature1D::OpenHalfUniform);
 }

-void QuadratureFunctions1D::ClosedGL(const int np, IntegrationRule* ir)
-{
-   ir->SetSize(np);
-   ir->IntPoint(0).x = 0.0;
-   ir->IntPoint(np-1).x = 1.0;
-
-   if ( np > 2 )
-   {
-      IntegrationRule gl_ir;
-      GaussLegendre(np-1, &gl_ir);
-
-      for (int i = 1; i < np-1; ++i)
-      {
-         ir->IntPoint(i).x = (gl_ir.IntPoint(i-1).x + gl_ir.IntPoint(i).x)/2;
-      }
-   }
-
-   CalculateUniformWeights(ir, Quadrature1D::ClosedGL);
-}
-
 void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
                                           const int type)
 {
@@ -670,11 +650,6 @@ void QuadratureFunctions1D::GivePolyPoints(const int np, double *pts,
         OpenHalfUniform(np, &ir);
         break;
      }
-      case Quadrature1D::ClosedGL:
-      {
-         ClosedGL(np, &ir);
-         break;
-      }
      default:
      {
         MFEM_ABORT("Asking for an unknown type of 1D Quadrature points, "
@@ -272,7 +272,6 @@ public:
   void OpenUniform(const int np, IntegrationRule *ir);
   void ClosedUniform(const int np, IntegrationRule *ir);
   void OpenHalfUniform(const int np, IntegrationRule *ir);
-   void ClosedGL(const int np, IntegrationRule *ir);
   ///@}

   /// A helper function that will play nice with Poly_1D::OpenPoints and
@@ -294,8 +293,7 @@ public:
      GaussLobatto    = 1,
      OpenUniform     = 2,  ///< aka open Newton-Cotes
      ClosedUniform   = 3,  ///< aka closed Newton-Cotes
-      OpenHalfUniform = 4,  ///< aka "open half" Newton-Cotes
-      ClosedGL        = 5   ///< aka closed Gauss Legendre
+      OpenHalfUniform = 4   ///< aka "open half" Newton-Cotes
   };
   /** @brief If the Quadrature1D type is not closed return Invalid; otherwise
       return type. */
@@ -97,8 +97,6 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
   Vector qweight(Q);
   Vector shape_i(P);
   DenseMatrix grad_i(P, dim);
-
-   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
   const Table &el_dof = fes.GetElementToDofTable();
   Array<int> tp_el_dof(el_dof.Size_of_connections());
   const TensorBasisElement * tfe =
@@ -130,15 +128,7 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
         const int el_offset = fe->GetDof() * i;
         for (int j = 0; j < fe->GetDof(); j++)
         {
-            if (compstride == 1)
-            {
-               tp_el_dof[j + el_offset] = fes.GetVDim()*
-                                          el_dof.GetJ()[dof_map[j] + el_offset];
-            }
-            else
-            {
-               tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
-            }
+            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
         }
      }
   }
@@ -167,23 +157,20 @@ static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
      {
         for (int i = 0; i < P; i++)
         {
-            if (compstride == 1)
-            {
-               tp_el_dof[i + e*P] = fes.GetVDim()*el_dof.GetJ()[i + e*P];
-            }
-            else
-            {
-               tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
-            }
+            tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
         }
      }
   }
   CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
                     fe->GetDof(), ir.GetNPoints(), shape.GetData(),
                     grad.GetData(), qref.GetData(), qweight.GetData(), basis);
-   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
-                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
-                             CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedInterlaceMode imode = CEED_NONINTERLACED;
+   if (fes.GetOrdering()==Ordering::byVDIM)
+   {
+      imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
+                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
                             tp_el_dof.GetData(), restr);
 }

@@ -228,7 +215,6 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
                           grad1d.GetData(), qref1d.GetData(),
                           qweight1d.GetData(), basis);

-   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
   const Table &el_dof = fes.GetElementToDofTable();
   Array<int> tp_el_dof(el_dof.Size_of_connections());
   for (int i = 0; i < mesh->GetNE(); i++)
@@ -236,20 +222,16 @@ static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
      const int el_offset = fe->GetDof() * i;
      for (int j = 0; j < fe->GetDof(); j++)
      {
-         if (compstride == 1)
-         {
-            tp_el_dof[j + el_offset] = fes.GetVDim()*
-                                       el_dof.GetJ()[dof_map[j] + el_offset];
-         }
-         else
-         {
-            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
-         }
+         tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
      }
   }
-   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
-                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
-                             CEED_MEM_HOST, CEED_COPY_VALUES,
+   CeedInterlaceMode imode = CEED_NONINTERLACED;
+   if (fes.GetOrdering()==Ordering::byVDIM)
+   {
+      imode = CEED_INTERLACED;
+   }
+   CeedElemRestrictionCreate(ceed, imode, mesh->GetNE(), fe->GetDof(),
+                             fes.GetNDofs(), fes.GetVDim(), CEED_MEM_HOST, CEED_COPY_VALUES,
                             tp_el_dof.GetData(), restr);
 }

@@ -316,9 +298,8 @@ void CeedPAAssemble(const CeedPAOperator& op,
   CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);

   const int qdatasize = op.qdatasize;
-   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, qdatasize,
-                                    nelem*nqpts*qdatasize, CEED_STRIDES_BACKEND,
-                                    &ceedData.restr_i);
+   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, nelem*nqpts, qdatasize,
+                                    CEED_STRIDES_BACKEND, &ceedData.restr_i);

   CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
   CeedVectorSetArray(ceedData.node_coords, CEED_MEM_HOST, CEED_USE_POINTER,
@@ -199,19 +199,10 @@ void LinearForm::Assemble()
 void LinearForm::Update(FiniteElementSpace *f, Vector &v, int v_offset)
 {
   fes = f;
-   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, f->GetVSize()),
-                    f->GetVSize(), false);
+   NewDataAndSize((double *)v + v_offset, fes->GetVSize());
   ResetDeltaLocations();
 }

-void LinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
-{
-   MFEM_ASSERT(v.Size() >= v_offset + f->GetVSize(), "");
-   fes = f;
-   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
-}
-
 void LinearForm::AssembleDelta()
 {
   if (dlfi_delta.Size() == 0) { return; }
@@ -26,7 +26,7 @@ protected:
   /// FE space on which the LinearForm lives. Not owned.
   FiniteElementSpace *fes;

-   /** @brief Indicates the LinearFormIntegrator%s stored in #dlfi, #dlfi_delta,
+   /** @brief Indicates the LinerFormIntegrator%s stored in #dlfi, #dlfi_delta,
       #blfi, and #flfi are owned by another LinearForm. */
   int extern_lfs;

@@ -175,15 +175,6 @@ public:
       @note This method does not perform assembly. */
   void Update(FiniteElementSpace *f, Vector &v, int v_offset);

-   /** @brief Make the LinearForm reference external data on a new
-       FiniteElementSpace. */
-   /** This method changes the FiniteElementSpace associated with the LinearForm
-       @a *f and sets the data of the Vector @a v (plus the @a v_offset)
-       as external data in the LinearForm.
-       @note This version of the method will also perform bounds checks when
-       the build option MFEM_DEBUG is enabled. */
-   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
-
   /// Return the action of the LinearForm as a linear mapping.
   /** Linear forms are linear functionals which map GridFunctions to
       the real numbers.  This method performs this mapping which in
@@ -159,13 +159,10 @@ void BoundaryLFIntegrator::AssembleRHSElementVect(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
+      IntegrationPoint eip;
+      Tr.Loc1.Transform(ip, eip);

-      // Set the integration point in the face and the neighboring element
-      Tr.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
-
+      Tr.Face->SetIntPoint (&ip);
      double val = Tr.Face->Weight() * ip.weight * Q.Eval(*Tr.Face, ip);

      el.CalcShape(eip, shape);
@@ -362,12 +359,10 @@ void VectorBoundaryLFIntegrator::AssembleRHSElementVect(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
+      IntegrationPoint eip;
+      Tr.Loc1.Transform(ip, eip);

-      // Set the integration point in the face and the neighboring element
-      Tr.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+      Tr.SetIntPoint(&ip);

      // Use Tr transformation in case Q depends on boundary attribute
      Q.Eval(vec, Tr, ip);
@@ -688,14 +683,12 @@ void BoundaryFlowIntegrator::AssembleRHSElementVect(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring element
-      Tr.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+      IntegrationPoint eip;
+      Tr.Loc1.Transform(ip, eip);
      el.CalcShape(eip, shape);

+      Tr.SetIntPoint(&ip);
+
      // Use Tr.Elem1 transformation for u so that it matches the coefficient
      // used with the ConvectionIntegrator and/or the DGTraceIntegrator.
      u->Eval(vu, *Tr.Elem1, eip);
@@ -759,13 +752,10 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip;

-      // Set the integration point in the face and the neighboring element
-      Tr.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
-
+      Tr.Loc1.Transform(ip, eip);
+      Tr.SetIntPoint(&ip);
      if (dim == 1)
      {
         nor(0) = 2*eip.x - 1.0;
@@ -784,14 +774,14 @@ void DGDirichletLFIntegrator::AssembleRHSElementVect(
      {
         if (Q)
         {
-            w *= Q->Eval(*Tr.Elem1, eip);
+            w *= Q->Eval(Tr, ip);
         }
         ni.Set(w, nor);
      }
      else
      {
         nh.Set(w, nor);
-         MQ->Eval(mq, *Tr.Elem1, eip);
+         MQ->Eval(mq, Tr, ip);
         mq.MultTranspose(nh, ni);
      }
      CalcAdjugate(Tr.Elem1->Jacobian(), adjJ);
@@ -855,12 +845,9 @@ void DGElasticityDirichletLFIntegrator::AssembleRHSElementVect(
   for (int pi = 0; pi < ir->GetNPoints(); ++pi)
   {
      const IntegrationPoint &ip = ir->IntPoint(pi);
-
-      // Set the integration point in the face and the neighboring element
-      Tr.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Tr.GetElement1IntPoint();
+      IntegrationPoint eip;
+      Tr.Loc1.Transform(ip, eip);
+      Tr.SetIntPoint(&ip);

      // Evaluate the Dirichlet b.c. using the face transformation.
      uD.Eval(u_dir, Tr, ip);
@@ -135,18 +135,11 @@ void Multigrid::SetOperator(const Operator& op)
   MFEM_ABORT("SetOperator not supported in Multigrid");
 }

-void Multigrid::SmoothingStep(int level, bool transpose) const
+void Multigrid::SmoothingStep(int level) const
 {
   GetOperatorAtLevel(level)->Mult(*Y[level], *R[level]); // r = A x
   subtract(*X[level], *R[level], *R[level]);             // r = b - A x
-   if (transpose)
-   {
-      GetSmootherAtLevel(level)->MultTranspose(*R[level], *Z[level]); // z = S r
-   }
-   else
-   {
-      GetSmootherAtLevel(level)->Mult(*R[level], *Z[level]); // z = S r
-   }
+   GetSmootherAtLevel(level)->Mult(*R[level], *Z[level]); // z = S r
   add(*Y[level], 1.0, *Z[level], *Y[level]);             // x = x + S (b - A x)
 }

@@ -160,7 +153,7 @@ void Multigrid::Cycle(int level) const

   for (int i = 0; i < preSmoothingSteps; i++)
   {
-      SmoothingStep(level, false);
+      SmoothingStep(level);
   }

   // Compute residual
@@ -194,7 +187,7 @@ void Multigrid::Cycle(int level) const
   // Post-smooth
   for (int i = 0; i < postSmoothingSteps; i++)
   {
-      SmoothingStep(level, true);
+      SmoothingStep(level);
   }
 }

@@ -108,7 +108,7 @@ public:

 private:
   /// Application of a smoothing step at particular level
-   void SmoothingStep(int level, bool transpose) const;
+   void SmoothingStep(int level) const;

   /// Application of a cycle at particular level
   void Cycle(int level) const;
@@ -933,17 +933,6 @@ Operator &BlockNonlinearForm::GetGradientBlocked(const BlockVector &bx) const
      }
   }

-   if (!Grads(0,0)->Finalized())
-   {
-      for (int i=0; i<fes.Size(); ++i)
-      {
-         for (int j=0; j<fes.Size(); ++j)
-         {
-            Grads(i,j)->Finalize(skip_zeros);
-         }
-      }
-   }
-
   for (int s=0; s<fes.Size(); ++s)
   {
      for (int i = 0; i < ess_vdofs[s]->Size(); ++i)
@@ -963,6 +952,17 @@ Operator &BlockNonlinearForm::GetGradientBlocked(const BlockVector &bx) const
      }
   }

+   if (!Grads(0,0)->Finalized())
+   {
+      for (int i=0; i<fes.Size(); ++i)
+      {
+         for (int j=0; j<fes.Size(); ++j)
+         {
+            Grads(i,j)->Finalize(skip_zeros);
+         }
+      }
+   }
+
   for (int i=0; i<fes.Size(); ++i)
   {
      for (int j=0; j<fes.Size(); ++j)
@@ -198,9 +198,8 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
   for (int i = 0; i < nfaces; i++)
   {
      T = pmesh->GetSharedFaceTransformations(i);
-      int Elem2NbrNo = T->Elem2No - pmesh->GetNE();
      pfes->GetElementVDofs(T->Elem1No, vdofs1);
-      pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
+      pfes->GetFaceNbrElementVDofs(T->Elem2No, vdofs2);
      vdofs1.Copy(vdofs_all);
      for (int j = 0; j < vdofs2.Size(); j++)
      {
@@ -217,7 +216,7 @@ void ParBilinearForm::AssembleSharedFaces(int skip_zeros)
      for (int k = 0; k < fbfi.Size(); k++)
      {
         fbfi[k]->AssembleFaceMatrix(*pfes->GetFE(T->Elem1No),
-                                     *pfes->GetFaceNbrFE(Elem2NbrNo),
+                                     *pfes->GetFaceNbrFE(T->Elem2No),
                                     *T, elemmat);
         if (keep_nbr_block)
         {
@@ -241,7 +240,7 @@ void ParBilinearForm::Assemble(int skip_zeros)

   BilinearForm::Assemble(skip_zeros);

-   if (!ext && fbfi.Size() > 0)
+   if (fbfi.Size() > 0)
   {
      AssembleSharedFaces(skip_zeros);
   }
@@ -347,8 +347,6 @@ public:
   const FiniteElement *GetFaceNbrFE(int i) const;
   const FiniteElement *GetFaceNbrFaceFE(int i) const;
   const HYPRE_Int *GetFaceNbrGlobalDofMap() { return face_nbr_glob_dof_map; }
-   ElementTransformation *GetFaceNbrElementTransformation(int i) const
-   { return pmesh->GetFaceNbrElementTransformation(i); }

   void Lose_Dof_TrueDof_Matrix();
   void LoseDofOffsets() { dof_offsets.LoseData(); }
@@ -271,7 +271,6 @@ const
   {
      int fes_vdim = pfes->GetVDim();
      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
-      const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
      if (fes_vdim > 1)
      {
         int s = dofs.Size()/fes_vdim;
@@ -284,17 +283,7 @@ const
         face_nbr_data.GetSubVector(dofs, LocVec);
         DofVal.SetSize(dofs.Size());
      }
-      if (fe->GetMapType() == FiniteElement::VALUE)
-      {
-         fe->CalcShape(ip, DofVal);
-      }
-      else
-      {
-         ElementTransformation *Tr =
-            pfes->GetFaceNbrElementTransformation(nbr_el_no);
-         Tr->SetIntPoint(&ip);
-         fe->CalcPhysShape(*Tr, DofVal);
-      }
+      pfes->GetFaceNbrFE(nbr_el_no)->CalcShape(ip, DofVal);
   }
   else
   {
@@ -302,175 +291,14 @@ const
      fes->DofsToVDofs(vdim-1, dofs);
      DofVal.SetSize(dofs.Size());
      const FiniteElement *fe = fes->GetFE(i);
-      if (fe->GetMapType() == FiniteElement::VALUE)
-      {
-         fe->CalcShape(ip, DofVal);
-      }
-      else
-      {
-         ElementTransformation *Tr = fes->GetElementTransformation(i);
-         Tr->SetIntPoint(&ip);
-         fe->CalcPhysShape(*Tr, DofVal);
-      }
+      MFEM_ASSERT(fe->GetMapType() == FiniteElement::VALUE, "invalid FE map type");
+      fe->CalcShape(ip, DofVal);
      GetSubVector(dofs, LocVec);
   }

   return (DofVal * LocVec);
 }

-void ParGridFunction::GetVectorValue(int i, const IntegrationPoint &ip,
-                                     Vector &val) const
-{
-   int nbr_el_no = i - pfes->GetParMesh()->GetNE();
-   if (nbr_el_no >= 0)
-   {
-      Array<int> dofs;
-      pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
-      Vector loc_data;
-      face_nbr_data.GetSubVector(dofs, loc_data);
-      const FiniteElement *FElem = pfes->GetFaceNbrFE(nbr_el_no);
-      int dof = FElem->GetDof();
-      if (FElem->GetRangeType() == FiniteElement::SCALAR)
-      {
-         Vector shape(dof);
-         if (FElem->GetMapType() == FiniteElement::VALUE)
-         {
-            FElem->CalcShape(ip, shape);
-         }
-         else
-         {
-            ElementTransformation *Tr =
-               pfes->GetParMesh()->GetFaceNbrElementTransformation(nbr_el_no);
-            Tr->SetIntPoint(&ip);
-            FElem->CalcPhysShape(*Tr, shape);
-         }
-         int vdim = fes->GetVDim();
-         val.SetSize(vdim);
-         for (int k = 0; k < vdim; k++)
-         {
-            val(k) = shape * ((const double *)loc_data + dof * k);
-         }
-      }
-      else
-      {
-         int spaceDim = fes->GetMesh()->SpaceDimension();
-         DenseMatrix vshape(dof, spaceDim);
-         ElementTransformation *Tr =
-            pfes->GetParMesh()->GetFaceNbrElementTransformation(nbr_el_no);
-         Tr->SetIntPoint(&ip);
-         FElem->CalcVShape(*Tr, vshape);
-         val.SetSize(spaceDim);
-         vshape.MultTranspose(loc_data, val);
-      }
-   }
-   else
-   {
-      GridFunction::GetVectorValue(i, ip, val);
-   }
-}
-
-double ParGridFunction::GetValue(ElementTransformation &T,
-                                 const IntegrationPoint &ip,
-                                 int comp, Vector *tr) const
-{
-   // We can assume faces and edges are local
-   if (T.ElementType != ElementTransformation::ELEMENT)
-   {
-      return GridFunction::GetValue(T, ip, comp, tr);
-   }
-
-   // Check for evaluation in a local element
-   int nbr_el_no = T.ElementNo - pfes->GetParMesh()->GetNE();
-   if (nbr_el_no < 0)
-   {
-      return GridFunction::GetValue(T, ip, comp, tr);
-   }
-
-   // Evaluate using DoFs from a neighboring element
-   if (tr)
-   {
-      T.SetIntPoint(&ip);
-      T.Transform(ip, *tr);
-   }
-
-   Array<int> dofs;
-   const FiniteElement * fe = pfes->GetFaceNbrFE(nbr_el_no);
-   pfes->GetFaceNbrElementVDofs(nbr_el_no, dofs);
-
-   pfes->DofsToVDofs(comp-1, dofs);
-   Vector DofVal(dofs.Size()), LocVec;
-   if (fe->GetMapType() == FiniteElement::VALUE)
-   {
-      fe->CalcShape(ip, DofVal);
-   }
-   else
-   {
-      fe->CalcPhysShape(T, DofVal);
-   }
-   face_nbr_data.GetSubVector(dofs, LocVec);
-
-   return (DofVal * LocVec);
-}
-
-void ParGridFunction::GetVectorValue(ElementTransformation &T,
-                                     const IntegrationPoint &ip,
-                                     Vector &val, Vector *tr) const
-{
-   // We can assume faces and edges are local
-   if (T.ElementType != ElementTransformation::ELEMENT)
-   {
-      return GridFunction::GetVectorValue(T, ip, val, tr);
-   }
-
-   // Check for evaluation in a local element
-   int nbr_el_no = T.ElementNo - pfes->GetParMesh()->GetNE();
-   if (nbr_el_no < 0)
-   {
-      return GridFunction::GetVectorValue(T, ip, val, tr);
-   }
-
-   // Evaluate using DoFs from a neighboring element
-   if (tr)
-   {
-      T.SetIntPoint(&ip);
-      T.Transform(ip, *tr);
-   }
-
-   Array<int> vdofs;
-   pfes->GetFaceNbrElementVDofs(nbr_el_no, vdofs);
-   const FiniteElement *fe = pfes->GetFaceNbrFE(nbr_el_no);
-
-   int dof = fe->GetDof();
-   Vector loc_data;
-   face_nbr_data.GetSubVector(vdofs, loc_data);
-   if (fe->GetRangeType() == FiniteElement::SCALAR)
-   {
-      Vector shape(dof);
-      if (fe->GetMapType() == FiniteElement::VALUE)
-      {
-         fe->CalcShape(ip, shape);
-      }
-      else
-      {
-         fe->CalcPhysShape(T, shape);
-      }
-      int vdim = pfes->GetVDim();
-      val.SetSize(vdim);
-      for (int k = 0; k < vdim; k++)
-      {
-         val(k) = shape * ((const double *)loc_data + dof * k);
-      }
-   }
-   else
-   {
-      int spaceDim = pfes->GetMesh()->SpaceDimension();
-      DenseMatrix vshape(dof, spaceDim);
-      fe->CalcVShape(T, vshape);
-      val.SetSize(spaceDim);
-      vshape.MultTranspose(loc_data, val);
-   }
-}
-
 void ParGridFunction::ProjectCoefficient(Coefficient &coeff)
 {
   DeltaCoefficient *delta_c = dynamic_cast<DeltaCoefficient *>(&coeff);
@@ -711,9 +539,7 @@ void ParGridFunction::SaveAsOne(std::ostream &out)
   int *nfdofs = new int[NRanks];
   int *nrdofs = new int[NRanks];

-   double * h_data = const_cast<double *>(this->HostRead());
-
-   values[0] = h_data;
+   values[0] = data;
   nv[0]     = pfes -> GetVSize();
   nvdofs[0] = pfes -> GetNVDofs();
   nedofs[0] = pfes -> GetNEDofs();
@@ -814,7 +640,7 @@ void ParGridFunction::SaveAsOne(std::ostream &out)
      MPI_Send(&nvdofs[0], 1, MPI_INT, 0, 456, MyComm);
      MPI_Send(&nedofs[0], 1, MPI_INT, 0, 457, MyComm);
      MPI_Send(&nfdofs[0], 1, MPI_INT, 0, 458, MyComm);
-      MPI_Send(h_data, nv[0], MPI_DOUBLE, 0, 460, MyComm);
+      MPI_Send(data, nv[0], MPI_DOUBLE, 0, 460, MyComm);
   }

   delete [] values;
@@ -39,7 +39,7 @@ protected:
   Vector face_nbr_data;

   /** @brief Vector used as an MPI buffer to send face-neighbor data
-       in ExchangeFaceNbrData() to neighboring processors. */
+        in ExchangeFaceNbrData() to neighboring processors. */
   //TODO: Use temporary memory to avoid CUDA malloc allocation cost.
   Vector send_data;

@@ -209,18 +209,6 @@ public:
   double GetValue(ElementTransformation &T)
   { return GetValue(T.ElementNo, T.GetIntPoint()); }

-   // Redefine to handle the case when T describes a face-neighbor element
-   virtual double GetValue(ElementTransformation &T, const IntegrationPoint &ip,
-                           int comp = 0, Vector *tr = NULL) const;
-
-   virtual void GetVectorValue(int i, const IntegrationPoint &ip,
-                               Vector &val) const;
-
-   // Redefine to handle the case when T describes a face-neighbor element
-   virtual void GetVectorValue(ElementTransformation &T,
-                               const IntegrationPoint &ip,
-                               Vector &val, Vector *tr = NULL) const;
-
   using GridFunction::ProjectCoefficient;
   virtual void ProjectCoefficient(Coefficient &coeff);

@@ -21,6 +21,7 @@ namespace mfem
 void ParLinearForm::Update(ParFiniteElementSpace *pf)
 {
   if (pf) { pfes = pf; }
+
   LinearForm::Update(pfes);
 }

@@ -30,18 +31,6 @@ void ParLinearForm::Update(ParFiniteElementSpace *pf, Vector &v, int v_offset)
   LinearForm::Update(pf,v,v_offset);
 }

-void ParLinearForm::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
-{
-   LinearForm::MakeRef(f, v, v_offset);
-   pfes = dynamic_cast<ParFiniteElementSpace*>(f);
-}
-
-void ParLinearForm::MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset)
-{
-   LinearForm::MakeRef(pf, v, v_offset);
-   pfes = pf;
-}
-
 void ParLinearForm::ParallelAssemble(Vector &tv)
 {
   const Operator* prolong = pfes->GetProlongationMatrix();
@@ -92,25 +92,6 @@ public:
       @note This method does not perform assembly. */
   void Update(ParFiniteElementSpace *pf, Vector &v, int v_offset);

-
-   /** @brief Make the ParLinearForm reference external data on a new
-       FiniteElementSpace. */
-   /** This method changes the FiniteElementSpace associated with the ParLinearForm
-       to @a *f and sets the data of the Vector @a v (plus the @a v_offset) as external
-       data in the ParLinearForm.
-       @note This version of the method will also perform bounds checks when
-       the build option MFEM_DEBUG is enabled. */
-   virtual void MakeRef(FiniteElementSpace *f, Vector &v, int v_offset);
-
-   /** @brief Make the ParLinearForm reference external data on a new
-       ParFiniteElementSpace. */
-   /** This method changes the ParFiniteElementSpace associated with the ParLinearForm
-       to @a *pf and sets the data of the Vector @a v (plus the @a v_offset) as external
-       data in the ParLinearForm.
-       @note This version of the method will also perform bounds checks when
-       the build option MFEM_DEBUG is enabled. */
-   void MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset);
-
   /// Assemble the vector on the true dofs, i.e. P^t v.
   void ParallelAssemble(Vector &tv);

@@ -64,13 +64,12 @@ void ParNonlinearForm::Mult(const Vector &x, Vector &y) const
      for (int i = 0; i < n_shared_faces; i++)
      {
         tr = pmesh->GetSharedFaceTransformations(i, true);
-         int Elem2NbrNo = tr->Elem2No - pmesh->GetNE();

         fe1 = pfes->GetFE(tr->Elem1No);
-         fe2 = pfes->GetFaceNbrFE(Elem2NbrNo);
+         fe2 = pfes->GetFaceNbrFE(tr->Elem2No);

         pfes->GetElementVDofs(tr->Elem1No, vdofs1);
-         pfes->GetFaceNbrElementVDofs(Elem2NbrNo, vdofs2);
+         pfes->GetFaceNbrElementVDofs(tr->Elem2No, vdofs2);

         el_x.SetSize(vdofs1.Size() + vdofs2.Size());
         X.GetSubVector(vdofs1, el_x.GetData());
@@ -27,24 +27,35 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
                                           ElementDofOrdering e_ordering,
                                           FaceType type,
                                           L2FaceValues m)
-   : L2FaceRestriction(fes, type, m)
+   : fes(fes),
+     nf(fes.GetNFbyType(type)),
+     vdim(fes.GetVDim()),
+     byvdim(fes.GetOrdering() == Ordering::byVDIM),
+     ndofs(fes.GetNDofs()),
+     dof(nf>0 ?
+         fes.GetTraceElement(0, fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof()
+         : 0),
+     m(m),
+     nfdofs(nf*dof),
+     scatter_indices1(nf*dof),
+     scatter_indices2(m==L2FaceValues::DoubleValued?nf*dof:0),
+     offsets(ndofs+1),
+     gather_indices((m==L2FaceValues::DoubleValued? 2 : 1)*nf*dof)
 {
   if (nf==0) { return; }
   // If fespace == L2
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);
-   const FiniteElement *fe = pfes.GetFE(0);
+   const FiniteElement *fe = fes.GetFE(0);
   const TensorBasisElement *tfe = dynamic_cast<const TensorBasisElement*>(fe);
   MFEM_VERIFY(tfe != NULL &&
               (tfe->GetBasisType()==BasisType::GaussLobatto ||
                tfe->GetBasisType()==BasisType::Positive),
               "Only Gauss-Lobatto and Bernstein basis are supported in "
               "ParL2FaceRestriction.");
-   MFEM_VERIFY(pfes.GetMesh()->Conforming(),
+   MFEM_VERIFY(fes.GetMesh()->Conforming(),
               "Non-conforming meshes not yet supported with partial assembly.");
   // Assuming all finite elements are using Gauss-Lobatto dofs
   height = (m==L2FaceValues::DoubleValued? 2 : 1)*vdim*nf*dof;
-   width = pfes.GetVSize();
+   width = fes.GetVSize();
   const bool dof_reorder = (e_ordering == ElementDofOrdering::LEXICOGRAPHIC);
   if (!dof_reorder)
   {
@@ -52,32 +63,32 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
   }
   if (dof_reorder && nf > 0)
   {
-      for (int f = 0; f < pfes.GetNF(); ++f)
+      for (int f = 0; f < fes.GetNF(); ++f)
      {
         const FiniteElement *fe =
-            pfes.GetTraceElement(f, pfes.GetMesh()->GetFaceBaseGeometry(f));
+            fes.GetTraceElement(f, fes.GetMesh()->GetFaceBaseGeometry(f));
         const TensorBasisElement* el =
            dynamic_cast<const TensorBasisElement*>(fe);
         if (el) { continue; }
         mfem_error("Finite element not suitable for lexicographic ordering");
      }
   }
-   const Table& e2dTable = pfes.GetElementToDofTable();
+   const Table& e2dTable = fes.GetElementToDofTable();
   const int* elementMap = e2dTable.GetJ();
   Array<int> faceMap1(dof), faceMap2(dof);
   int e1, e2;
   int inf1, inf2;
   int face_id1, face_id2;
   int orientation;
-   const int dof1d = pfes.GetFE(0)->GetOrder()+1;
-   const int elem_dofs = pfes.GetFE(0)->GetDof();
-   const int dim = pfes.GetMesh()->SpaceDimension();
+   const int dof1d = fes.GetFE(0)->GetOrder()+1;
+   const int elem_dofs = fes.GetFE(0)->GetDof();
+   const int dim = fes.GetMesh()->SpaceDimension();
   // Computation of scatter indices
   int f_ind=0;
-   for (int f = 0; f < pfes.GetNF(); ++f)
+   for (int f = 0; f < fes.GetNF(); ++f)
   {
-      pfes.GetMesh()->GetFaceElements(f, &e1, &e2);
-      pfes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+      fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+      fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
      if (dof_reorder)
      {
         orientation = inf1 % 64;
@@ -125,7 +136,7 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
            {
               const int se2 = -1 - e2;
               Array<int> sharedDofs;
-               pfes.GetFaceNbrElementVDofs(se2, sharedDofs);
+               fes.GetFaceNbrElementVDofs(se2, sharedDofs);
               for (int d = 0; d < dof; ++d)
               {
                  const int pd = PermuteFaceL2(dim, face_id1, face_id2,
@@ -169,10 +180,10 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
      offsets[i] = 0;
   }
   f_ind = 0;
-   for (int f = 0; f < pfes.GetNF(); ++f)
+   for (int f = 0; f < fes.GetNF(); ++f)
   {
-      pfes.GetMesh()->GetFaceElements(f, &e1, &e2);
-      pfes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+      fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+      fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
      if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
          (type==FaceType::Boundary && e2<0 && inf2<0) )
      {
@@ -211,10 +222,10 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,
      offsets[i] += offsets[i - 1];
   }
   f_ind = 0;
-   for (int f = 0; f < pfes.GetNF(); ++f)
+   for (int f = 0; f < fes.GetNF(); ++f)
   {
-      pfes.GetMesh()->GetFaceElements(f, &e1, &e2);
-      pfes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+      fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+      fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
      if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
          (type==FaceType::Boundary && e2<0 && inf2<0) )
      {
@@ -261,10 +272,8 @@ ParL2FaceRestriction::ParL2FaceRestriction(const ParFiniteElementSpace &fes,

 void ParL2FaceRestriction::Mult(const Vector& x, Vector& y) const
 {
-   const ParFiniteElementSpace &pfes =
-      static_cast<const ParFiniteElementSpace&>(this->fes);
   ParGridFunction x_gf;
-   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&pfes),
+   x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(&fes),
                const_cast<Vector&>(x), 0);
   x_gf.ExchangeFaceNbrData();

@@ -328,122 +337,34 @@ void ParL2FaceRestriction::Mult(const Vector& x, Vector& y) const
   }
 }

-static MFEM_HOST_DEVICE int AddNnz(const int iE, int *I, const int dofs)
+void ParL2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
 {
-   int val = AtomicAdd(I[iE],dofs);
-   return val;
-}
-
-void ParL2FaceRestriction::FillI(SparseMatrix &mat,
-                                 SparseMatrix &face_mat) const
-{
-   const int face_dofs = dof;
-   const int Ndofs = ndofs;
-   auto d_indices1 = scatter_indices1.Read();
-   auto d_indices2 = scatter_indices2.Read();
-   auto I = mat.ReadWriteI();
-   auto I_face = face_mat.ReadWriteI();
-   MFEM_FORALL(i, ne*elemDofs*vdim+1,
+   // Assumes all elements have the same number of dofs
+   const int nd = dof;
+   const int vd = vdim;
+   const bool t = byvdim;
+   const int dofs = nfdofs;
+   auto d_offsets = offsets.Read();
+   auto d_indices = gather_indices.Read();
+   auto d_x = Reshape(x.Read(), nd, vd, 2, nf);
+   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   MFEM_FORALL(i, ndofs,
   {
-      I_face[i] = 0;
-   });
-   MFEM_FORALL(fdof, nf*face_dofs,
-   {
-      const int f  = fdof/face_dofs;
-      const int iF = fdof%face_dofs;
-      const int iE1 = d_indices1[f*face_dofs+iF];
-      if (iE1 < Ndofs)
+      const int offset = d_offsets[i];
+      const int nextOffset = d_offsets[i + 1];
+      for (int c = 0; c < vd; ++c)
      {
-         for (int jF = 0; jF < face_dofs; jF++)
+         double dofValue = 0;
+         for (int j = offset; j < nextOffset; ++j)
         {
-            const int jE2 = d_indices2[f*face_dofs+jF];
-            if (jE2 < Ndofs)
-            {
-               AddNnz(iE1,I,1);
-            }
-            else
-            {
-               AddNnz(iE1,I_face,1);
-            }
-         }
-      }
-      const int iE2 = d_indices2[f*face_dofs+iF];
-      if (iE2 < Ndofs)
-      {
-         for (int jF = 0; jF < face_dofs; jF++)
-         {
-            const int jE1 = d_indices1[f*face_dofs+jF];
-            if (jE1 < Ndofs)
-            {
-               AddNnz(iE2,I,1);
-            }
-            else
-            {
-               AddNnz(iE2,I_face,1);
-            }
-         }
-      }
-   });
-}
-
-void ParL2FaceRestriction::FillJAndData(const Vector &ea_data,
-                                        SparseMatrix &mat,
-                                        SparseMatrix &face_mat) const
-{
-   const int face_dofs = dof;
-   const int Ndofs = ndofs;
-   auto d_indices1 = scatter_indices1.Read();
-   auto d_indices2 = scatter_indices2.Read();
-   auto mat_fea = Reshape(ea_data.Read(), face_dofs, face_dofs, 2, nf);
-   auto I = mat.ReadWriteI();
-   auto I_face = face_mat.ReadWriteI();
-   auto J = mat.WriteJ();
-   auto J_face = face_mat.WriteJ();
-   auto Data = mat.WriteData();
-   auto Data_face = face_mat.WriteData();
-   MFEM_FORALL(fdof, nf*face_dofs,
-   {
-      const int f  = fdof/face_dofs;
-      const int iF = fdof%face_dofs;
-      const int iE1 = d_indices1[f*face_dofs+iF];
-      if (iE1 < Ndofs)
-      {
-         for (int jF = 0; jF < face_dofs; jF++)
-         {
-            const int jE2 = d_indices2[f*face_dofs+jF];
-            if (jE2 < Ndofs)
-            {
-               const int offset = AddNnz(iE1,I,1);
-               J[offset] = jE2;
-               Data[offset] = mat_fea(jF,iF,1,f);
-            }
-            else
-            {
-               const int offset = AddNnz(iE1,I_face,1);
-               J_face[offset] = jE2-Ndofs;
-               Data_face[offset] = mat_fea(jF,iF,1,f);
-            }
-         }
-      }
-      const int iE2 = d_indices2[f*face_dofs+iF];
-      if (iE2 < Ndofs)
-      {
-         for (int jF = 0; jF < face_dofs; jF++)
-         {
-            const int jE1 = d_indices1[f*face_dofs+jF];
-            if (jE1 < Ndofs)
-            {
-               const int offset = AddNnz(iE2,I,1);
-               J[offset] = jE1;
-               Data[offset] = mat_fea(jF,iF,0,f);
-            }
-            else
-            {
-               const int offset = AddNnz(iE2,I_face,1);
-               J_face[offset] = jE1-Ndofs;
-               Data_face[offset] = mat_fea(jF,iF,0,f);
-            }
+            int idx_j = d_indices[j];
+            bool isE1 = idx_j < dofs;
+            idx_j = isE1 ? idx_j : idx_j - dofs;
+            dofValue +=  isE1 ?
+            d_x(idx_j % nd, c, 0, idx_j / nd)
+            :d_x(idx_j % nd, c, 1, idx_j / nd);
         }
+         d_y(t?c:i,t?i:c) += dofValue;
      }
   });
 }
@@ -26,21 +26,28 @@ class ParFiniteElementSpace;
 /// Operator that extracts Face degrees of freedom in parallel.
 /** Objects of this type are typically created and owned by FiniteElementSpace
    objects, see FiniteElementSpace::GetFaceRestriction(). */
-class ParL2FaceRestriction : public L2FaceRestriction
+class ParL2FaceRestriction : public Operator
 {
+protected:
+   const ParFiniteElementSpace &fes;
+   const int nf;
+   const int vdim;
+   const bool byvdim;
+   const int ndofs;
+   const int dof;
+   const L2FaceValues m;
+   const int nfdofs;
+   Array<int> scatter_indices1;
+   Array<int> scatter_indices2;
+   Array<int> offsets;
+   Array<int> gather_indices;
+
 public:
   ParL2FaceRestriction(const ParFiniteElementSpace&, ElementDofOrdering,
                        FaceType type,
                        L2FaceValues m = L2FaceValues::DoubleValued);
   void Mult(const Vector &x, Vector &y) const;
-   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this ParL2FaceRestriction. */
-   void FillI(SparseMatrix &mat, SparseMatrix &face_mat) const;
-   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this ParL2FaceRestriction, and the values of ea_data. */
-   void FillJAndData(const Vector &ea_data,
-                     SparseMatrix &mat,
-                     SparseMatrix &face_mat) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
 };

 }
@@ -150,6 +150,7 @@ void QuadratureInterpolator::Eval3D(
   const int nq = maps.nqpt;
   const int ND = T_ND ? T_ND : nd;
   const int NQ = T_NQ ? T_NQ : nq;
+   const int NMAX = NQ > ND ? NQ : ND;
   const int VDIM = T_VDIM ? T_VDIM : vdim;
   MFEM_VERIFY(ND <= MAX_ND3D, "");
   MFEM_VERIFY(NQ <= MAX_NQ3D, "");
@@ -160,22 +161,24 @@ void QuadratureInterpolator::Eval3D(
   auto val = Reshape(q_val.Write(), NQ, VDIM, NE);
   auto der = Reshape(q_der.Write(), NQ, VDIM, 3, NE);
   auto det = Reshape(q_det.Write(), NQ, NE);
-   MFEM_FORALL(e, NE,
+   MFEM_FORALL_2D(e, NE, NMAX, 1, 1,
   {
      const int ND = T_ND ? T_ND : nd;
      const int NQ = T_NQ ? T_NQ : nq;
      const int VDIM = T_VDIM ? T_VDIM : vdim;
      constexpr int max_ND = T_ND ? T_ND : MAX_ND3D;
      constexpr int max_VDIM = T_VDIM ? T_VDIM : MAX_VDIM3D;
-      double s_E[max_VDIM*max_ND];
-      for (int d = 0; d < ND; d++)
+      MFEM_SHARED double s_E[max_VDIM*max_ND];
+      MFEM_FOREACH_THREAD(d, x, ND)
      {
         for (int c = 0; c < VDIM; c++)
         {
            s_E[c+d*VDIM] = E(d,c,e);
         }
      }
-      for (int q = 0; q < NQ; ++q)
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(q, x, NQ)
      {
         if (eval_flags & VALUES)
         {
@@ -495,8 +495,9 @@ void FaceQuadratureInterpolator::Mult(
   }
 }

-void FaceQuadratureInterpolator::Values(
-   const Vector &e_vec, Vector &q_val) const
+
+void FaceQuadratureInterpolator::Values(const Vector &e_vec,
+                                        Vector &q_val) const
 {
   Vector q_der, q_det, q_nor;
   Mult(e_vec, VALUES, q_val, q_der, q_det, q_nor);
@@ -17,6 +17,55 @@
 namespace mfem
 {

+L2ElementRestriction::L2ElementRestriction(const FiniteElementSpace &fes)
+   : ne(fes.GetNE()),
+     vdim(fes.GetVDim()),
+     byvdim(fes.GetOrdering() == Ordering::byVDIM),
+     ndof(ne > 0 ? fes.GetFE(0)->GetDof() : 0),
+     ndofs(fes.GetNDofs())
+{
+   height = vdim*ne*ndof;
+   width = vdim*ne*ndof;
+}
+
+void L2ElementRestriction::Mult(const Vector &x, Vector &y) const
+{
+   const int nd = ndof;
+   const int vd = vdim;
+   const bool t = byvdim;
+   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
+   auto d_y = Reshape(y.Write(), nd, vd, ne);
+   MFEM_FORALL(i, ndofs,
+   {
+      const int idx = i;
+      const int dof = idx % nd;
+      const int e = idx / nd;
+      for (int c = 0; c < vd; ++c)
+      {
+         d_y(dof, c, e) = d_x(t?c:idx, t?idx:c);
+      }
+   });
+}
+
+void L2ElementRestriction::MultTranspose(const Vector &x, Vector &y) const
+{
+   const int nd = ndof;
+   const int vd = vdim;
+   const bool t = byvdim;
+   auto d_x = Reshape(x.Read(), nd, vd, ne);
+   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
+   MFEM_FORALL(i, ndofs,
+   {
+      const int idx = i;
+      const int dof = idx % nd;
+      const int e = idx / nd;
+      for (int c = 0; c < vd; ++c)
+      {
+         d_y(t?c:idx,t?idx:c) = d_x(dof, c, e);
+      }
+   });
+}
+
 ElementRestriction::ElementRestriction(const FiniteElementSpace &f,
                                       ElementDofOrdering e_ordering)
   : fes(f),
@@ -232,309 +281,7 @@ void ElementRestriction::BooleanMask(Vector& y) const
   }
 }

-void ElementRestriction::FillSparseMatrix(const Vector &mat_ea,
-                                          SparseMatrix &mat) const
-{
-   mat.GetMemoryI().New(mat.Height()+1, mat.GetMemoryI().GetMemoryType());
-   const int nnz = FillI(mat);
-   mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
-   mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
-   FillJAndData(mat_ea, mat);
-}
-
-template <int MaxNbNbr>
-static MFEM_HOST_DEVICE int GetMinElt(const int *my_elts, const int nbElts,
-                                      const int *nbr_elts, const int nbrNbElts)
-{
-   // Building the intersection
-   int inter[MaxNbNbr];
-   int cpt = 0;
-   for (int i = 0; i < nbElts; i++)
-   {
-      const int e_i = my_elts[i];
-      for (int j = 0; j < nbrNbElts; j++)
-      {
-         if (e_i==nbr_elts[j])
-         {
-            inter[cpt] = e_i;
-            cpt++;
-         }
-      }
-   }
-   // Finding the minimum
-   int min = inter[0];
-   for (int i = 1; i < cpt; i++)
-   {
-      if (inter[i] < min)
-      {
-         min = inter[i];
-      }
-   }
-   return min;
-}
-
-/** Returns the index where a non-zero entry should be added and increment the
-    number of non-zeros for the row i_L. */
-static MFEM_HOST_DEVICE int GetAndIncrementNnzIndex(const int i_L, int* I)
-{
-   int ind = AtomicAdd(I[i_L],1);
-   return ind;
-}
-
-int ElementRestriction::FillI(SparseMatrix &mat) const
-{
-   static constexpr int Max = MaxNbNbr;
-   const int all_dofs = ndofs;
-   const int vd = vdim;
-   const int elt_dofs = dof;
-   auto I = mat.ReadWriteI();
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_gatherMap = gatherMap.Read();
-   MFEM_FORALL(i_L, vd*all_dofs+1,
-   {
-      I[i_L] = 0;
-   });
-   MFEM_FORALL(e, ne,
-   {
-      for (int i = 0; i < elt_dofs; i++)
-      {
-         int i_elts[Max];
-         const int i_E = e*elt_dofs + i;
-         const int i_L = d_gatherMap[i_E];
-         const int i_offset = d_offsets[i_L];
-         const int i_nextOffset = d_offsets[i_L+1];
-         const int i_nbElts = i_nextOffset - i_offset;
-         for (int e_i = 0; e_i < i_nbElts; ++e_i)
-         {
-            const int i_E = d_indices[i_offset+e_i];
-            i_elts[e_i] = i_E/elt_dofs;
-         }
-         for (int j = 0; j < elt_dofs; j++)
-         {
-            const int j_E = e*elt_dofs + j;
-            const int j_L = d_gatherMap[j_E];
-            const int j_offset = d_offsets[j_L];
-            const int j_nextOffset = d_offsets[j_L+1];
-            const int j_nbElts = j_nextOffset - j_offset;
-            if (i_nbElts == 1 || j_nbElts == 1) // no assembly required
-            {
-               GetAndIncrementNnzIndex(i_L, I);
-            }
-            else // assembly required
-            {
-               int j_elts[Max];
-               for (int e_j = 0; e_j < j_nbElts; ++e_j)
-               {
-                  const int j_E = d_indices[j_offset+e_j];
-                  const int elt = j_E/elt_dofs;
-                  j_elts[e_j] = elt;
-               }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
-               if (e == min_e) // add the nnz only once
-               {
-                  GetAndIncrementNnzIndex(i_L, I);
-               }
-            }
-         }
-      }
-   });
-   // We need to sum the entries of I, we do it on CPU as it is very sequential.
-   auto h_I = mat.HostReadWriteI();
-   const int nTdofs = vd*all_dofs;
-   int sum = 0;
-   for (int i = 0; i < nTdofs; i++)
-   {
-      const int nnz = h_I[i];
-      h_I[i] = sum;
-      sum+=nnz;
-   }
-   h_I[nTdofs] = sum;
-   // We return the number of nnz
-   return h_I[nTdofs];
-}
-
-void ElementRestriction::FillJAndData(const Vector &ea_data,
-                                      SparseMatrix &mat) const
-{
-   static constexpr int Max = MaxNbNbr;
-   const int all_dofs = ndofs;
-   const int vd = vdim;
-   const int elt_dofs = dof;
-   auto I = mat.ReadWriteI();
-   auto J = mat.WriteJ();
-   auto Data = mat.WriteData();
-   auto d_offsets = offsets.Read();
-   auto d_indices = indices.Read();
-   auto d_gatherMap = gatherMap.Read();
-   auto mat_ea = Reshape(ea_data.Read(), elt_dofs, elt_dofs, ne);
-   MFEM_FORALL(e, ne,
-   {
-      for (int i = 0; i < elt_dofs; i++)
-      {
-         int i_elts[Max];
-         int i_B[Max];
-         const int i_E = e*elt_dofs + i;
-         const int i_L = d_gatherMap[i_E];
-         const int i_offset = d_offsets[i_L];
-         const int i_nextOffset = d_offsets[i_L+1];
-         const int i_nbElts = i_nextOffset - i_offset;
-         for (int e_i = 0; e_i < i_nbElts; ++e_i)
-         {
-            const int i_E = d_indices[i_offset+e_i];
-            i_elts[e_i] = i_E/elt_dofs;
-            i_B[e_i]    = i_E%elt_dofs;
-         }
-         for (int j = 0; j < elt_dofs; j++)
-         {
-            const int j_E = e*elt_dofs + j;
-            const int j_L = d_gatherMap[j_E];
-            const int j_offset = d_offsets[j_L];
-            const int j_nextOffset = d_offsets[j_L+1];
-            const int j_nbElts = j_nextOffset - j_offset;
-            if (i_nbElts == 1 || j_nbElts == 1) // no assembly required
-            {
-               const int nnz = GetAndIncrementNnzIndex(i_L, I);
-               J[nnz] = j_L;
-               Data[nnz] = mat_ea(j,i,e);
-            }
-            else // assembly required
-            {
-               int j_elts[Max];
-               int j_B[Max];
-               for (int e_j = 0; e_j < j_nbElts; ++e_j)
-               {
-                  const int j_E = d_indices[j_offset+e_j];
-                  const int elt = j_E/elt_dofs;
-                  j_elts[e_j] = elt;
-                  j_B[e_j]    = j_E%elt_dofs;
-               }
-               int min_e = GetMinElt<Max>(i_elts, i_nbElts, j_elts, j_nbElts);
-               if (e == min_e) // add the nnz only once
-               {
-                  double val = 0.0;
-                  for (int i = 0; i < i_nbElts; i++)
-                  {
-                     const int e_i = i_elts[i];
-                     const int i_Bloc = i_B[i];
-                     for (int j = 0; j < j_nbElts; j++)
-                     {
-                        const int e_j = j_elts[j];
-                        const int j_Bloc = j_B[j];
-                        if (e_i == e_j)
-                        {
-                           val += mat_ea(j_Bloc, i_Bloc, e_i);
-                        }
-                     }
-                  }
-                  const int nnz = GetAndIncrementNnzIndex(i_L, I);
-                  J[nnz] = j_L;
-                  Data[nnz] = val;
-               }
-            }
-         }
-      }
-   });
-   // We need to shift again the entries of I, we do it on CPU as it is very
-   // sequential.
-   auto h_I = mat.HostReadWriteI();
-   const int size = vd*all_dofs;
-   for (int i = 0; i < size; i++)
-   {
-      h_I[size-i] = h_I[size-(i+1)];
-   }
-   h_I[0] = 0;
-}
-
-L2ElementRestriction::L2ElementRestriction(const FiniteElementSpace &fes)
-   : ne(fes.GetNE()),
-     vdim(fes.GetVDim()),
-     byvdim(fes.GetOrdering() == Ordering::byVDIM),
-     ndof(ne > 0 ? fes.GetFE(0)->GetDof() : 0),
-     ndofs(fes.GetNDofs())
-{
-   height = vdim*ne*ndof;
-   width = vdim*ne*ndof;
-}
-
-void L2ElementRestriction::Mult(const Vector &x, Vector &y) const
-{
-   const int nd = ndof;
-   const int vd = vdim;
-   const bool t = byvdim;
-   auto d_x = Reshape(x.Read(), t?vd:ndofs, t?ndofs:vd);
-   auto d_y = Reshape(y.Write(), nd, vd, ne);
-   MFEM_FORALL(i, ndofs,
-   {
-      const int idx = i;
-      const int dof = idx % nd;
-      const int e = idx / nd;
-      for (int c = 0; c < vd; ++c)
-      {
-         d_y(dof, c, e) = d_x(t?c:idx, t?idx:c);
-      }
-   });
-}
-
-void L2ElementRestriction::MultTranspose(const Vector &x, Vector &y) const
-{
-   const int nd = ndof;
-   const int vd = vdim;
-   const bool t = byvdim;
-   auto d_x = Reshape(x.Read(), nd, vd, ne);
-   auto d_y = Reshape(y.Write(), t?vd:ndofs, t?ndofs:vd);
-   MFEM_FORALL(i, ndofs,
-   {
-      const int idx = i;
-      const int dof = idx % nd;
-      const int e = idx / nd;
-      for (int c = 0; c < vd; ++c)
-      {
-         d_y(t?c:idx,t?idx:c) = d_x(dof, c, e);
-      }
-   });
-}
-
-void L2ElementRestriction::FillI(SparseMatrix &mat) const
-{
-   const int elem_dofs = ndof;
-   const int vd = vdim;
-   auto I = mat.WriteI();
-   MFEM_FORALL(dof, ne*elem_dofs*vd,
-   {
-      I[dof] = elem_dofs;
-   });
-}
-
-static MFEM_HOST_DEVICE int AddNnz(const int iE, int *I, const int dofs)
-{
-   int val = AtomicAdd(I[iE],dofs);
-   return val;
-}
-
-void L2ElementRestriction::FillJAndData(const Vector &ea_data,
-                                        SparseMatrix &mat) const
-{
-   const int elem_dofs = ndof;
-   const int vd = vdim;
-   auto I = mat.ReadWriteI();
-   auto J = mat.WriteJ();
-   auto Data = mat.WriteData();
-   auto mat_ea = Reshape(ea_data.Read(), elem_dofs, elem_dofs, ne);
-   MFEM_FORALL(iE, ne*elem_dofs*vd,
-   {
-      const int offset = AddNnz(iE,I,elem_dofs);
-      const int e = iE/elem_dofs;
-      const int i = iE%elem_dofs;
-      for (int j = 0; j < elem_dofs; j++)
-      {
-         J[offset+j] = e*elem_dofs+j;
-         Data[offset+j] = mat_ea(j,i,e);
-      }
-   });
-}
-
-// Return the face degrees of freedom returned in Lexicographic order.
+/// Return the face degrees of freedom returned in Lexicographic order.
 void GetFaceDofs(const int dim, const int face_id,
                 const int dof1d, Array<int> &faceMap)
 {
@@ -953,7 +700,7 @@ static int PermuteFace3D(const int face_id1, const int face_id2,
   return ToLexOrdering3D(face_id2, size1d, new_i, new_j);
 }

-// Permute dofs or quads on a face for e2 to match with the ordering of e1
+/// Permute dofs or quads on a face for e2 to match with the ordering of e1
 int PermuteFaceL2(const int dim, const int face_id1,
                  const int face_id2, const int orientation,
                  const int size1d, const int index)
@@ -973,32 +720,23 @@ int PermuteFaceL2(const int dim, const int face_id1,
 }

 L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
+                                     const ElementDofOrdering e_ordering,
                                     const FaceType type,
                                     const L2FaceValues m)
   : fes(fes),
     nf(fes.GetNFbyType(type)),
-     ne(fes.GetNE()),
     vdim(fes.GetVDim()),
     byvdim(fes.GetOrdering() == Ordering::byVDIM),
     ndofs(fes.GetNDofs()),
     dof(nf > 0 ?
         fes.GetTraceElement(0, fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof()
         : 0),
-     elemDofs(fes.GetFE(0)->GetDof()),
     m(m),
     nfdofs(nf*dof),
     scatter_indices1(nf*dof),
     scatter_indices2(m==L2FaceValues::DoubleValued?nf*dof:0),
     offsets(ndofs+1),
     gather_indices((m==L2FaceValues::DoubleValued? 2 : 1)*nf*dof)
-{
-}
-
-L2FaceRestriction::L2FaceRestriction(const FiniteElementSpace &fes,
-                                     const ElementDofOrdering e_ordering,
-                                     const FaceType type,
-                                     const L2FaceValues m)
-   : L2FaceRestriction(fes, type, m)
 {
   // If fespace == L2
   const FiniteElement *fe = fes.GetFE(0);
@@ -1296,113 +1034,6 @@ void L2FaceRestriction::MultTranspose(const Vector& x, Vector& y) const
   }
 }

-void L2FaceRestriction::FillI(SparseMatrix &mat,
-                              SparseMatrix &face_mat) const
-{
-   const int face_dofs = dof;
-   auto d_indices1 = scatter_indices1.Read();
-   auto d_indices2 = scatter_indices2.Read();
-   auto I = mat.ReadWriteI();
-   MFEM_FORALL(fdof, nf*face_dofs,
-   {
-      const int iE1 = d_indices1[fdof];
-      const int iE2 = d_indices2[fdof];
-      AddNnz(iE1,I,face_dofs);
-      AddNnz(iE2,I,face_dofs);
-   });
-}
-
-void L2FaceRestriction::FillJAndData(const Vector &ea_data,
-                                     SparseMatrix &mat,
-                                     SparseMatrix &face_mat) const
-{
-   const int face_dofs = dof;
-   auto d_indices1 = scatter_indices1.Read();
-   auto d_indices2 = scatter_indices2.Read();
-   auto I = mat.ReadWriteI();
-   auto mat_fea = Reshape(ea_data.Read(), face_dofs, face_dofs, 2, nf);
-   auto J = mat.WriteJ();
-   auto Data = mat.WriteData();
-   MFEM_FORALL(fdof, nf*face_dofs,
-   {
-      const int f  = fdof/face_dofs;
-      const int iF = fdof%face_dofs;
-      const int iE1 = d_indices1[f*face_dofs+iF];
-      const int iE2 = d_indices2[f*face_dofs+iF];
-      const int offset1 = AddNnz(iE1,I,face_dofs);
-      const int offset2 = AddNnz(iE2,I,face_dofs);
-      for (int jF = 0; jF < face_dofs; jF++)
-      {
-         const int jE1 = d_indices1[f*face_dofs+jF];
-         const int jE2 = d_indices2[f*face_dofs+jF];
-         J[offset2+jF] = jE1;
-         J[offset1+jF] = jE2;
-         Data[offset2+jF] = mat_fea(jF,iF,0,f);
-         Data[offset1+jF] = mat_fea(jF,iF,1,f);
-      }
-   });
-}
-
-void L2FaceRestriction::AddFaceMatricesToElementMatrices(Vector &fea_data,
-                                                         Vector &ea_data) const
-{
-   const int face_dofs = dof;
-   const int elem_dofs = elemDofs;
-   const int NE = ne;
-   if (m==L2FaceValues::DoubleValued)
-   {
-      auto d_indices1 = scatter_indices1.Read();
-      auto d_indices2 = scatter_indices2.Read();
-      auto mat_fea = Reshape(fea_data.Read(), face_dofs, face_dofs, 2, nf);
-      auto mat_ea = Reshape(ea_data.ReadWrite(), elem_dofs, elem_dofs, ne);
-      MFEM_FORALL(f, nf,
-      {
-         const int e1 = d_indices1[f*face_dofs]/elem_dofs;
-         const int e2 = d_indices2[f*face_dofs]/elem_dofs;
-         for (int j = 0; j < face_dofs; j++)
-         {
-            const int jB1 = d_indices1[f*face_dofs+j]%elem_dofs;
-            for (int i = 0; i < face_dofs; i++)
-            {
-               const int iB1 = d_indices1[f*face_dofs+i]%elem_dofs;
-               AtomicAdd(mat_ea(iB1,jB1,e1), mat_fea(i,j,0,f));
-            }
-         }
-         if (e2 < NE)
-         {
-            for (int j = 0; j < face_dofs; j++)
-            {
-               const int jB2 = d_indices2[f*face_dofs+j]%elem_dofs;
-               for (int i = 0; i < face_dofs; i++)
-               {
-                  const int iB2 = d_indices2[f*face_dofs+i]%elem_dofs;
-                  AtomicAdd(mat_ea(iB2,jB2,e2), mat_fea(i,j,1,f));
-               }
-            }
-         }
-      });
-   }
-   else
-   {
-      auto d_indices = scatter_indices1.Read();
-      auto mat_fea = Reshape(fea_data.Read(), face_dofs, face_dofs, nf);
-      auto mat_ea = Reshape(ea_data.ReadWrite(), elem_dofs, elem_dofs, ne);
-      MFEM_FORALL(f, nf,
-      {
-         const int e = d_indices[f*face_dofs]/elem_dofs;
-         for (int j = 0; j < face_dofs; j++)
-         {
-            const int jE = d_indices[f*face_dofs+j]%elem_dofs;
-            for (int i = 0; i < face_dofs; i++)
-            {
-               const int iE = d_indices[f*face_dofs+i]%elem_dofs;
-               AtomicAdd(mat_ea(iE,jE,e), mat_fea(i,j,f));
-            }
-         }
-      });
-   }
-}
-
 int ToLexOrdering(const int dim, const int face_id, const int size1d,
                  const int index)
 {
@@ -30,11 +30,6 @@ enum class L2FaceValues : bool {SingleValued, DoubleValued};
    objects, see FiniteElementSpace::GetElementRestriction(). */
 class ElementRestriction : public Operator
 {
-private:
-   /** This number defines the maximum number of elements any dof can belong to
-       for the FillSparseMatrix method. */
-   static const int MaxNbNbr = 16;
-
 protected:
   const FiniteElementSpace &fes;
   const int ne;
@@ -64,16 +59,6 @@ public:
       emulate SetSubVector and its transpose on GPUs. This method is running on
       the host, since the `processed` array requires a large shared memory. */
   void BooleanMask(Vector& y) const;
-
-   /// Fill a Sparse Matrix with Element Matrices.
-   void FillSparseMatrix(const Vector &mat_ea, SparseMatrix &mat) const;
-
-   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this ElementRestriction. */
-   int FillI(SparseMatrix &mat) const;
-   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this ElementRestriction, and the values of ea_data. */
-   void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
 };

 /// Operator that converts L2 FiniteElementSpace L-vectors to E-vectors.
@@ -92,12 +77,6 @@ public:
   L2ElementRestriction(const FiniteElementSpace&);
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
-   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this ElementRestriction. */
-   void FillI(SparseMatrix &mat) const;
-   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this L2FaceRestriction, and the values of ea_data. */
-   void FillJAndData(const Vector &ea_data, SparseMatrix &mat) const;
 };

 /// Operator that extracts Face degrees of freedom.
@@ -132,12 +111,10 @@ class L2FaceRestriction : public Operator
 protected:
   const FiniteElementSpace &fes;
   const int nf;
-   const int ne;
   const int vdim;
   const bool byvdim;
   const int ndofs;
   const int dof;
-   const int elemDofs;
   const L2FaceValues m;
   const int nfdofs;
   Array<int> scatter_indices1;
@@ -145,27 +122,12 @@ protected:
   Array<int> offsets;
   Array<int> gather_indices;

-   L2FaceRestriction(const FiniteElementSpace&,
-                     const FaceType,
-                     const L2FaceValues m = L2FaceValues::DoubleValued);
-
 public:
   L2FaceRestriction(const FiniteElementSpace&, const ElementDofOrdering,
                     const FaceType,
                     const L2FaceValues m = L2FaceValues::DoubleValued);
-   virtual void Mult(const Vector &x, Vector &y) const;
+   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
-   /** Fill the I array of SparseMatrix corresponding to the sparsity pattern
-       given by this L2FaceRestriction. */
-   virtual void FillI(SparseMatrix &mat, SparseMatrix &face_mat) const;
-   /** Fill the J and Data arrays of SparseMatrix corresponding to the sparsity
-       pattern given by this L2FaceRestriction, and the values of ea_data. */
-   virtual void FillJAndData(const Vector &ea_data,
-                             SparseMatrix &mat,
-                             SparseMatrix &face_mat) const;
-   /// This methods adds the DG face matrices to the element matrices.
-   void AddFaceMatricesToElementMatrices(Vector &fea_data,
-                                         Vector &ea_data) const;
 };

 // Return the face degrees of freedom returned in Lexicographic order.
@@ -177,6 +177,24 @@ double TMOP_Metric_SSA2D::EvalW(const DenseMatrix &Jpt) const
   return Mat.FNorm2();
 }

+// mu_85 = |T-T'|^2, where T'= |T|*I/sqrt(2)
+double TMOP_Metric_SS2D::EvalW(const DenseMatrix &Jpt) const
+{
+   MFEM_VERIFY(Jtr != NULL,
+               "Requires a target Jacobian, use SetTargetJacobian().");
+
+   DenseMatrix Id(2,2);
+   DenseMatrix Mat(2,2);
+   Mat = Jpt;
+
+   Id(0,0) = 1; Id(0,1) = 0;
+   Id(1,0) = 0; Id(1,1) = 1;
+   Id *= Mat.FNorm()/pow(2,0.5);
+
+   Mat.Add(-1.,Id);
+   return Mat.FNorm2();
+}
+
 double TMOP_Metric_002::EvalW(const DenseMatrix &Jpt) const
 {
   ie.SetJacobian(Jpt.GetData());
@@ -466,24 +484,6 @@ void TMOP_Metric_077::AssembleH(const DenseMatrix &Jpt,
   ie.Assemble_TProd(weight * I2inv_sq / I2, ie.Get_dI2(), A.GetData());
 }

-// mu_85 = |T-T'|^2, where T'= |T|*I/sqrt(2)
-double TMOP_Metric_085::EvalW(const DenseMatrix &Jpt) const
-{
-   MFEM_VERIFY(Jtr != NULL,
-               "Requires a target Jacobian, use SetTargetJacobian().");
-
-   DenseMatrix Id(2,2);
-   DenseMatrix Mat(2,2);
-   Mat = Jpt;
-
-   Id(0,0) = 1; Id(0,1) = 0;
-   Id(1,0) = 0; Id(1,1) = 1;
-   Id *= Mat.FNorm()/pow(2,0.5);
-
-   Mat.Add(-1.,Id);
-   return Mat.FNorm2();
-}
-
 double TMOP_Metric_211::EvalW(const DenseMatrix &Jpt) const
 {
   // mu_211 = (det(J) - 1)^2 - det(J) + (det(J)^2 + eps)^{1/2}
@@ -927,20 +927,9 @@ void TargetConstructor::ComputeElementTargets(int e_id, const FiniteElement &fe,
   }
 }

-void TargetConstructor::ComputeElementTargetsGradient(const IntegrationRule &ir,
-                                                      const Vector &elfun,
-                                                      IsoparametricTransformation &Tpr,
-                                                      DenseTensor &dJtr) const
-{
-   MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != NULL, "");
-
-   // TODO: Compute derivative for targets with GIVEN_SHAPE or/and GIVEN_SIZE
-   for (int i = 0; i < Tpr.GetFE()->GetDim()*ir.GetNPoints(); i++) { dJtr(i) = 0.; }
-}
-
 void AnalyticAdaptTC::SetAnalyticTargetSpec(Coefficient *sspec,
                                            VectorCoefficient *vspec,
-                                            TMOPMatrixCoefficient *mspec)
+                                            MatrixCoefficient *mspec)
 {
   scalar_tspec = sspec;
   vector_tspec = vspec;
@@ -981,39 +970,6 @@ void AnalyticAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
   }
 }

-void AnalyticAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
-                                                    const Vector &elfun,
-                                                    IsoparametricTransformation &Tpr,
-                                                    DenseTensor &dJtr) const
-{
-   const FiniteElement *fe = Tpr.GetFE();
-   DenseMatrix point_mat;
-   point_mat.UseExternalData(elfun.GetData(), fe->GetDof(), fe->GetDim());
-
-   switch (target_type)
-   {
-      case GIVEN_FULL:
-      {
-         MFEM_VERIFY(matrix_tspec != NULL,
-                     "Target type GIVEN_FULL requires a TMOPMatrixCoefficient.");
-
-         for (int d = 0; d < fe->GetDim(); d++)
-         {
-            for (int i = 0; i < ir.GetNPoints(); i++)
-            {
-               const IntegrationPoint &ip = ir.IntPoint(i);
-               Tpr.SetIntPoint(&ip);
-               DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-               matrix_tspec->EvalGrad(dJtr_i, Tpr, ip, d);
-            }
-         }
-         break;
-      }
-      default:
-         MFEM_ABORT("Incompatible target type for analytic adaptation!");
-   }
-}
-
 #ifdef MFEM_USE_MPI
 void DiscreteAdaptTC::FinalizeParDiscreteTargetSpec(const ParGridFunction
                                                    &tspec_)
@@ -1102,6 +1058,7 @@ void DiscreteAdaptTC::SetDiscreteTargetBase(const GridFunction &tspec_)
   // make a copy of tspec->tspec_temp, increase its size, and
   // copy data from tspec_temp -> tspec, then add new entries
   Vector tspec_temp = tspec;
+   tspec.UseDevice(true);
   tspec.SetSize(ncomp*dof_cnt);

   for (int i = 0; i < tspec_temp.Size(); i++)
@@ -1212,7 +1169,7 @@ void DiscreteAdaptTC::UpdateTargetSpecificationAtNode(const FiniteElement &el,

   Array<int> dofs;
   tspec_fes->GetElementDofs(T.ElementNo, dofs);
-   const int cnt = tspec.Size()/ncomp; // dofs per scalar-field
+   const int cnt = tspec.Size()/ncomp; //dofs per scalar-field

   for (int i = 0; i < ncomp; i++)
   {
@@ -1240,9 +1197,6 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                                            DenseTensor &Jtr) const
 {
   MFEM_VERIFY(tspec_fesv, "No target specifications have been set.");
-   const int dim = fe.GetDim(),
-             nqp = ir.GetNPoints();
-   Jtrcomp.SetSize(dim, dim, 4*nqp);

   switch (target_type)
   {
@@ -1252,7 +1206,7 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         const DenseMatrix &Wideal =
            Geometries.GetGeomToPerfGeomJac(fe.GetGeomType());
         const int dim = Wideal.Height(),
-                   ndofs = tspec_fes->GetFE(e_id)->GetDof(),
+                   ndofs = tspec_fes->GetFE(0)->GetDof(),
                   ntspec_dofs = ndofs*ncomp;

         Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
@@ -1261,34 +1215,28 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
         Array<int> dofs;
         DenseMatrix D_rho(dim), Q_phi(dim), R_theta(dim);
         tspec_fesv->GetElementVDofs(e_id, dofs);
+         tspec.UseDevice(true);
         tspec.GetSubVector(dofs, tspec_vals);

-         for (int q = 0; q < nqp; q++)
+         for (int i = 0; i < ir.GetNPoints(); i++)
         {
-            const IntegrationPoint &ip = ir.IntPoint(q);
+            const IntegrationPoint &ip = ir.IntPoint(i);
            tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-            Jtr(q) = Wideal; // Initialize to identity
-            for (int d = 0; d < 4; d++)
-            {
-               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(d + 4*q), dim, dim);
-               Jtrcomp_q = Wideal; // Initialize to identity
-            }
+            Jtr(i) = Wideal; //Initialize to identity

-            if (sizeidx != -1) // Set size
+            if (sizeidx != -1) //Set size
            {
               par_vals.SetDataAndSize(tspec_vals.GetData()+sizeidx*ndofs, ndofs);
               const double min_size = par_vals.Min();
               MFEM_VERIFY(min_size > 0.0,
                           "Non-positive size propagated in the target definition.");
               const double size = std::max(shape * par_vals, min_size);
-               Jtr(q).Set(std::pow(size, 1.0/dim), Jtr(q));
-               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(0 + 4*q), dim, dim);
-               Jtrcomp_q = Jtr(q);
-            } // Done size
+               Jtr(i).Set(std::pow(size, 1.0/dim), Jtr(i));
+            } //Done size

            if (target_type == IDEAL_SHAPE_GIVEN_SIZE) { continue; }

-            if (aspectratioidx != -1) // Set aspect ratio
+            if (aspectratioidx != -1) //Set aspect ratio
            {
               if (dim == 2)
               {
@@ -1316,13 +1264,12 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                  D_rho(1,1) = pow(rho2,2./3.);
                  D_rho(2,2) = pow(rho3,2./3.);
               }
-               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(1 + 4*q), dim, dim);
-               Jtrcomp_q = D_rho;
-               DenseMatrix Temp = Jtr(q);
-               Mult(D_rho, Temp, Jtr(q));
-            } // Done aspect ratio

-            if (skewidx != -1) // Set skew
+               DenseMatrix Temp = Jtr(i);
+               Mult(D_rho, Temp, Jtr(i));
+            } //Done aspect ratio
+
+            if (skewidx != -1) //Set skew
            {
               if (dim == 2)
               {
@@ -1358,13 +1305,12 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,

                  Q_phi(2,2) = sin(phi13)*sin(chi);
               }
-               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(2 + 4*q), dim, dim);
-               Jtrcomp_q = Q_phi;
-               DenseMatrix Temp = Jtr(q);
-               Mult(Q_phi, Temp, Jtr(q));
-            } // Done skew

-            if (orientationidx != -1) // Set orientation
+               DenseMatrix Temp = Jtr(i);
+               Mult(Q_phi, Temp, Jtr(i));
+            } // done skew
+
+            if (orientationidx != -1) //Set orientation
            {
               if (dim == 2)
               {
@@ -1389,28 +1335,33 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
                  const double psi   = shape * par_vals_c2;
                  const double beta  = shape * par_vals_c3;

+                  DenseMatrix R_tp(dim), R_beta(dim), R_theta(dim);
                  double ct = cos(theta), st = sin(theta),
-                         cp = cos(psi),   sp = sin(psi),
-                         cb = cos(beta),  sb = sin(beta);
+                         cp = cos(psi),   sp = sin(psi);
+                  R_tp(0,0) = ct*sp;
+                  R_tp(1,0) = st*sp;
+                  R_tp(2,0) = cp;

-                  R_theta = 0.;
-                  R_theta(0,0) = ct*sp;
-                  R_theta(1,0) = st*sp;
-                  R_theta(2,0) = cp;
+                  R_tp(0,1) = -(ct*st*sp*sp)/(1+cp);
+                  R_tp(1,1) = cp+(pow(ct,2.)*pow(sp,2.))/(1+cp);
+                  R_tp(2,1) = -st*sp;

-                  R_theta(0,1) = -st*cb + ct*cp*sb;
-                  R_theta(1,1) = ct*cb + st*cp*sb;
-                  R_theta(2,1) = -sp*sb;
+                  R_tp(0,2) = -cp-(pow(st,2.)*pow(sp,2.))/(1+cp);
+                  R_tp(1,2) = -R_tp(0,1);
+                  R_tp(2,2) =  ct*sp;

-                  R_theta(0,0) = -st*sb - ct*cp*cb;
-                  R_theta(1,0) = ct*sb - st*cp*cb;
-                  R_theta(2,0) = sp*cb;
+                  R_beta = 0.;
+                  R_beta(0,0) = 1.;
+                  R_beta(1,1) =  cos(beta);
+                  R_beta(1,2) = -sin(beta);
+                  R_beta(2,1) =  sin(beta);
+                  R_beta(2,2) =  cos(beta);
+
+                  Mult(R_tp, R_beta, R_theta);
               }
-               DenseMatrix Jtrcomp_q(Jtrcomp.GetData(3 + 4*q), dim, dim);
-               Jtrcomp_q = R_theta;
-               DenseMatrix Temp = Jtr(q);
-               Mult(R_theta, Temp, Jtr(q));
-            } // Done orientation
+               DenseMatrix Temp = Jtr(i);
+               Mult(R_theta, Temp, Jtr(i));
+            } // done orientation
         }
         break;
      }
@@ -1419,353 +1370,6 @@ void DiscreteAdaptTC::ComputeElementTargets(int e_id, const FiniteElement &fe,
   }
 }

-void DiscreteAdaptTC::ComputeElementTargetsGradient(const IntegrationRule &ir,
-                                                    const Vector &elfun,
-                                                    IsoparametricTransformation &Tpr,
-                                                    DenseTensor &dJtr) const
-{
-   MFEM_ASSERT(target_type == IDEAL_SHAPE_UNIT_SIZE || nodes != NULL, "");
-
-   MFEM_VERIFY(tspec_fesv, "No target specifications have been set.");
-
-   dJtr = 0.;
-   const int e_id = Tpr.ElementNo;
-   const FiniteElement *fe = Tpr.GetFE();
-
-   switch (target_type)
-   {
-      case IDEAL_SHAPE_GIVEN_SIZE:
-      case GIVEN_SHAPE_AND_SIZE:
-      {
-         const DenseMatrix &Wideal =
-            Geometries.GetGeomToPerfGeomJac(fe->GetGeomType());
-         const int dim = Wideal.Height(),
-                   ndofs = fe->GetDof(),
-                   ntspec_dofs = ndofs*ncomp;
-
-         Vector shape(ndofs), tspec_vals(ntspec_dofs), par_vals,
-                par_vals_c1(ndofs), par_vals_c2(ndofs), par_vals_c3(ndofs);
-
-         Array<int> dofs;
-         DenseMatrix dD_rho(dim), dQ_phi(dim), dR_theta(dim);
-         DenseMatrix dQ_phi13(dim), dQ_phichi(dim); // dQ_phi is used for dQ/dphi12 in 3D
-         DenseMatrix dR_psi(dim), dR_beta(dim);
-         tspec_fesv->GetElementVDofs(e_id, dofs);
-         tspec.GetSubVector(dofs, tspec_vals);
-
-         DenseMatrix grad_e_c1(ndofs, dim),
-                     grad_e_c2(ndofs, dim),
-                     grad_e_c3(ndofs, dim);
-         Vector grad_ptr_c1(grad_e_c1.GetData(), ndofs*dim),
-                grad_ptr_c2(grad_e_c2.GetData(), ndofs*dim),
-                grad_ptr_c3(grad_e_c3.GetData(), ndofs*dim);
-
-         DenseMatrix grad_phys; // This will be (dof x dim, dof).
-         fe->ProjectGrad(*fe, Tpr, grad_phys);
-
-         for (int i = 0; i < ir.GetNPoints(); i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            DenseMatrix Jtrcomp_s(Jtrcomp.GetData(0 + 4*i), dim, dim); // size
-            DenseMatrix Jtrcomp_d(Jtrcomp.GetData(1 + 4*i), dim, dim); // aspect-ratio
-            DenseMatrix Jtrcomp_q(Jtrcomp.GetData(2 + 4*i), dim, dim); // skew
-            DenseMatrix Jtrcomp_r(Jtrcomp.GetData(3 + 4*i), dim, dim); // orientation
-            DenseMatrix work1(dim), work2(dim), work3(dim);
-
-            if (sizeidx != -1) // Set size
-            {
-               par_vals.SetDataAndSize(tspec_vals.GetData()+sizeidx*ndofs, ndofs);
-
-               grad_phys.Mult(par_vals, grad_ptr_c1);
-               Vector grad_q(dim);
-               tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-               grad_e_c1.MultTranspose(shape, grad_q);
-
-               const double min_size = par_vals.Min();
-               MFEM_VERIFY(min_size > 0.0,
-                           "Non-positive size propagated in the target definition.");
-               const double size = std::max(shape * par_vals, min_size);
-               double dz_dsize = (1./dim)*pow(size, 1./dim - 1.);
-
-               Mult(Jtrcomp_q, Jtrcomp_d, work1); // Q*D
-               Mult(Jtrcomp_r, work1, work2);     // R*Q*D
-
-               for (int d = 0; d < dim; d++)
-               {
-                  DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                  work1 = Wideal;
-                  work1.Set(dz_dsize, work1);    // dz/dsize
-                  work1 *= grad_q(d);            // dz/dsize*dsize/dx
-                  AddMult(work1, work2, dJtr_i); // dz/dx*R*Q*D
-               }
-            } // Done size
-
-            if (target_type == IDEAL_SHAPE_GIVEN_SIZE) { continue; }
-
-            if (aspectratioidx != -1) // Set aspect ratio
-            {
-               if (dim == 2)
-               {
-                  par_vals.SetDataAndSize(tspec_vals.GetData()+
-                                          aspectratioidx*ndofs, ndofs);
-
-                  grad_phys.Mult(par_vals, grad_ptr_c1);
-                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-                  grad_e_c1.MultTranspose(shape, grad_q);
-
-                  const double aspectratio = shape * par_vals;
-                  dD_rho = 0.;
-                  dD_rho(0,0) = -0.5*pow(aspectratio,-1.5);
-                  dD_rho(1,1) = 0.5*pow(aspectratio,-0.5);
-
-                  Mult(Jtrcomp_s, Jtrcomp_r, work1); // z*R
-                  Mult(work1, Jtrcomp_q, work2);     // z*R*Q
-
-                  for (int d = 0; d < dim; d++)
-                  {
-                     DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                     work1 = dD_rho;
-                     work1 *= grad_q(d); // work1 = dD/drho*drho/dx
-                     AddMult(work2, work1, dJtr_i); // z*R*Q*dD/dx
-                  }
-               }
-               else // 3D
-               {
-                  par_vals.SetDataAndSize(tspec_vals.GetData()+
-                                          aspectratioidx*ndofs, ndofs*3);
-                  par_vals_c1.SetData(par_vals.GetData());
-                  par_vals_c2.SetData(par_vals.GetData()+ndofs);
-                  par_vals_c3.SetData(par_vals.GetData()+2*ndofs);
-
-                  grad_phys.Mult(par_vals_c1, grad_ptr_c1);
-                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
-                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
-                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-                  grad_e_c1.MultTranspose(shape, grad_q1);
-                  grad_e_c2.MultTranspose(shape, grad_q2);
-                  grad_e_c3.MultTranspose(shape, grad_q3);
-
-                  const double rho1 = shape * par_vals_c1;
-                  const double rho2 = shape * par_vals_c2;
-                  const double rho3 = shape * par_vals_c3;
-                  dD_rho = 0.;
-                  dD_rho(0,0) = (2./3.)*pow(rho1,-1./3.);
-                  dD_rho(1,1) = (2./3.)*pow(rho2,-1./3.);
-                  dD_rho(2,2) = (2./3.)*pow(rho3,-1./3.);
-
-                  Mult(Jtrcomp_s, Jtrcomp_r, work1); // z*R
-                  Mult(work1, Jtrcomp_q, work2);     // z*R*Q
-
-
-                  for (int d = 0; d < dim; d++)
-                  {
-                     DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                     work1 = dD_rho;
-                     work1(0,0) *= grad_q1(d);
-                     work1(1,2) *= grad_q2(d);
-                     work1(2,2) *= grad_q3(d);
-                     // work1 = dD/dx = dD/drho1*drho1/dx + dD/drho2*drho2/dx
-                     AddMult(work2, work1, dJtr_i); // z*R*Q*dD/dx
-                  }
-               }
-            } // Done aspect ratio
-
-            if (skewidx != -1) // Set skew
-            {
-               if (dim == 2)
-               {
-                  par_vals.SetDataAndSize(tspec_vals.GetData()+
-                                          skewidx*ndofs, ndofs);
-
-                  grad_phys.Mult(par_vals, grad_ptr_c1);
-                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-                  grad_e_c1.MultTranspose(shape, grad_q);
-
-                  const double skew = shape * par_vals;
-
-                  dQ_phi = 0.;
-                  dQ_phi(0,0) = 1.;
-                  dQ_phi(0,1) = -sin(skew);
-                  dQ_phi(1,1) = cos(skew);
-
-                  Mult(Jtrcomp_s, Jtrcomp_r, work2); // z*R
-
-                  for (int d = 0; d < dim; d++)
-                  {
-                     DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                     work1 = dQ_phi;
-                     work1 *= grad_q(d); // work1 = dQ/dphi*dphi/dx
-                     Mult(work1, Jtrcomp_d, work3); // dQ/dx*D
-                     AddMult(work2, work3, dJtr_i); // z*R*dQ/dx*D
-                  }
-               }
-               else
-               {
-                  par_vals.SetDataAndSize(tspec_vals.GetData()+
-                                          skewidx*ndofs, ndofs*3);
-                  par_vals_c1.SetData(par_vals.GetData());
-                  par_vals_c2.SetData(par_vals.GetData()+ndofs);
-                  par_vals_c3.SetData(par_vals.GetData()+2*ndofs);
-
-                  grad_phys.Mult(par_vals_c1, grad_ptr_c1);
-                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
-                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
-                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-                  grad_e_c1.MultTranspose(shape, grad_q1);
-                  grad_e_c2.MultTranspose(shape, grad_q2);
-                  grad_e_c3.MultTranspose(shape, grad_q3);
-
-                  const double phi12  = shape * par_vals_c1;
-                  const double phi13  = shape * par_vals_c2;
-                  const double chi = shape * par_vals_c3;
-
-                  dQ_phi = 0.;
-                  dQ_phi(0,0) = 1.;
-                  dQ_phi(0,1) = -sin(phi12);
-                  dQ_phi(1,1) = cos(phi12);
-
-                  dQ_phi13 = 0.;
-                  dQ_phi13(0,2) = -sin(phi13);
-                  dQ_phi13(1,2) = cos(phi13)*cos(chi);
-                  dQ_phi13(2,2) = cos(phi13)*sin(chi);
-
-                  dQ_phichi = 0.;
-                  dQ_phichi(1,2) = -sin(phi13)*sin(chi);
-                  dQ_phichi(2,2) =  sin(phi13)*cos(chi);
-
-                  Mult(Jtrcomp_s, Jtrcomp_r, work2); // z*R
-
-                  for (int d = 0; d < dim; d++)
-                  {
-                     DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                     work1 = dQ_phi;
-                     work1 *= grad_q1(d); // work1 = dQ/dphi12*dphi12/dx
-                     work1.Add(grad_q2(d), dQ_phi13);  // + dQ/dphi13*dphi13/dx
-                     work1.Add(grad_q3(d), dQ_phichi); // + dQ/dchi*dchi/dx
-                     Mult(work1, Jtrcomp_d, work3); // dQ/dx*D
-                     AddMult(work2, work3, dJtr_i); // z*R*dQ/dx*D
-                  }
-               }
-            } // Done skew
-
-            if (orientationidx != -1) // Set orientation
-            {
-               if (dim == 2)
-               {
-                  par_vals.SetDataAndSize(tspec_vals.GetData()+
-                                          orientationidx*ndofs, ndofs);
-
-                  grad_phys.Mult(par_vals, grad_ptr_c1);
-                  Vector grad_q(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-                  grad_e_c1.MultTranspose(shape, grad_q);
-
-                  const double theta = shape * par_vals;
-                  dR_theta(0,0) = -sin(theta);
-                  dR_theta(0,1) = -cos(theta);
-                  dR_theta(1,0) =  cos(theta);
-                  dR_theta(1,1) = -sin(theta);
-
-                  Mult(Jtrcomp_q, Jtrcomp_d, work1); // Q*D
-                  Mult(Jtrcomp_s, work1, work2);     // z*Q*D
-                  for (int d = 0; d < dim; d++)
-                  {
-                     DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                     work1 = dR_theta;
-                     work1 *= grad_q(d); // work1 = dR/dtheta*dtheta/dx
-                     AddMult(work1, work2, dJtr_i);  // z*dR/dx*Q*D
-                  }
-               }
-               else
-               {
-                  par_vals.SetDataAndSize(tspec_vals.GetData()+
-                                          orientationidx*ndofs, ndofs*3);
-                  par_vals_c1.SetData(par_vals.GetData());
-                  par_vals_c2.SetData(par_vals.GetData()+ndofs);
-                  par_vals_c3.SetData(par_vals.GetData()+2*ndofs);
-
-                  grad_phys.Mult(par_vals_c1, grad_ptr_c1);
-                  grad_phys.Mult(par_vals_c2, grad_ptr_c2);
-                  grad_phys.Mult(par_vals_c3, grad_ptr_c3);
-                  Vector grad_q1(dim), grad_q2(dim), grad_q3(dim);
-                  tspec_fes->GetFE(e_id)->CalcShape(ip, shape);
-                  grad_e_c1.MultTranspose(shape, grad_q1);
-                  grad_e_c2.MultTranspose(shape, grad_q2);
-                  grad_e_c3.MultTranspose(shape, grad_q3);
-
-                  const double theta = shape * par_vals_c1;
-                  const double psi   = shape * par_vals_c2;
-                  const double beta  = shape * par_vals_c3;
-
-                  const double ct = cos(theta), st = sin(theta),
-                               cp = cos(psi),   sp = sin(psi),
-                               cb = cos(beta),  sb = sin(beta);
-
-                  dR_theta = 0.;
-                  dR_theta(0,0) = -st*sp;
-                  dR_theta(1,0) = ct*sp;
-                  dR_theta(2,0) = 0;
-
-                  dR_theta(0,1) = -ct*cb - st*cp*sb;
-                  dR_theta(1,1) = -st*cb + ct*cp*sb;
-                  dR_theta(2,1) = 0.;
-
-                  dR_theta(0,0) = -ct*sb + st*cp*cb;
-                  dR_theta(1,0) = -st*sb - ct*cp*cb;
-                  dR_theta(2,0) = 0.;
-
-                  dR_beta = 0.;
-                  dR_beta(0,0) = 0.;
-                  dR_beta(1,0) = 0.;
-                  dR_beta(2,0) = 0.;
-
-                  dR_beta(0,1) = st*sb + ct*cp*cb;
-                  dR_beta(1,1) = -ct*sb + st*cp*cb;
-                  dR_beta(2,1) = -sp*cb;
-
-                  dR_beta(0,0) = -st*cb + ct*cp*sb;
-                  dR_beta(1,0) = ct*cb + st*cp*sb;
-                  dR_beta(2,0) = 0.;
-
-                  dR_psi = 0.;
-                  dR_psi(0,0) = ct*cp;
-                  dR_psi(1,0) = st*cp;
-                  dR_psi(2,0) = -sp;
-
-                  dR_psi(0,1) = 0. - ct*sp*sb;
-                  dR_psi(1,1) = 0. + st*sp*sb;
-                  dR_psi(2,1) = -cp*sb;
-
-                  dR_psi(0,0) = 0. + ct*sp*cb;
-                  dR_psi(1,0) = 0. + st*sp*cb;
-                  dR_psi(2,0) = cp*cb;
-
-                  Mult(Jtrcomp_q, Jtrcomp_d, work1); // Q*D
-                  Mult(Jtrcomp_s, work1, work2);     // z*Q*D
-                  for (int d = 0; d < dim; d++)
-                  {
-                     DenseMatrix &dJtr_i = dJtr(i + d*ir.GetNPoints());
-                     work1 = dR_theta;
-                     work1 *= grad_q1(d); // work1 = dR/dtheta*dtheta/dx
-                     work1.Add(grad_q2(d), dR_psi);  // +dR/dpsi*dpsi/dx
-                     work1.Add(grad_q3(d), dR_beta); // +dR/dbeta*dbeta/dx
-                     AddMult(work1, work2, dJtr_i);  // z*dR/dx*Q*D
-                  }
-               }
-            } // Done orientation
-         }
-         break;
-      }
-      default:
-         MFEM_ABORT("Incompatible target type for discrete adaptation!");
-   }
-   Jtrcomp.Clear();
-}
-
 void DiscreteAdaptTC::UpdateGradientTargetSpecification(const Vector &x,
                                                        const double dx,
                                                        bool use_flag)
@@ -2095,7 +1699,6 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
 {
   const int dof = el.GetDof(), dim = el.GetDim();

-   DenseMatrix Amat(dim), work1(dim), work2(dim);
   DSh.SetSize(dof, dim);
   DS.SetSize(dof, dim);
   Jrt.SetSize(dim);
@@ -2111,15 +1714,14 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
   elvect = 0.0;
   Vector weights(nqp);
   DenseTensor Jtr(dim, dim, nqp);
-   DenseTensor dJtr(dim, dim, dim*nqp);
   targetC->ComputeElementTargets(T.ElementNo, el, *ir, elfun, Jtr);

   // Limited case.
   DenseMatrix pos0;
   Vector shape, p, p0, d_vals, grad;
-   shape.SetSize(dof);
   if (coeff0)
   {
+      shape.SetSize(dof);
      p.SetSize(dim);
      p0.SetSize(dim);
      pos0.SetSize(dof, dim);
@@ -2139,23 +1741,16 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,

   // Define ref->physical transformation, when a Coefficient is specified.
   IsoparametricTransformation *Tpr = NULL;
-   if (coeff1 || coeff0 || zeta || exact_action)
+   if (coeff1 || coeff0 || zeta)
   {
      Tpr = new IsoparametricTransformation;
      Tpr->SetFE(&el);
      Tpr->ElementNo = T.ElementNo;
+      Tpr->ElementType = ElementTransformation::ELEMENT;
      Tpr->Attribute = T.Attribute;
      Tpr->GetPointMat().Transpose(PMatI); // PointMat = PMatI^T
-      if (exact_action)
-      {
-         targetC->ComputeElementTargetsGradient(*ir, elfun, *Tpr, dJtr);
-      }
   }

-
-   Vector d_detW_dx(dim);
-   Vector d_Winv_dx(dim);
-
   for (int q = 0; q < nqp; q++)
   {
      const IntegrationPoint &ip = ir->IntPoint(q);
@@ -2174,44 +1769,13 @@ void TMOP_Integrator::AssembleElementVectorExact(const FiniteElement &el,
      if (coeff1) { weight_m *= coeff1->Eval(*Tpr, ip); }

      P *= weight_m;
-      AddMultABt(DS, P, PMatO); // w_q det(W) dmu/dx : dA/dx Winv
+      AddMultABt(DS, P, PMatO);

-      if (exact_action)
-      {
-         el.CalcShape(ip, shape);
-         // Derivatives of adaptivity-based targets.
-         // First term: w_q d*(Det W)/dx * mu(T)
-         // d(Det W)/dx = det(W)*Tr[Winv*dW/dx]
-         DenseMatrix dwdx(dim);
-         for (int d = 0; d < dim; d++)
-         {
-            const DenseMatrix &dJtr_q = dJtr(q + d*ir->GetNPoints());
-            Mult(Jrt, dJtr_q, dwdx );
-            d_detW_dx(d) = dwdx.Trace();
-         }
-         d_detW_dx *= weight_m*metric->EvalW(Jpt); // *[w_q*det(W)]*mu(T)
-
-         // Second term: w_q det(W) dmu/dx : AdWinv/dx
-         // dWinv/dx = -Winv*dW/dx*Winv
-         MultAtB(PMatI, DSh, Amat);
-         for (int d = 0; d < dim; d++)
-         {
-            const DenseMatrix &dJtr_q = dJtr(q + d*nqp);
-            Mult(Jrt, dJtr_q, work1); // Winv*dw/dx
-            Mult(work1, Jrt, work2);  // Winv*dw/dx*Winv
-            Mult(Amat, work2, work1); // A*Winv*dw/dx*Winv
-            MultAtB(P, work1, work2); // dmu/dT^T*A*Winv*dw/dx*Winv
-            d_Winv_dx(d) = work2.Trace(); // Tr[dmu/dT : AWinv*dw/dx*Winv]
-         }
-         d_Winv_dx *= -weight_m; // Include (-) factor as well
-
-         d_detW_dx += d_Winv_dx;
-         AddMultVWt(shape, d_detW_dx, PMatO);
-      }
+      // TODO: derivatives of adaptivity-based targets.

      if (coeff0)
      {
-         if (!exact_action) { el.CalcShape(ip, shape); }
+         el.CalcShape(ip, shape);
         PMatI.MultTranspose(shape, p);
         pos0.MultTranspose(shape, p0);
         lim_func->Eval_d1(p, p0, d_vals(q), grad);
@@ -2652,8 +2216,7 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,
   Jpt.SetSize(dim);

   const IntegrationRule *ir = EnergyIntegrationRule(*fe);
-   const int nqp =  ir->GetNPoints();
-   DenseTensor Jtr(dim, dim, nqp);
+   DenseTensor Jtr(dim, dim, ir->GetNPoints());

   metric_energy = 0.0;
   lim_energy = 0.0;
@@ -2666,12 +2229,12 @@ void TMOP_Integrator::ComputeNormalizationEnergies(const GridFunction &x,

      targetC->ComputeElementTargets(i, *fe, *ir, x_vals, Jtr);

-      for (int q = 0; q < nqp; q++)
+      for (int i = 0; i < ir->GetNPoints(); i++)
      {
-         const IntegrationPoint &ip = ir->IntPoint(q);
-         metric->SetTargetJacobian(Jtr(q));
-         CalcInverse(Jtr(q), Jrt);
-         const double weight = ip.weight * Jtr(q).Det();
+         const IntegrationPoint &ip = ir->IntPoint(i);
+         metric->SetTargetJacobian(Jtr(i));
+         CalcInverse(Jtr(i), Jrt);
+         const double weight = ip.weight * Jtr(i).Det();

         fe->CalcDShape(ip, DSh);
         MultAtB(PMatI, DSh, Jpr);
@@ -162,6 +162,21 @@ public:
   { MFEM_ABORT("Not implemented"); }
 };

+/// Shape+Size metric, 2D.
+class TMOP_Metric_SS2D : public TMOP_QualityMetric
+{
+public:
+   // W = 0.5 (1 - cos(theta_Jpr - theta_Jtr)).
+   virtual double EvalW(const DenseMatrix &Jpt) const;
+
+   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
+   { MFEM_ABORT("Not implemented"); }
+
+   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
+                          const double weight, DenseMatrix &A) const
+   { MFEM_ABORT("Not implemented"); }
+};
+
 /// Shape, ideal barrier metric, 2D
 class TMOP_Metric_002 : public TMOP_QualityMetric
 {
@@ -316,21 +331,6 @@ public:

 };

-/// Shape & orientation metric, 2D.
-class TMOP_Metric_085 : public TMOP_QualityMetric
-{
-public:
-   // W = |T-T'|^2, where T'= |T|*I/sqrt(2).
-   virtual double EvalW(const DenseMatrix &Jpt) const;
-
-   virtual void EvalP(const DenseMatrix &Jpt, DenseMatrix &P) const
-   { MFEM_ABORT("Not implemented"); }
-
-   virtual void AssembleH(const DenseMatrix &Jpt, const DenseMatrix &DS,
-                          const double weight, DenseMatrix &A) const
-   { MFEM_ABORT("Not implemented"); }
-};
-
 /// Untangling metric, 2D
 class TMOP_Metric_211 : public TMOP_QualityMetric
 {
@@ -677,25 +677,6 @@ public:
                                      const IntegrationRule &ir,
                                      const Vector &elfun,
                                      DenseTensor &Jtr) const;
-
-   virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
-                                              const Vector &elfun,
-                                              IsoparametricTransformation &Tpr,
-                                              DenseTensor &dJtr) const;
-};
-
-class TMOPMatrixCoefficient  : public MatrixCoefficient
-{
-public:
-   explicit TMOPMatrixCoefficient(int dim) : MatrixCoefficient(dim, dim) { }
-
-   /** @brief Evaluate the derivative of the matrix coefficient with respect to
-       @a comp in the element described by @a T at the point @a ip, storing the
-       result in @a K. */
-   virtual void EvalGrad(DenseMatrix &K, ElementTransformation &T,
-                         const IntegrationPoint &ip, int comp) = 0;
-
-   virtual ~TMOPMatrixCoefficient() { }
 };

 class AnalyticAdaptTC : public TargetConstructor
@@ -704,7 +685,7 @@ protected:
   // Analytic target specification.
   Coefficient *scalar_tspec;
   VectorCoefficient *vector_tspec;
-   TMOPMatrixCoefficient *matrix_tspec;
+   MatrixCoefficient *matrix_tspec;

 public:
   AnalyticAdaptTC(TargetType ttype)
@@ -713,7 +694,7 @@ public:

   virtual void SetAnalyticTargetSpec(Coefficient *sspec,
                                      VectorCoefficient *vspec,
-                                      TMOPMatrixCoefficient *mspec);
+                                      MatrixCoefficient *mspec);

   /** @brief Given an element and quadrature rule, computes ref->target
       transformation Jacobians for each quadrature point in the element.
@@ -722,11 +703,6 @@ public:
                                      const IntegrationRule &ir,
                                      const Vector &elfun,
                                      DenseTensor &Jtr) const;
-
-   virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
-                                              const Vector &elfun,
-                                              IsoparametricTransformation &Tpr,
-                                              DenseTensor &dJtr) const;
 };

 #ifdef MFEM_USE_MPI
@@ -748,17 +724,13 @@ protected:
   // eta1(x+h,y), eta2(x+h,y) ... etan(x+h,y), eta1(x,y+h), eta2(x,y+h) ...
   // same for tspec_pert2h and tspec_pertmix.

-   // Components of Target Jacobian at each quadrature point of an element. This
-   // is required for computation of the derivative using chain rule.
-   mutable DenseTensor Jtrcomp;
-
   // Note: do not use the Nodes of this space as they may not be on the
   // positions corresponding to the values of tspec.
   const FiniteElementSpace *tspec_fes;
   const FiniteElementSpace *tspec_fesv;

-   // These flags can be used by outside functions to avoid recomputing the
-   // tspec and tspec_perth fields again on the same mesh.
+   // These flags can be used by outside functions to avoid recomputing
+   // the tspec and tspec_perth fields again on the same mesh.
   bool good_tspec, good_tspec_grad, good_tspec_hess;

   // Evaluation of the discrete target specification on different meshes.
@@ -865,11 +837,6 @@ public:
                                      const IntegrationRule &ir,
                                      const Vector &elfun,
                                      DenseTensor &Jtr) const;
-
-   virtual void ComputeElementTargetsGradient(const IntegrationRule &ir,
-                                              const Vector &elfun,
-                                              IsoparametricTransformation &Tpr,
-                                              DenseTensor &dJtr) const;
 };

 class TMOPNewtonSolver;
@@ -922,9 +889,6 @@ protected:
   // Specifies that ComputeElementTargets is being called by a FD function.
   // It's used to skip terms that have exact derivative calculations.
   bool fd_call_flag;
-   // Compute the exact action of the Integrator (includes derivative of the
-   // target with respect to spatial position)
-   bool exact_action;

   Array <Vector *> ElemDer;        //f'(x)
   Array <Vector *> ElemPertEnergy; //f(x+h)
@@ -1014,7 +978,7 @@ public:
        lim_dist(NULL), lim_func(NULL), lim_normal(1.0),
        zeta_0(NULL), zeta(NULL), coeff_zeta(NULL), adapt_eval(NULL),
        discr_tc(dynamic_cast<DiscreteAdaptTC *>(tc)),
-        fdflag(false), dxscale(1.0e3), fd_call_flag(false), exact_action(false)
+        fdflag(false), dxscale(1.0e3), fd_call_flag(false)
   { }

   ~TMOP_Integrator();
@@ -1102,9 +1066,6 @@ public:
   void   SetFDhScale(double _dxscale) { dxscale = _dxscale; }
   bool   GetFDFlag() const { return fdflag; }
   double GetFDh()    const { return dx; }
-
-   /** @brief Flag to control if exact action of Integration is effected. */
-   void SetExactActionFlag(bool flag_) { exact_action = flag_; }
 };

 class TMOPComboIntegrator : public NonlinearFormIntegrator
@@ -33,10 +33,11 @@ void AdvectorCG::ComputeAtNewPosition(const Vector &new_nodes,
   const int pnt_cnt = new_field.Size()/ncomp;

   new_field = field0;
-
+   new_field.HostReadWrite();
+   Vector new_field_temp;
   for (int i = 0; i < ncomp; i++)
   {
-      Vector new_field_temp(new_field.GetData()+i*pnt_cnt, pnt_cnt);
+      new_field_temp.MakeRef(new_field, i*pnt_cnt, pnt_cnt);
      ComputeAtNewPositionScalar(new_nodes, new_field_temp);
   }

@@ -94,6 +95,7 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
   double v_max = 0.0;
   const int s = new_field.Size();

+   u.HostReadWrite();
   for (int i = 0; i < s; i++)
   {
      double vel = 0.;
@@ -149,6 +151,7 @@ void AdvectorCG::ComputeAtNewPositionScalar(const Vector &new_nodes,
 #endif

   // Trim the overshoots and undershoots.
+   new_field.HostReadWrite();
   for (int i = 0; i < s; i++)
   {
      if (new_field(i) < glob_minv) { new_field(i) = glob_minv; }
@@ -353,59 +356,20 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      energy_in = nlf->GetEnergy(x);
   }

+   const bool have_b = (b.Size() == Height());
+
   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
   Array<int> xdofs(dof * dim);
   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
   Vector posV(pos.Data(), dof * dim);
-   Vector x_out_loc(fes->GetVSize());

-   if (serial)
-   {
-      const SparseMatrix *cP = fes->GetConformingProlongation();
-      if (!cP) { x_out_loc = x; }
-      else     { cP->Mult(x, x_out_loc); }
-   }
-#ifdef MFEM_USE_MPI
-   else
-   {
-      fes->GetProlongationMatrix()->Mult(x, x_out_loc);
-   }
-#endif
-
-   double min_detJ = infinity();
-   for (int i = 0; i < NE; i++)
-   {
-      fes->GetElementVDofs(i, xdofs);
-      x_out_loc.GetSubVector(xdofs, posV);
-
-      for (int j = 0; j < nsp; j++)
-      {
-         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-         MultAtB(pos, dshape, Jpr);
-         min_detJ = std::min(min_detJ, Jpr.Det());
-      }
-   }
-   double min_detJ_all = min_detJ;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
-                    p_nlf->ParFESpace()->GetComm());
-   }
-#endif
-   bool untangling = false;
-   if (min_detJ_all <= 0) { untangling = true; }
-
-   const bool have_b = (b.Size() == Height());
-
-   Vector x_out(x.Size());
+   Vector x_out(x.Size()), x_out_loc(fes->GetVSize());
   bool x_out_ok = false;
   double scale = 1.0, energy_out = 0.0;
   double norm0 = Norm(r);

-   const double detJ_factor = (solver_type == 1) ? 0.25 : 0.5;
-
+   // Decreases the scaling of the update until the new mesh is valid.
   for (int i = 0; i < 12; i++)
   {
      add(x, -scale, c, x_out);
@@ -423,39 +387,35 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
      }
 #endif

-      // Check det(Jpr) > 0.
-      if (!untangling)
+      int jac_ok = 1;
+      for (int i = 0; i < NE; i++)
      {
-         int jac_ok = 1;
-         for (int i = 0; i < NE; i++)
+         fes->GetElementVDofs(i, xdofs);
+         x_out_loc.GetSubVector(xdofs, posV);
+         for (int j = 0; j < nsp; j++)
         {
-            fes->GetElementVDofs(i, xdofs);
-            x_out_loc.GetSubVector(xdofs, posV);
-            for (int j = 0; j < nsp; j++)
-            {
-               fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
-               MultAtB(pos, dshape, Jpr);
-               if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
-            }
+            fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+            MultAtB(pos, dshape, Jpr);
+            if (Jpr.Det() <= 0.0) { jac_ok = 0; goto break2; }
         }
+      }

-      break2:
-         int jac_ok_all = jac_ok;
+   break2:
+      int jac_ok_all = jac_ok;
 #ifdef MFEM_USE_MPI
-         if (parallel)
-         {
-            MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
-                          p_nlf->ParFESpace()->GetComm());
-         }
+      if (parallel)
+      {
+         MPI_Allreduce(&jac_ok, &jac_ok_all, 1, MPI_INT, MPI_LAND,
+                       p_nlf->ParFESpace()->GetComm());
+      }
 #endif

-         if (jac_ok_all == 0)
-         {
-            if (print_level >= 0)
-            { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
-            scale *= detJ_factor; continue;
-         }
-      } // endif(!untangling)
+      if (jac_ok_all == 0)
+      {
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Neg det(J) found.\n"; }
+         scale *= 0.5; continue;
+      }

      ProcessNewState(x_out);
      if (serial)
@@ -468,37 +428,25 @@ double TMOPNewtonSolver::ComputeScalingFactor(const Vector &x,
         energy_out = p_nlf->GetParGridFunctionEnergy(x_out_loc);
      }
 #endif
-
-      if (untangling)
+      if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
      {
-         if (energy_out > energy_in || std::isnan(energy_out) != 0)
-         {
-            scale *= 0.5;
-         }
-         else { x_out_ok = true; break; }
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
+         scale *= 0.5; continue;
      }
-      else
+
+      oper->Mult(x_out, r);
+      if (have_b) { r -= b; }
+      double norm = Norm(r);
+
+      if (norm > 1.2*norm0)
      {
-         if (energy_out > 1.2*energy_in || std::isnan(energy_out) != 0)
-         {
-            if (print_level >= 0)
-            { mfem::out << "Scale = " << scale << " Increasing energy.\n"; }
-            scale *= 0.5; continue;
-         }
-
-         oper->Mult(x_out, r);
-         if (have_b) { r -= b; }
-         double norm = Norm(r);
-
-         if (norm > 1.2*norm0)
-         {
-            if (print_level >= 0)
-            { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
-            scale *= 0.5; continue;
-         }
-         else { x_out_ok = true; break; }
-      } // endif (untangling)
-   } // enddo (i)
+         if (print_level >= 0)
+         { mfem::out << "Scale = " << scale << " Norm increased.\n"; }
+         scale *= 0.5; continue;
+      }
+      else { x_out_ok = true; break; }
+   }

   if (print_level >= 0)
   {
@@ -625,6 +573,105 @@ void TMOPNewtonSolver::UpdateDiscreteTC(const TMOP_Integrator &ti,
   }
 }

+double TMOPDescentNewtonSolver::ComputeScalingFactor(const Vector &x,
+                                                     const Vector &b) const
+{
+   const FiniteElementSpace *fes = NULL;
+   double energy_in = 0.0;
+#ifdef MFEM_USE_MPI
+   const ParNonlinearForm *p_nlf = dynamic_cast<const ParNonlinearForm *>(oper);
+   MFEM_VERIFY(!(parallel && p_nlf == NULL), "Invalid Operator subclass.");
+   if (parallel)
+   {
+      fes = p_nlf->FESpace();
+      energy_in = p_nlf->GetEnergy(x);
+   }
+#endif
+   const bool serial = !parallel;
+   const NonlinearForm *nlf = dynamic_cast<const NonlinearForm *>(oper);
+   MFEM_VERIFY(!(serial && nlf == NULL), "Invalid Operator subclass.");
+   if (serial)
+   {
+      fes = nlf->FESpace();
+      energy_in = nlf->GetEnergy(x);
+   }
+
+   const int NE = fes->GetMesh()->GetNE(), dim = fes->GetFE(0)->GetDim(),
+             dof = fes->GetFE(0)->GetDof(), nsp = ir.GetNPoints();
+   Array<int> xdofs(dof * dim);
+   DenseMatrix Jpr(dim), dshape(dof, dim), pos(dof, dim);
+   Vector posV(pos.Data(), dof * dim);
+   Vector x_loc(fes->GetVSize());
+
+   double min_detJ = infinity();
+   for (int i = 0; i < NE; i++)
+   {
+      fes->GetElementVDofs(i, xdofs);
+      // TODO x_loc doesn't have valid values here!
+      MFEM_ABORT("This function has to be fixed!");
+      x_loc.GetSubVector(xdofs, posV);
+
+      for (int j = 0; j < nsp; j++)
+      {
+         fes->GetFE(i)->CalcDShape(ir.IntPoint(j), dshape);
+         MultAtB(pos, dshape, Jpr);
+         min_detJ = std::min(min_detJ, Jpr.Det());
+      }
+   }
+   double min_detJ_all = min_detJ;
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      MPI_Allreduce(&min_detJ, &min_detJ_all, 1, MPI_DOUBLE, MPI_MIN,
+                    p_nlf->ParFESpace()->GetComm());
+   }
+#endif
+   if (print_level >= 0)
+   {
+      mfem::out << "Minimum det(J) = " << min_detJ_all << '\n';
+   }
+
+   Vector x_out(x.Size());
+   bool x_out_ok = false;
+   double scale = 1.0, energy_out = 0.0;
+
+   for (int i = 0; i < 7; i++)
+   {
+      add(x, -scale, c, x_out);
+      if (serial)
+      {
+         const SparseMatrix *cP = fes->GetConformingProlongation();
+         if (!cP) { x_loc = x_out; }
+         else     { cP->Mult(x_out,x_loc); }
+         energy_out = nlf->GetGridFunctionEnergy(x_loc);
+      }
+#ifdef MFEM_USE_MPI
+      else
+      {
+         fes->GetProlongationMatrix()->Mult(x_out, x_loc);
+         energy_out = p_nlf->GetParGridFunctionEnergy(x_loc);
+      }
+#endif
+
+      if (energy_out > energy_in || std::isnan(energy_out) != 0)
+      {
+         scale *= 0.5;
+      }
+      else { x_out_ok = true; break; }
+   }
+
+   if (print_level >= 0)
+   {
+      mfem::out << "Energy decrease: "
+                << (energy_in - energy_out) / energy_in * 100.0
+                << "% with " << scale << " scaling.\n";
+   }
+
+   if (x_out_ok == false) { return 0.0; }
+
+   return scale;
+}
+
 #ifdef MFEM_USE_MPI
 // Metric values are visualized by creating an L2 finite element functions and
 // computing the metric values at the nodes.
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tom Stitt	0e8625cf89	add optional MemoryType to the SparseMatrix copy-ctor	2020-07-24 09:13:21 -07:00
Tom Stitt	3f7a432e74	Cherry-picking changes on top of our current hash. There are things that hurt performance in laster mfem@master commits, not sure what yet. This includes: - feature/artv3/cusparse-Spmv - feature/tomstitt/temp-mem-type - patches to tmop.cpp and tmop_tools.cpp to address memory issues - patch to mem_manager/device to use mfem's default allcator instead of the host umpire one	2020-07-22 15:51:04 -07:00
Tom Stitt	a57fd02a4c	Use flattened FORALL instead of FORALL_2D.	2020-06-23 12:11:33 -07:00
Tom Stitt	2ae97ff2da	Converts PADiffusionSetup3D and QuadratureInterpolator::Eval3D kernels from 1 element per thread to 1 qpt/dof per thread for better performance when offloading (there are not enough units of work with 1 element/thread)	2020-06-22 14:42:48 -07:00
Arturo Vargas	2b712207c6	fix merge conflicts	2020-06-22 14:29:28 -07:00
Arturo Vargas	2930c1477f	promove temp vector class member	2020-06-18 13:35:11 -07:00
Arturo Vargas	b3ee631aa6	tmop_tools	2020-05-14 14:33:32 -07:00
Arturo Vargas	daf2fdecec	use tmop_tools from master	2020-05-14 14:24:52 -07:00
Arturo Vargas	d3a0d0a181	Merge branch 'master' into feature/apps/mesh-helper-tmop-ea	2020-05-14 14:19:13 -07:00
Arturo Vargas	07853b9c62	add missing host reads to tmop	2020-05-09 21:41:26 -07:00
Arturo Vargas	49a31c0cf7	Merge branch 'yohann/mass-emat' into feature/apps/mesh-helper-tmop-ea	2020-05-01 10:17:42 -07:00
Arturo Vargas	ba9b251007	Merge branch 'master' into feature/apps/mesh-helper-tmop-ea	2020-05-01 10:13:46 -07:00
Arturo Vargas	5ce2fa9ab9	fix free	2020-04-30 18:06:29 -07:00
Arturo Vargas	78c93de6ce	Merge branch 'gpu/artv3/batch-lu' into feature/apps/mesh-helper-tmop-ea	2020-04-30 13:09:53 -07:00
Arturo Vargas	3d4aa157cb	testing super code	2020-04-28 16:54:23 -07:00
Tomov	a7f1c177c5	Fixed a parallel communication bug in tmop's AdvectorCG.	2020-03-30 22:29:51 -07:00
Arturo Vargas	2d4e3cf77e	minor helper functions	2020-03-24 14:11:37 -07:00