Adding cusparse ILU and incomplete Cholesky solvers on GPU.

2020-08-20 18:55:01 -07:00
234 changed files with 13399 additions and 33711 deletions
@@ -1,61 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale
-# label is closed.  Set to false to disable. If disabled, issues still need to
-# be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale.
-# Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set
-# to `[]` to disable
-exemptLabels:
-  - bug
-  - WIP
-  - ready-for-review
-  - in-review
-  - in-next
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will 
-  be automatically closed.* Thank you for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed
@@ -102,29 +102,28 @@ examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*

-examples/amgx/ex1
-examples/amgx/ex1p
-examples/amgx/.logamgx
-examples/amgx/refined.mesh
-examples/amgx/sol.gf
-examples/amgx/mesh.*
-examples/amgx/sol.*
+examples/sundials/ex9
+examples/sundials/ex1[06]
+examples/sundials/ex9p
+examples/sundials/ex1[06]p

-examples/gingko/ex1
-examples/gingko/refined.mesh
-examples/gingko/sol.gf
-examples/gingko/mesh.*
-examples/gingko/sol.*
-
-examples/hiop/ex9
-examples/hiop/ex9p
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
+examples/sundials/ex9.mesh
+examples/sundials/ex9-mesh.*
+examples/sundials/ex9-init.*
+examples/sundials/ex9-final.*
+examples/sundials/Example9*
+examples/sundials/deformed.*
+examples/sundials/velocity.*
+examples/sundials/elastic_energy.*
+examples/sundials/ex16.mesh
+examples/sundials/ex16-mesh.*
+examples/sundials/ex16-init.*
+examples/sundials/ex16-final.*
+examples/sundials/Example16*

 examples/petsc/ex[1-69]p
 examples/petsc/ex1[0-1]p
+
 examples/petsc/mesh.*
 examples/petsc/sol.*
 examples/petsc/sol_p.*
@@ -142,51 +141,28 @@ examples/petsc/mode_*

 examples/pumi/ex1
 examples/pumi/ex[126]p
+
+examples/hiop/ex9.mesh
+examples/hiop/ex9-mesh.*
+examples/hiop/ex9-init.*
+examples/hiop/ex9-final.*
+
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
 examples/pumi/sol.*
 examples/pumi/displaced.mesh

-examples/sundials/ex9
-examples/sundials/ex1[06]
-examples/sundials/ex9p
-examples/sundials/ex1[06]p
-examples/sundials/ex9.mesh
-examples/sundials/ex9-mesh.*
-examples/sundials/ex9-init.*
-examples/sundials/ex9-final.*
-examples/sundials/Example9*
-examples/sundials/deformed.*
-examples/sundials/velocity.*
-examples/sundials/elastic_energy.*
-examples/sundials/ex16.mesh
-examples/sundials/ex16-mesh.*
-examples/sundials/ex16-init.*
-examples/sundials/ex16-final.*
-examples/sundials/Example16*
-
-examples/superlu/ex1p
-examples/superlu/mesh.*
-examples/superlu/sol.*
-
-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
+
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*

-miniapps/gslib/field-diff
-miniapps/gslib/field-interp
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
@@ -199,7 +175,7 @@ miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
-miniapps/meshing/polar-nc
+
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -211,27 +187,10 @@ miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
-miniapps/meshing/polar-nc.mesh
-
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*

 miniapps/performance/ex1
 miniapps/performance/ex1p
+
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
@@ -249,6 +208,7 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
+
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -263,23 +223,41 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*
+
+miniapps/gslib/field-diff
+miniapps/gslib/findpts
+miniapps/gslib/pfindpts
+
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
+
+miniapps/adjoint/cvsRoberts_ASAi_dns
+miniapps/adjoint/adjoint_advection_diffusion
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/ceed_tests

 # Test script output
 tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg

-# Other tests
-tests/convergence/rates
-tests/convergence/prates
-tests/par-mesh-format/ex1p
-
 # VPATH builds
 build-*/*
@@ -71,8 +71,6 @@ stages:
  - build
  - test
  - deallocate
-  - lassen_build
-  - lassen_test
  - baseline_check
  - baseline_publish

@@ -81,11 +79,7 @@ stages:
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
-setup:
-  tags:
-    - shell
-    - quartz
+.setup:
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -106,15 +100,6 @@ setup:
  before_script:
    - module load gcc/6.1.0

-# On lassen
-.with_gcc_8_3_1:
-  variables:
-    TOOLCHAIN: gcc_8_3_1
-    CXX: g++
-    CC: gcc
-  before_script:
-    - module load gcc/8.3.1
-
 .with_gcc_4_9_3:
  variables:
    TOOLCHAIN: gcc_4_9_3
@@ -305,4 +290,3 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
  - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml
@@ -1,57 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  variables:
-    PLAT: lassen
-
-# Build MFEM
-build_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
-
-build_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
-
-# Sanity check
-sanitycheck_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 15 -q pdebug make -j test
-
-sanitycheck_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_debug_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 30 -q pdebug make -j test
@@ -22,6 +22,10 @@
    MAKE_PAR: 6
    BASELINE_PAR: 18

+# Setup
+setup_quartz:
+  extends: [.setup, .on_quartz]
+
 # Allocate
 allocate_quartz:
  variables:
@@ -11,16 +11,6 @@
 Version 4.1.1 (development)
 ===========================

-libCEED integration improvements
--------------------------------
- Add support for all types of (scalar) Coefficient.
-
- Add support for VectorMassIntegrator and VectorDiffusionIntegrator.
-
- Add support for AssemblyLevel::NONE for MassIntegrator, DiffusionIntegrator,
-  VectorMassIntegrator, and VectorDiffusionIntegrator. This level of assembly
-  fully applies on the fly the finite element operator.
-
 Meshing improvements
 --------------------
 - The graph linear ordering library Gecko, previously an external dependency, is
@@ -48,11 +38,6 @@ Meshing improvements
 - Added complete action of the TMOP Integrator to account for the spatial
  derivatives of discrete and analytic targets.

- Added support for initialization of (serial) non-conforming meshes. Hanging
-  nodes can be marked with Mesh::AddVertexParents when building the mesh with
-  the "init" constructor. The usage is demonstrated in a new meshing miniapp
-  (polar-nc) which generates meshes that are non-conforming from the start.
-
 Performance improvements
 ------------------------
 - Added support for explicit vectorization in the high-performance templated
@@ -66,30 +51,10 @@ Performance improvements

 Improved GPU capabilities
 -------------------------
- Added a new solver class for simple integration with NVIDIA's multigrid
-  library, AmgX. The AmgX class is designed to work as a standalone solver or
-  preconditioner for existing MFEM solvers. It uses MFEM's sparse matrix format
-  for serial runs and the HypreParMatrix format for parallel runs.
-
-  The new solver may be configured to run with one GPU per MPI rank or with more
-  MPI ranks than GPUs. In the latter case, matrices and vectors are consolidated
-  to ranks communicating with the GPUs and the solution is then broadcasted.
-
-  Although CUDA is required to build, the AmgX support is compatible with the
-  MFEM CPU device configuration. The examples/amgx folder illustrates how to
-  integrate AmgX in existing MFEM applications.
-
-  The AmgX solver class is partially based on: "AmgXWrapper: An interface
-  between PETSc and the NVIDIA AmgX library", by Pi-Yueh Chuang and Lorena
-  A. Barba, doi:10.21105/joss.00280.
-
 - Added support for Chebyshev accelerated polynomial smoother on GPU.

 - Optimized AMD/HIP kernel support.

- Enabled HIP support in the libCEED integration, which is now available via the
-  "ceed-hip" device backend.
-
 - Added a Full Assembly mode compatible with Device kernel execution. This
  assembly level builds on top of the current Element Assembly kernels to
  compute a global sparse matrix. All integrators supported by element assembly
@@ -99,13 +64,6 @@ Improved GPU capabilities

 - Added support for BlockOperator on GPU. See the updated Example 5.

- Added partial assembly and GPU support for complex operators, including the
-  classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
-  [Par]SesquilinearForm. See the updated Example 22.
-
- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -132,25 +90,8 @@ Discretization improvements
  Additionally, new LinearForm integrators were also added which make use of
  these new QuadratureFunction coefficient classes.

- Non-conforming meshes are now supported with block nonlinear forms. See the
-  updated Example 19/19p.
-
 - Added support face integrals on the boundaries of NURBS meshes.

- Added support for interpolation of functions in L2, H(div) and H(curl)
-  spaces using GSLIB-FindPoints.
-
- Added support for computing asymptotic error estimates and convergence rates
-  for the whole de Rham sequence based on the new class ConvergenceStudy and new
-  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
-  the tests/convergence directory for sample usage.
-
- The C-function based coefficient classes (FunctionCoefficient,
-  VectorFunctionCoefficient, and MatrixFunctionCoefficient) now use the more
-  general std::function class template. This allows the classes to be backward
-  compatible (i.e. they can still work with C-functions) and, in addition,
-  support any "callable", e.g. lambda functions.
-
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -159,9 +100,6 @@ Linear and nonlinear solvers
 - Added initial support for h- and p-multigrid solvers and preconditioners for
  matrix-based and matrix-free discretizations with basic GPU capability.

- Added wrappers for Hypre's flexible GMRES solver and the new parallel ILU
-  preconditioner. The latter requires hypre version 2.19.0 or later.
-
 - Added a new IterativeSolverMonitor class that allows to monitor the residual
  and solution during the solving process of an IterativeSolver after every
  iteration.
@@ -179,18 +117,6 @@ Linear and nonlinear solvers

 - Added support for the SLEPc eigensolver package.

- Added partially assembled convergent diagonal preconditioner for adaptively
-  refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.
-
- Upgraded SuperLU interface to use SuperLU_DIST 6.3.1. Added a simple SuperLU
-  example in the new directory examples/superlu.
-
- Extended the KINSOL (SUNDIALS) nonlinear solver interface to support the
-  Jacobian-free Newton-Krylov method. A usage example is shown in Example 10p.
-
- Added an interface to the MKL CPardiso solver -- an MPI-parallel sparse direct
-  solver developed by Intel. See Example 11p for an illustration of its usage.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -227,9 +153,6 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
  the Dirichlet problem for the minimal surface equation.

- Added a new meshing miniapp, Polar NC, which demonstrates the construction of
-  polar non-conforming meshes.
-
 - Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
  preconditioning.

@@ -244,55 +167,30 @@ New and updated examples and miniapps
  mesh based on element attributes. Any newly exposed boundary elements are
  assigned attribute numbers related to the trimmed element attributes.

- Added a new miniapp (field-interp) that demonstrates transfer of grid function
-  between different meshes using GSLIB-FindPoints.
-
- Added diagonal preconditioner in Example 6/6p for partial assembly with AMR.
-
 - Added device support in Example 5/5p.

- Added partial assembly and device support to Example 22/22p, with diagonal
-  preconditioning.
-
 - Added the option to plot a function in Mesh Explorer.

 Improved testing
 ----------------
- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
-
 - Added a GitLab pipeline that automates PR testing on supercomputing systems
  and Linux clusters at Lawrence Livermore National Lab (LLNL). This can be
  triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
  and the updated CONTRIBUTING.md file.

- Add tests for the libCEED integration in MFEM.
-
- Added testing of the parallel mesh format in tests/par-mesh-format.
-
 Miscellaneous
 -------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. The
  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal node data. Cell centered data is accessible by
-  ADIOS2 data readers (e.g. Python), but currently not yet implement as of
-  ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
-  of data substreams/subfiles at scale. See examples 5, 9, 12, 16.
-
- Added VTU output of boundary elements and attributes and parallel VTU (PVTU)
-  output of parallel meshes for visualization using ParaView.
+  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
+  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.

 - The integration order used in the ComputeLpError and ComputeElementLpError
  methods of class GridFunction has been increased.

 - Various other simplifications, extensions, and bugfixes in the code.

- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
-  as DEBUG is sometimes used as a macro.
-
- Change the IntegrationRule inside VectorDiffusionIntegrator to use the same
-  quadrature as DiffusionIntegrator.
-

 Version 4.1, released on March 10, 2020
 =======================================
@@ -240,14 +240,12 @@ endif()

 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
-  if (MFEM_USE_MPI)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
+  if (NOT MFEM_USE_MPI)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
+  else()
+    find_package(SUNDIALS REQUIRED
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
  endif()
-  if (MFEM_USE_CUDA)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
-  endif()
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

 # Mesquite
@@ -297,10 +295,6 @@ if (MFEM_USE_CEED)
  find_package(libCEED REQUIRED)
 endif()

-if (MFEM_USE_AMGX)
-  find_package(AMGX REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -352,12 +346,6 @@ if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
 endif()

-if (MFEM_USE_MKL_CPARDISO)
-  if (MFEM_USE_MPI)
-   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
-  endif()
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -384,7 +372,7 @@ endif()
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX)
+    CUSPARSE)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -98,18 +98,15 @@ The MFEM source code has the following structure:
  │   └── web
  │       └── examples
  ├── examples
-  │   ├── amgx
  │   ├── ginkgo
  │   ├── hiop
  │   ├── petsc
  │   ├── pumi
  │   └── sundials
-  |   └── superlu
  ├── fem
  │   └── libceed
  ├── general
  ├── linalg
-  │   └── simd
  ├── mesh
  ├── miniapps
  │   ├── adjoint
@@ -117,14 +114,11 @@ The MFEM source code has the following structure:
  │   ├── electromagnetics
  │   ├── gslib
  │   ├── meshing
-  │   ├── navier
  │   ├── nurbs
  │   ├── performance
  │   ├── tools
  │   └── toys
  └── tests
-      ├── convergence
-      ├── par-mesh-format
      ├── scripts
      ├── unit
      │   ├── ...
@@ -344,10 +344,6 @@ MFEM_USE_SUPERLU = YES/NO
   SuperLURowLocMatrix a distributed CSR matrix class needed by SuperLU. When
   enabled, this option uses the SUPERLU_* library options, see below.

-MFEM_USE_SUPERLU5 = YES/NO
-   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
-   the more recent 6+ versions.
-
 MFEM_USE_STRUMPACK = YES/NO
   Enable MFEM functionality based on the STRUMPACK sparse direct solver and
   preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -360,11 +356,6 @@ MFEM_USE_GINKGO = YES/NO
   https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
   Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.

-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AmgX multigrid library from NVIDIA.
-   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
-   systems with the routines from the AmgX library.
-
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -495,13 +486,6 @@ MFEM_USE_CEED = YES/NO
   library for performant high-order operator evaluation developed by the Center
   for Efficient Exascale Discretizations in the Exascale Computing Project.

-MFEM_USE_MKL_CPARDISO = YES/NO
-   Enables the interface to MKL CPardiso: the Intel MKL Parallel Direct Sparse
-   Solver for Clusters. Make sure to set the correct values for MKL_MPI_WRAPPER
-   and MKL_LIBRARY_SUBDIR as shown in defaults.mk. If you configure MFEM with
-   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
-   OpenMP capabilities are disabled at link time.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -551,14 +535,11 @@ The specific libraries and their options are:
  Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
  Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
  Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
-  When MFEM_USE_CUDA is enabled, only SUNDIALS v5.4.0+ is supported.
  If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
  both MPI and hypre.
-  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
-  for CUDA.
  URL: http://computation.llnl.gov/projects/sundials/sundials-software
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA suppport.
+  Versions: SUNDIALS >= 5.0.0.

 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
  URL: http://trilinos.org/oldsite/packages/mesquite
@@ -593,11 +574,6 @@ The specific libraries and their options are:
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT (Not used), GINKGO_LIB.

- AmgX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AmgX >= 2.1, older versions may work too.
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -669,11 +645,6 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.5.

- MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
-  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
-  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
-  Versions: Intel MKL >= 2020.
-
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
@@ -686,13 +657,13 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.1.0.
+  Versions: OCCA >= 1.0.9.

 - libCEED (optional), used when MFEM_USE_CEED = YES.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.7.
+  Versions: libCEED > 0.6, git-hash fe5822c.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -836,7 +807,6 @@ MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
 MFEM_USE_GINKGO
-MFEM_USE_AMGX
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
@@ -885,12 +855,11 @@ The CMake build system adds auto-detection for the following packages/libraries:

 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
- - ParMETIS
 - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
+ - ParMETIS
 - Ginkgo
- - AMGX
 - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
 - NETCDF
 - MPFR
@@ -25,8 +25,7 @@ This project distributes the sources of several external software products with
 their own respective licenses which can be found in their code and attached
 license files. These software products and their licenses are as follows:

-* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
-* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
+* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -34,7 +34,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
-set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
@@ -74,9 +74,6 @@
 // [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_LEGACY_OPENMP

-// Internal MFEM option: enable group/batch allocation for some small objects.
-#cmakedefine MFEM_USE_MEMALLOC
-
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE

@@ -89,12 +86,12 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

+// Internal MFEM option: enable group/batch allocation for some small objects.
+#cmakedefine MFEM_USE_MEMALLOC
+
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO

-// Enable MFEM functionality based on the AmgX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

@@ -159,7 +156,4 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX

-// Enable interface to the MKL CPardiso library.
-#cmakedefine MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER
@@ -38,19 +38,7 @@ if(NOT ADIOS2_FOUND)
  endif()

  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  
-  # adios2 version 2.5.0 
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
-  
-  # adios2 version 2.6.0 and onwards
-  if(NOT ADIOS2_LIBRARY)
-  	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
-  	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
-  	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
-  	if(MFEM_USE_MPI)
-  	  add_definitions(-DADIOS2_USE_MPI)
-  	endif()
-  endif() 
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 

  include(FindPackageHandleStandardArgs)
  find_package_handle_standard_args(ADIOS2
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - AMGX_FOUND
-#   - AMGX_LIBRARIES
-#   - AMGX_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
-mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
-  "Paths to headers required by AMGX." "Libraries required by AMGX.")
@@ -1,32 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MKL_CPARDISO_FOUND
-#   - MKL_CPARDISO_LIBRARIES
-#   - MKL_CPARDISO_INCLUDE_DIRS
-
-if(NOT MKL_MPI_WRAPPER_LIB)
-  message(FATAL_ERROR "MKL CPardiso enabled but no MKL MPI Wrapper lib specified")
-endif()
-
-if(NOT MKL_LIBRARY_DIR)
-  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
-  set(MKL_LIBRARY_DIR "lib")
-endif()
-
-include(MfemCmakeUtilities)
-mfem_find_package(MKL_CPARDISO MKL_CPARDISO
-    MKL_CPARDISO_DIR "include" mkl_cluster_sparse_solver.h ${MKL_LIBRARY_DIR} mkl_core
-  "Paths to headers required by MKL CPardiso." "Libraries required by MKL CPARDISO."
-  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
-  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential
-  ADD_COMPONENT MKL_MPI_WRAPPER "include" "" ${MKL_LIBRARY_DIR} ${MKL_MPI_WRAPPER_LIB})
@@ -20,14 +20,10 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  "Paths to headers required by SUNDIALS." "Libraries required by SUNDIALS."
  ADD_COMPONENT NVector_Serial
    "include" nvector/nvector_serial.h "lib" sundials_nvecserial
-  ADD_COMPONENT NVector_Cuda
-    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT NVector_Parallel
    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
-  ADD_COMPONENT NVector_MPIPlusX
-    "include" nvector/nvector_mpiplusx.h "lib" sundials_nvecmpiplusx
+  ADD_COMPONENT NVector_ParHyp
+    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
@@ -738,11 +738,10 @@ function(mfem_export_mk_files)
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
+      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -45,9 +45,6 @@
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_MKL_CPARDISO
-#error Building with MKL CPARDISO (MFEM_USE_MKL_CPARDISO=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -93,7 +93,6 @@

 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
-// #define MFEM_USE_SUPERLU5

 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
@@ -101,9 +100,6 @@
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO

-// Enable MFEM functionality based on the AmgX library.
-// #define MFEM_USE_AMGX
-
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

@@ -167,7 +163,4 @@
 // library.
 // #define MFEM_USE_SIMMETRIX

-// Enable interface to the MKL CPardiso library.
-// #define MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER
@@ -32,10 +32,8 @@ MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
-MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -54,7 +52,6 @@ MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
-MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -33,10 +33,8 @@ option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
-option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
-option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
@@ -54,7 +52,6 @@ option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -91,7 +88,7 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
 # and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
@@ -112,7 +109,7 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
    "Additional packages required by ParMETIS.")

-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_5.1.0" CACHE PATH
    "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
 set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
@@ -148,8 +145,6 @@ set(ScaLAPACK_TARGET_NAMES scalapack)

 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")

-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -185,10 +180,6 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
    "Packages that HiOp depends on.")

-set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
-set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
-set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
@@ -120,10 +120,8 @@ MFEM_USE_SUNDIALS      = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
-MFEM_USE_SUPERLU5      = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -142,7 +140,6 @@ MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
-MFEM_USE_MKL_CPARDISO  = NO

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -192,19 +189,15 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
+# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
-   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
-endif
-ifeq ($(MFEM_USE_CUDA),YES)
-   SUNDIALS_LIB += -lsundials_nveccuda
+   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
 endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
@@ -223,15 +216,9 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
 -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)

 # SuperLU library configuration
-ifeq ($(MFEM_USE_SUPERLU5),YES)
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0   
-else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
-endif
+SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
+SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
+SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -264,13 +251,7 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
- -lginkgo_omp -lginkgo_cuda -lginkgo_reference
-
-# AmgX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -391,15 +372,6 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

-# MKL CPardiso library configuration
-MKL_CPARDISO_DIR ?=
-MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
-MKL_LIBRARY_SUBDIR ?= lib
-MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
-
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -78,14 +78,6 @@ groups_parallel=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -115,14 +107,6 @@ groups_all=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -396,15 +380,10 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   # Strip leading and trailing spaces from $1 and store the result in cmd_line
-   shopt -s extglob
-   local cmd_line="${1##+( )}"
-   cmd_line="${cmd_line%%+( )}"
-   shopt -u extglob
-   eval local cmd=(${cmd_line})
+   local cmd=("$@")
   local res=""
   echo $sep
-   echo "<${group}>" "${cmd_line}"
+   echo "<${group}>" "${cmd[@]}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
      timed_run "${cmd[@]}"
@@ -416,15 +395,15 @@ function go()
   else
      res="${red}FAILED${none}"
   fi
-   printf "[${res}] <${group}> ${cmd_line}\n"
+   printf "[${res}] <${group}> ${cmd[*]}\n"
   if [ "${timing}" == "yes" ]; then
      printf "Run time: %s\n" "${timer}"
      timer=(${timer})
      timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
+      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
      summary=("${summary[@]}" "$line")
   else
-      summary=("${summary[@]}" "[${res}] ${cmd_line}")
+      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
   fi
   echo $sep
 }
@@ -459,7 +438,7 @@ function go_group()
      fi
      for run in "${runs[@]}"; do
         if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
+         eval go \${run_prefix} \${run} \${run_suffix} $output
      done
   done
   ${make} clean-exec
@@ -525,7 +504,7 @@ function echo_run()
 {
   echo "   $@"
   { echo "   $@"; echo "$sep";
-     eval "$@"
+     "$@"
     echo "$sep"; } >> "$echo_log" 2>&1
 }

@@ -545,28 +524,6 @@ function build_all()
   echo_run ${make} config ${mfem_config} || exit 1
   echo_run ${make} ${make_j} || exit 1
   echo_run ${make} ${make_all} ${make_j} || exit 1
-   # Build groups in directories other than the directories built by 'make all':
-   for group_params in "${groups[@]}"; do
-      eval params=(${group_params})
-      group_dir="${params[2]}"
-      case "$group_dir" in
-         (examples*|miniapps*)
-            # Built by 'make all'
-            ;;
-         (*)
-            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
-               echo_run mkdir -p "${group_dir}" || exit 1
-               echo_run cd "${group_dir}" || exit 1
-               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
-            else
-               echo_run cd "${group_dir}" || exit 1
-            fi
-            echo_run ${make} clean || exit 1
-            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
-            echo_run cd "${mfem_build_dir}" || exit 1
-            ;;
-      esac
-   done
 }

 # Function that runs all sample runs, given by the array variable "groups".
@@ -149,7 +149,6 @@ namespace mfem {
 * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
 * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
 * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
 * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
 * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -162,7 +161,6 @@ namespace mfem {
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
 * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions betwen meshes
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -127,11 +127,6 @@ if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
 endif()

-# Include the examples/amgx directory if AmgX is enabled
-if (MFEM_USE_AMGX)
-  add_subdirectory(amgx)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -145,3 +140,4 @@ endif()
 if (MFEM_USE_HIOP)
  add_subdirectory(hiop)
 endif()
+
@@ -1,79 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(AMGX_EXAMPLES_SRCS)
-list(APPEND AMGX_EXAMPLES_SRCS
-  ex1.cpp
-  )
-
-if (MFEM_USE_MPI)
-  list(APPEND AMGX_EXAMPLES_SRCS
-    ex1p.cpp
-    )
-endif()
-
-set(AMGX_JSON_FILES amg_pcg.json multi_gs.json precon.json)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add targets to copy *.json files from the source directory
-foreach(JSON_FILE ${AMGX_JSON_FILES})
-  add_custom_command(OUTPUT ${JSON_FILE}
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-    ${CMAKE_CURRENT_SOURCE_DIR}/${JSON_FILE} ${JSON_FILE}
-    COMMENT "copy ${JSON_FILE}")
-endforeach()
-add_custom_target(copy_amgx_json_files DEPENDS ${AMGX_JSON_FILES}
-  COMMENT "Copying AMGX example json files ...")
-
-# Add "test_amgx" target, see below.
-add_custom_target(test_amgx
-  ${CMAKE_CTEST_COMMAND} -R amgx USES_TERMINAL)
-
-# Add one executable per cpp file, adding "amgx_" as prefix. Sets
-# "copy_amgx_json_files" as a prerequisite for the given examples. Also, sets
-# "test_amgx" as a target that depends on the given examples.
-set(PFX amgx_)
-add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
-
-# Testing.
-# The AMGX tests can be run separately using the target "test_amgx"
-# which builds the examples and runs:
-#   ctest -R amgx
-
-# Command line options for the tests.
-# Example 1/1p:
-set(EX1_TEST_OPTS)
-set(EX1P_TEST_OPTS)
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on NVIDIA's multigrid library AmgX.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_AMGX = YES", see the top-level INSTALL file for details (version
-2.1 of AmgX is recommended, though older versions may work too.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -1,38 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "preconditioner": {
-            "print_grid_stats": 1,
-            "print_vis_data": 0,
-            "solver": "AMG",
-            "smoother": {
-                "scope": "jacobi",
-                "solver": "BLOCK_JACOBI",
-                "relaxation_factor": 0.7,
-                "monitor_residual": 0,
-                "print_solve_stats": 0
-            },
-            "print_solve_stats": 0,
-            "presweeps": 1,
-            "interpolator": "D2",
-            "max_row_sum" : 0.9,
-            "strength_threshold" : 0.25,
-            "max_iters": 2,
-            "monitor_residual": 0,
-            "store_res_history": 0,
-            "scope": "amg",
-            "max_levels": 100,
-            "cycle": "V",
-            "postsweeps": 1
-        },
-        "solver": "PCG",
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "max_iters": 100,
-        "monitor_residual": 1,
-        "convergence": "RELATIVE_MAX",
-        "scope": "main",
-        "tolerance": 1e-12,
-        "norm": "L2"
-    }
-}
@@ -1,255 +0,0 @@
-//                                MFEM Example 1
-//                              AmgX Modification
-//
-// Compile with: make ex1
-//
-// AmgX sample runs:
-//               ex1
-//               ex1 -d cuda
-//               ex1 --amgx-file multi_gs.json --amgx-solver
-//               ex1 --amgx-file precon.json --amgx-preconditioner
-//               ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
-//               ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_lib = true;
-   bool amgx_solver = true;
-   const char* amgx_json_file = ""; // JSON file for AmgX
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
-                  "--no-amgx-lib", "Use AmgX in example.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_solver, "--amgx-solver", "--amgx-solver",
-                  "--amgx-preconditioner", "--amgx-preconditioner",
-                  "Configure AMGX as solver or preconditioner.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (pa)
-   {
-      // Jacobi preconditioning in partial assembly mode
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
-   {
-      bool amgx_verbose = false;
-      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
-      amgx.SetOperator(*A.As<SparseMatrix>());
-      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
-   {
-      AmgXSolver amgx;
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-      amgx.InitSerial();
-      amgx.SetOperator(*A.As<SparseMatrix>());
-
-      if (amgx_solver)
-      {
-         amgx.Mult(B,X);
-      }
-      else
-      {
-         PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
-      }
-   }
-   else
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}
@@ -1,321 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                              AmgX Modification
-//
-// Compile with: make ex1p
-//
-// AmgX sample runs:
-//               mpirun -np 4 ex1p
-//               mpirun -np 4 ex1p -d cuda
-//               mpirun -np 10 ex1p --amgx-file amg_pcg.json --amgx-mpi-teams
-//               mpirun -np 4 ex1p --amgx-file amg_pcg.json
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_lib = true;
-   bool amgx_mpi_teams = false;
-   const char* amgx_json_file = ""; // JSON file for AmgX
-   int ndevices = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
-                  "--no-amgx-lib", "Use AmgX in example.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_mpi_teams, "--amgx-mpi-teams", "--amgx-mpi-teams",
-                  "--amgx-mpi-gpu-exclusive", "--amgx-mpi-gpu-exclusive",
-                  "Create MPI teams when using AmgX to load balance between ranks and GPUs.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&ndevices, "-nd","--gpus-per-node-in-teams-mode",
-                  "Number of GPU devices per node (Only used if amgx_mpi_teams is true).");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * If AmgX is available solve using amg preconditioner.
-   //     * With partial assembly, use Jacobi smoothing, for now.
-   Solver *prec = NULL;
-   if (pa)
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
-   {
-      MFEM_VERIFY(!amgx_mpi_teams,
-                  "Please add JSON file to try AmgX with MPI teams mode");
-
-      bool amgx_verbose = false;
-      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
-                            amgx_verbose);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
-   {
-      AmgXSolver amgx;
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-
-      if (amgx_mpi_teams)
-      {
-         // Forms MPI teams to load balance between MPI ranks and GPUs
-         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
-      }
-      else
-      {
-         // Assumes each MPI rank is paired with a GPU
-         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
-      }
-
-      amgx.SetOperator(*A.As<HypreParMatrix>());
-      amgx.Mult(B, X);
-
-      // Release MPI communicators and resources created by AmgX
-      amgx.Finalize();
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-   }
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,75 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_AMGX),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with AMGX)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial AMGX example
-PARALLEL_NAME := Parallel AMGX example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not build)
-
-clean: clean-build
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f .logamgx refined.mesh sol.gf mesh.* sol.*
@@ -1,24 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 10000,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0000001,
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "monitor_residual": 1,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}
@@ -1,21 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 2,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}
@@ -35,7 +35,6 @@
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
 //             * ex1 -pa -d ceed-cuda
-//             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
@@ -176,7 +175,8 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   //a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a.AddDomainIntegrator(new MassIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
@@ -185,19 +185,40 @@ int main(int argc, char *argv[])
   if (static_cond) { a.EnableStaticCondensation(); }
   a.Assemble();

-   OperatorPtr A;
+   OperatorPtr A, As;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   Array<int> empty_list;
+   a.FormSystemMatrix(empty_list, As);
+   //a.FormLinearSystem(empty_list, x, b, A, X, B);
+   //a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);

-   cout << "Size of linear system: " << A->Height() << endl;
+   //cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
   if (!pa)
   {
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
+      //GSSmoother M((SparseMatrix&)(*A));
+
+      //SparseMatrix &Asp = *As.As<SparseMatrix>();
+      SparseMatrix &Asp = a.SpMat();
+
+      Asp.Finalize();
+      Asp.SortColumnIndices();
+
+      Vector tmpx(B.Size());
+      Vector tmpy(B.Size());
+      tmpx = 1.0;
+      tmpy = 0.0;
+
+      //As.As<SparseMatrix>()->Mult(tmpx, tmpy);
+      Asp.Mult(tmpx, tmpy);
+
+      //IncompleteCholesky M(*As.As<SparseMatrix>());
+      IncompleteCholesky M(Asp);
+      //ILUcusparse M(*A.As<SparseMatrix>());
+      PCG(*As, M, B, X, 1, 200, 1e-12, 0.0);
 #else
      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
      UMFPackSolver umf_solver;
@@ -72,7 +72,6 @@ int main(int argc, char *argv[])
   int seed = 75;
   bool slu_solver  = false;
   bool sp_solver = false;
-   bool cpardiso_solver = false;
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -96,10 +95,6 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
                  "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-#ifdef MFEM_USE_MKL_CPARDISO
-   args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
-                  "--no-cpardiso", "Use the MKL CPardiso Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -241,7 +236,7 @@ int main(int argc, char *argv[])
   //    preconditioner for A to be used within the solver. Set the matrices
   //    which define the generalized eigenproblem A x = lambda M x.
   Solver * precond = NULL;
-   if (!slu_solver && !sp_solver && !cpardiso_solver)
+   if (!slu_solver && !sp_solver)
   {
      HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
      amg->SetPrintLevel(0);
@@ -273,19 +268,10 @@ int main(int argc, char *argv[])
         strumpack->SetFromCommandLine();
         precond = strumpack;
      }
-#endif
-#ifdef MFEM_USE_MKL_CPARDISO
-      if (cpardiso_solver)
-      {
-         auto cpardiso = new CPardisoSolver(A->GetComm());
-         cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
-         cpardiso->SetPrintLevel(1);
-         cpardiso->SetOperator(*A);
-         precond = cpardiso;
-      }
 #endif
   }

+
   HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
   lobpcg->SetNumModes(nev);
   lobpcg->SetRandomSeed(seed);
@@ -8,7 +8,6 @@
 //    ex19 -m ../data/beam-hex.mesh
 //    ex19 -m ../data/beam-tet.mesh
 //    ex19 -m ../data/beam-wedge.mesh
-//    ex19 -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -97,7 +96,7 @@ protected:
   Array<FiniteElementSpace *> spaces;

   // Offsets for extracting block vector segments
-   Array<int> &block_trueOffsets;
+   Array<int> &block_offsets;

   // Jacobian for block access
   BlockOperator *jacobian;
@@ -153,7 +152,7 @@ protected:
   Coefficient &mu;

   // Block offsets for variable access
-   Array<int> &block_trueOffsets;
+   Array<int> &block_offsets;

 public:
   RubberOperator(Array<FiniteElementSpace *> &fes, Array<Array<int> *>&ess_bdr,
@@ -247,8 +246,8 @@ int main(int argc, char *argv[])
   spaces[0] = &R_space;
   spaces[1] = &W_space;

-   int R_size = R_space.GetTrueVSize();
-   int W_size = W_space.GetTrueVSize();
+   int R_size = R_space.GetVSize();
+   int W_size = W_space.GetVSize();

   // 6. Define the Dirichlet conditions (set to boundary attribute 1 and 2)
   Array<Array<int> *> ess_bdr(2);
@@ -272,13 +271,13 @@ int main(int argc, char *argv[])
   std::cout << "***********************************************************\n";

   // 8. Define the block structure of the solution vector (u then p)
-   Array<int> block_trueOffsets(3);
-   block_trueOffsets[0] = 0;
-   block_trueOffsets[1] = R_space.GetTrueVSize();
-   block_trueOffsets[2] = W_space.GetTrueVSize();
-   block_trueOffsets.PartialSum();
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = R_space.GetVSize();
+   block_offsets[2] = W_space.GetVSize();
+   block_offsets.PartialSum();

-   BlockVector xp(block_trueOffsets);
+   BlockVector xp(block_offsets);

   // 9. Define grid functions for the current configuration, reference
   //    configuration, final deformation, and pressure
@@ -287,8 +286,8 @@ int main(int argc, char *argv[])
   GridFunction x_def(&R_space);
   GridFunction p_gf(&W_space);

-   x_gf.MakeTRef(&R_space, xp.GetBlock(0), 0);
-   p_gf.MakeTRef(&W_space, xp.GetBlock(1), 0);
+   x_gf.MakeRef(&R_space, xp.GetBlock(0), 0);
+   p_gf.MakeRef(&W_space, xp.GetBlock(1), 0);

   VectorFunctionCoefficient deform(dim, InitialDeformation);
   VectorFunctionCoefficient refconfig(dim, ReferenceConfiguration);
@@ -297,19 +296,14 @@ int main(int argc, char *argv[])
   x_ref.ProjectCoefficient(refconfig);
   p_gf = 0.0;

-   x_gf.SetTrueVector();
-   p_gf.SetTrueVector();
-
   // 10. Initialize the incompressible neo-Hookean operator
-   RubberOperator oper(spaces, ess_bdr, block_trueOffsets,
+   RubberOperator oper(spaces, ess_bdr, block_offsets,
                       newton_rel_tol, newton_abs_tol, newton_iter, c_mu);

   // 11. Solve the Newton system
   oper.Solve(xp);

   // 12. Compute the final deformation
-   x_gf.SetFromTrueVector();
-   p_gf.SetFromTrueVector();
   subtract(x_gf, x_ref, x_def);

   // 13. Visualize the results if requested
@@ -355,7 +349,7 @@ int main(int argc, char *argv[])
 JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
                                               SparseMatrix &mass,
                                               Array<int> &offsets)
-   : Solver(offsets[2]), block_trueOffsets(offsets), pressure_mass(&mass)
+   : Solver(offsets[2]), block_offsets(offsets), pressure_mass(&mass)
 {
   fes.Copy(spaces);

@@ -387,18 +381,18 @@ JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_offsets[0],
+                  block_offsets[1]-block_offsets[0]);
+   Vector pres_in(k.GetData() + block_offsets[1],
+                  block_offsets[2]-block_offsets[1]);

-   Vector disp_out(y.GetData() + block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_offsets[0],
+                   block_offsets[1]-block_offsets[0]);
+   Vector pres_out(y.GetData() + block_offsets[1],
+                   block_offsets[2]-block_offsets[1]);

-   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector temp(block_offsets[1]-block_offsets[0]);
+   Vector temp2(block_offsets[1]-block_offsets[0]);

   // Perform the block elimination for the preconditioner
   mass_pcg->Mult(pres_in, pres_out);
@@ -453,9 +447,9 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
                               double abs_tol,
                               int iter,
                               Coefficient &c_mu)
-   : Operator(fes[0]->GetTrueVSize() + fes[1]->GetTrueVSize()),
+   : Operator(fes[0]->GetVSize() + fes[1]->GetVSize()),
     newton_solver(), newton_monitor("Newton", 1),
-     j_monitor("  GMRES", 3), mu(c_mu), block_trueOffsets(offsets)
+     j_monitor("  GMRES", 3), mu(c_mu), block_offsets(offsets)
 {
   Array<Vector *> rhs(2);
   rhs = NULL; // Set all entries in the array
@@ -477,16 +471,12 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   a->AddDomainIntegrator(new MassIntegrator(one));
   a->Assemble();
   a->Finalize();
-
-   OperatorPtr op;
-   Array<int> p_ess_tdofs;
-   a->FormSystemMatrix(p_ess_tdofs, op);
   pressure_mass = a->LoseMat();
   delete a;

   // Initialize the Jacobian preconditioner
   JacobianPreconditioner *jac_prec =
-      new JacobianPreconditioner(fes, *pressure_mass, block_trueOffsets);
+      new JacobianPreconditioner(fes, *pressure_mass, block_offsets);
   j_prec = jac_prec;

   // Set up the Jacobian solver
@@ -8,7 +8,6 @@
 //    mpirun -np 2 ex19p -m ../data/beam-hex.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-tet.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-wedge.mesh
-//    mpirun -np 2 ex19p -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -197,8 +196,10 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI
-   MPI_Session mpi;
-   const int myid = mpi.WorldRank();
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);

   // 2. Parse command-line options
   const char *mesh_file = "../data/beam-tet.mesh";
@@ -238,6 +239,7 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
+      MPI_Finalize();
      return 1;
   }
   if (myid == 0)
@@ -397,6 +399,8 @@ int main(int argc, char *argv[])
   // 19. Free the used memory
   delete pmesh;

+   MPI_Finalize();
+
   return 0;
 }

@@ -470,11 +474,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
   {
      HypreBoomerAMG *stiff_prec_amg = new HypreBoomerAMG();
      stiff_prec_amg->SetPrintLevel(0);
-
-      if (!spaces[0]->GetParMesh()->Nonconforming())
-      {
-         stiff_prec_amg->SetElasticityOptions(spaces[0]);
-      }
+      stiff_prec_amg->SetElasticityOptions(spaces[0]);

      stiff_prec = stiff_prec_amg;

@@ -33,7 +33,6 @@
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
@@ -123,7 +122,7 @@ int main(int argc, char *argv[])
   {
      int ref_levels =
         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
+      for (int l = 0; l < ref_levels-1; l++)
      {
         mesh.UniformRefinement();
      }
@@ -135,7 +134,7 @@ int main(int argc, char *argv[])
   ParMesh pmesh(MPI_COMM_WORLD, mesh);
   mesh.Clear();
   {
-      int par_ref_levels = 2;
+      int par_ref_levels = 1;
      for (int l = 0; l < par_ref_levels; l++)
      {
         pmesh.UniformRefinement();
@@ -217,6 +216,13 @@ int main(int argc, char *argv[])
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);

+   SparseMatrix Asp;
+   A.As<HypreParMatrix>()->GetDiag(Asp);
+   Vector diag;
+
+   StopWatch sw;
+   sw.Start();
+
   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
   //     * With partial assembly, use Jacobi smoothing, for now.
@@ -230,7 +236,14 @@ int main(int argc, char *argv[])
   }
   else
   {
-      prec = new HypreBoomerAMG;
+      //prec = new HypreBoomerAMG;
+      Asp.Finalize();
+      Asp.SortColumnIndices();
+
+      Asp.GetDiag(diag);
+      prec = new OperatorJacobiSmoother(diag, ess_tdof_list);
+      //prec = new IncompleteCholesky(Asp);
+      //prec = new ILUcusparse(Asp);
   }
   CGSolver cg(MPI_COMM_WORLD);
   cg.SetRelTol(1e-12);
@@ -241,6 +254,9 @@ int main(int argc, char *argv[])
   cg.Mult(B, X);
   delete prec;

+   sw.Stop();
+   cout << "Step 13 solve time " << sw.RealTime() << endl;
+
   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a.RecoverFEMSolution(X, b, x);
@@ -6,19 +6,17 @@
 //               ex22 -m ../data/inline-tri.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
 //               ex22 -m ../data/inline-tet.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
+// With partial assembly:
+//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
+//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
+//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -84,7 +82,6 @@ int main(int argc, char *argv[])
   bool herm_conv = true;
   bool exact_sol = true;
   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -117,8 +114,6 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -148,18 +143,13 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase resolution. In this example we do
+   // 3. Refine the mesh to increase resolution. In this example we do
   //    'ref_levels' of uniform refinement where the user specifies
   //    the number of levels with the '-r' option.
   for (int l = 0; l < ref_levels; l++)
@@ -167,7 +157,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
   //    order.
   if (dim == 1 && prob != 0 )
@@ -189,7 +179,7 @@ int main(int argc, char *argv[])
   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
        << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined based on the type
   //    of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -201,12 +191,12 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
   ComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 8. Define the solution vector u as a complex finite element grid function
+   // 7. Define the solution vector u as a complex finite element grid function
   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
   //    the exact solution if it is known.
   ComplexGridFunction u(fespace);
@@ -228,6 +218,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -280,7 +271,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 9. Set up the sesquilinear form a(.,.) on the finite element space
+   // 8. Set up the sesquilinear form a(.,.) on the finite element space
   //    corresponding to the damped harmonic oscillator operator of the
   //    appropriate type:
   //
@@ -323,7 +314,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 9a. Set up the bilinear form for the preconditioner corresponding to the
+   // 8a. Set up the bilinear form for the preconditioner corresponding to the
   //     appropriate operator
   //
   //      0) A scalar H1 field
@@ -358,9 +349,9 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 10. Assemble the form and the corresponding linear system, applying any
-   //     necessary transformations such as: assembly, eliminating boundary
-   //     conditions, conforming constraints for non-conforming AMR, etc.
+   // 9. Assemble the form and the corresponding linear system, applying any
+   //    necessary transformations such as: assembly, eliminating boundary
+   //    conditions, conforming constraints for non-conforming AMR, etc.
   a->Assemble();
   pcOp->Assemble();

@@ -371,7 +362,7 @@ int main(int argc, char *argv[])

   cout << "Size of linear system: " << A->Width() << endl << endl;

-   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
+   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
   //     preconditioner based on the appropriate sparse smoother.
   {
      Array<int> blockOffsets;
@@ -428,7 +419,7 @@ int main(int argc, char *argv[])
      gmres.Mult(B, U);
   }

-   // 12. Recover the solution as a finite element grid function and compute the
+   // 11. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a->RecoverFEMSolution(U, b, u);

@@ -460,7 +451,7 @@ int main(int argc, char *argv[])
      cout << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -475,7 +466,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -534,7 +525,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -7,18 +7,16 @@
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
 //               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
+// With partial assembly:
+//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
+//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
+//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa
 //
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
@@ -48,6 +46,7 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.

+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -91,7 +90,6 @@ int main(int argc, char *argv[])
   bool herm_conv = true;
   bool exact_sol = true;
   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -126,8 +124,6 @@ int main(int argc, char *argv[])
                  "Enable or disable GLVis visualization.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -164,24 +160,19 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ser_ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -191,7 +182,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
   //    the specified order.
   if (dim == 1 && prob != 0 )
@@ -219,7 +210,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    based on the type of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -231,14 +222,14 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
   ParComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 10. Define the solution vector u as a parallel complex finite element grid
-   //     function corresponding to fespace. Initialize u with initial guess of
-   //     1+0i or the exact solution if it is known.
+   // 9. Define the solution vector u as a parallel complex finite element grid
+   //    function corresponding to fespace. Initialize u with initial guess of
+   //    1+0i or the exact solution if it is known.
   ParComplexGridFunction u(fespace);
   ParComplexGridFunction * u_exact = NULL;
   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -258,6 +249,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -312,7 +304,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
   //     space corresponding to the damped harmonic oscillator operator of the
   //     appropriate type:
   //
@@ -355,7 +347,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 11a. Set up the parallel bilinear form for the preconditioner
+   // 10a. Set up the parallel bilinear form for the preconditioner
   //      corresponding to the appropriate operator
   //
   //      0) A scalar H1 field
@@ -389,7 +381,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
@@ -407,7 +399,7 @@ int main(int argc, char *argv[])
           << 2 * fespace->GlobalTrueVSize() << endl << endl;
   }

-   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
   //     diagonal preconditioner based on the appropriate multigrid
   //     preconditioner from hypre.
   {
@@ -468,7 +460,7 @@ int main(int argc, char *argv[])
      fgmres.SetPrintLevel(1);
      fgmres.Mult(B, U);
   }
-   // 14. Recover the parallel grid function corresponding to U. This is the
+   // 13. Recover the parallel grid function corresponding to U. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(U, b, u);

@@ -503,7 +495,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -523,7 +515,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -588,7 +580,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -82,27 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -368,19 +365,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -422,13 +419,13 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
@@ -766,7 +763,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -777,13 +774,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -794,13 +792,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -811,13 +810,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -831,18 +831,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -855,18 +856,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -879,13 +881,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -82,27 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -396,19 +393,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -456,13 +453,13 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));
@@ -822,7 +819,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -833,13 +830,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -850,13 +848,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -867,13 +866,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -887,18 +887,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -911,18 +912,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -935,13 +937,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -60,7 +60,6 @@ int main(int argc, char *argv[])
   bool static_cond = false;
   bool visualization = 1;
   bool amg_elast = 0;
-   bool reorder_space = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -76,8 +75,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
-                  "Use byNODES ordering of vector space instead of byVDIM");
   args.Parse();
   if (!args.Good())
   {
@@ -159,14 +156,7 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order, dim);
-      if (reorder_space)
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
-      }
-      else
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
-      }
+      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
   }
   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
@@ -259,7 +249,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      amg->SetSystemsOptions(dim, reorder_space);
+      amg->SetSystemsOptions(dim);
   }
   HyprePCG *pcg = new HyprePCG(A);
   pcg->SetTol(1e-8);
@@ -108,11 +108,7 @@ int main(int argc, char *argv[])
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   BilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   LinearForm b(&fespace);

   ConstantCoefficient one(1.0);
@@ -203,10 +199,9 @@ int main(int argc, char *argv[])
         umf_solver.Mult(B, X);
 #endif
      }
-      else // Diagonal preconditioning in partial assembly mode.
+      else // No preconditioning for now in partial assembly mode.
      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 3, 2000, 1e-12, 0.0);
+         CG(*A, B, X, 3, 2000, 1e-12, 0.0);
      }

      // 18. After solving the linear system, reconstruct the solution as a
@@ -129,11 +129,7 @@ int main(int argc, char *argv[])
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   ParBilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   ParLinearForm b(&fespace);

   ConstantCoefficient one(1.0);
@@ -224,26 +220,17 @@ int main(int argc, char *argv[])

      // 17. Solve the linear system A X = B.
      //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-      //     * With partial assembly, use a diagonal preconditioner.
-      Solver *M = NULL;
-      if (pa)
-      {
-         M = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-      else
-      {
-         HypreBoomerAMG *amg = new HypreBoomerAMG;
-         amg->SetPrintLevel(0);
-         M = amg;
-      }
+      //     * With partial assembly, use no preconditioner, for now.
+      HypreBoomerAMG *amg = NULL;
+      if (!pa) { amg = new HypreBoomerAMG; amg->SetPrintLevel(0); }
      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-6);
      cg.SetMaxIter(2000);
      cg.SetPrintLevel(3); // print the first and the last iterations only
-      cg.SetPreconditioner(*M);
+      if (amg) { cg.SetPreconditioner(*amg); }
      cg.SetOperator(*A);
      cg.Mult(B, X);
-      delete M;
+      delete amg;

      // 18. Switch back to the host and extract the parallel grid function
      //     corresponding to the finite element approximation X. This is the
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Ginkgo high-performance linear algebra library
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM for solving nonlinear constrained optimization problems, including
@@ -48,12 +48,6 @@ endif
 ifeq ($(MFEM_USE_GINKGO),YES)
   SUBDIRS += ginkgo
 endif
-ifeq ($(MFEM_USE_AMGX),YES)
-   SUBDIRS += amgx
-endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-   SUBDIRS += superlu
-endif

 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -125,11 +119,6 @@ ex11p-test-superlu: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
 test-par-YES: ex11p-test-superlu
 endif
-ifeq ($(MFEM_USE_MKL_CPARDISO),YES)
-ex11p-test-cpardiso: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
-test-par-YES: ex11p-test-cpardiso
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the PETSc suite.
@@ -410,16 +410,21 @@ int main(int argc, char *argv[])
   }

   // 12. Free the used memory.
-   delete lobpcg;
-   delete slepc;
+   if (!use_slepc)
+   {
+      delete lobpcg;
+   }
+   else
+   {
+      delete slepc;
+   }
   delete precond;
   delete M;
   delete A;
-   delete pA;
-   delete pM;
 #if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
   delete Arow;
 #endif
+
   delete fespace;
   if (order > 0)
   {
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Parallel Unstructured Mesh Infrastructure,
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the SUNDIALS suite of time integration and
@@ -721,14 +721,12 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,

   if (nls_type == KINSOL)
   {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_LINESEARCH, true);
-      kinsolver->SetJFNK(true);
-      kinsolver->SetLSMaxIter(100);
+      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_NONE, true);
      newton_solver = kinsolver;
      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(200);
      newton_solver->SetRelTol(rel_tol);
-      newton_solver->SetPrintLevel(1);
+      newton_solver->SetPrintLevel(0);
      kinsolver->SetMaxSetupCalls(4);
   }
   else
@@ -13,27 +13,17 @@
 //    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 7 -dt 0.005 -tf 9
 //    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 8 -dt 0.02 -tf 8 -o 2
 //
-// Device sample runs:
-//    ex9 -pa
-//    ex9 -ea
-//    ex9 -fa
-//    ex9 -pa -m ../../data/periodic-cube.mesh
-//    ex9 -pa -m ../../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -63,54 +53,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &K, A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         A = K;
-         A *= -dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -120,21 +62,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   BilinearForm &M, &K;
+   SparseMatrix &M, &K;
   const Vector &b;
-   Solver *M_prec;
+   DSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };


@@ -145,16 +85,11 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
   int ref_levels = 2;
   int order = 3;
-   bool pa = false;
-   bool ea = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   int ode_solver_type = 7;
+   int ode_solver_type = 4;
   double t_final = 10.0;
   double dt = 0.01;
-   bool visualization = false;
+   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;

@@ -173,14 +108,6 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver:\n\t"
                  "1 - Forward Euler,\n\t"
@@ -201,9 +128,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -223,9 +147,6 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   Device device(device_config);
-   device.Print();
-
   // 2. Read the mesh from the given mesh file. We can handle geometrically
   //    periodic meshes in this code.
   Mesh mesh(mesh_file, 1, 1);
@@ -247,7 +168,7 @@ int main(int argc, char *argv[])

   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
   FiniteElementSpace fes(&mesh, &fec);

   cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -260,23 +181,8 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   if (pa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-   else if (ea)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
+   BilinearForm k(&fes);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -288,11 +194,11 @@ int main(int argc, char *argv[])
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

   m.Assemble();
+   m.Finalize();
   int skip_zeros = 0;
   k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
   k.Finalize(skip_zeros);
+   b.Assemble();

   // 6. Define the initial conditions, save the corresponding grid function to
   //    a file and (optionally) save data in the VisIt format and initialize
@@ -333,20 +239,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9", &mesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", &u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
   socketstream sout;
   if (visualization)
   {
@@ -373,7 +265,7 @@ int main(int argc, char *argv[])

   // 7. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(m, k, b);
+   FE_Evolution adv(m.SpMat(), k.SpMat(), b);

   double t = 0.0;
   adv.SetTime(t);
@@ -397,18 +289,12 @@ int main(int argc, char *argv[])
         cvode->UseSundialsLinearSolver();
         ode_solver = cvode; break;
      case 8:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         arkode->SetOrder(4);
-         ode_solver = arkode; break;
      case 9:
         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(FEHLBERG_13_7_8);
+         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
         ode_solver = arkode; break;
   }

@@ -443,13 +329,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
      }
   }

@@ -463,7 +342,6 @@ int main(int argc, char *argv[])

   // 10. Free the used memory.
   delete ode_solver;
-   delete pd;
   delete dc;

   return 0;
@@ -471,23 +349,12 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
 {
-   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-      M_solver.SetOperator(M);
-      dg_solver = NULL;
-   }
-   M_solver.SetPreconditioner(*M_prec);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);
+
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -503,21 +370,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
   M_solver.Mult(z, y);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   MFEM_VERIFY(dg_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   K.Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -13,28 +13,17 @@
 //    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 7 -dt 0.0025 -tf 9
 //    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 8 -dt 0.01 -tf 8 -o 2
 //
-// Device sample runs:
-//    mpirun -np 4 ex9p -pa
-//    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
-//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh
-//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. Saving of time-dependent data files for visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org), as
-//               well as the optional saving with ADIOS2 (adios2.readthedocs.io)
-//               are also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -63,66 +52,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &K;
-   SparseMatrix M_diag;
-   HypreParMatrix *A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        A(NULL),
-        linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-
-      M.GetDiag(M_diag);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         delete A;
-         A = Add(-dt, K, 0.0, K);
-         SparseMatrix A_diag;
-         A->GetDiag(A_diag);
-         A_diag.Add(1.0, M_diag);
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   ~DG_Solver()
-   {
-      delete A;
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -132,21 +61,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   OperatorHandle M, K;
+   HypreParMatrix &M, &K;
   const Vector &b;
-   Solver *M_prec;
+   HypreSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };


@@ -164,17 +91,11 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 0;
   int order = 3;
-   bool pa = false;
-   bool ea = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   int ode_solver_type = 7;
+   int ode_solver_type = 4;
   double t_final = 10.0;
   double dt = 0.01;
-   bool visualization = false;
+   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
-   bool adios2 = false;
   bool binary = false;
   int vis_steps = 5;

@@ -195,14 +116,6 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly in parallel.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver:\n\t"
                  "1 - Forward Euler,\n\t"
@@ -223,12 +136,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -248,7 +155,6 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-
   // check for vaild ODE solver option
   if (ode_solver_type < 1 || ode_solver_type > 9)
   {
@@ -260,9 +166,6 @@ int main(int argc, char *argv[])
      return 3;
   }

-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle geometrically periodic meshes in this code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -294,7 +197,7 @@ int main(int argc, char *argv[])

   // 6. Define the parallel discontinuous DG finite element space on the
   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);

   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
@@ -311,24 +214,8 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   if (pa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-   else if (ea)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
   m->AddDomainIntegrator(new MassIntegrator);
+   ParBilinearForm *k = new ParBilinearForm(fes);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -339,13 +226,15 @@ int main(int argc, char *argv[])
   b->AddBdrFaceIntegrator(
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

-   int skip_zeros = 0;
   m->Assemble();
-   k->Assemble(skip_zeros);
-   b->Assemble();
   m->Finalize();
+   int skip_zeros = 0;
+   k->Assemble(skip_zeros);
   k->Finalize(skip_zeros);
+   b->Assemble();

+   HypreParMatrix *M = m->ParallelAssemble();
+   HypreParMatrix *K = k->ParallelAssemble();
   HypreParVector *B = b->ParallelAssemble();

   // 8. Define the initial conditions, save the corresponding grid function to
@@ -384,8 +273,6 @@ int main(int argc, char *argv[])
      {
         dc = new VisItDataCollection("Example9-Parallel", pmesh);
         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
      }
      dc->RegisterField("solution", u);
      dc->SetCycle(0);
@@ -393,41 +280,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9P", pmesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex9-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
   socketstream sout;
   if (visualization)
   {
@@ -460,7 +312,7 @@ int main(int argc, char *argv[])

   // 9. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*M, *K, *B);

   double t = 0.0;
   adv.SetTime(t);
@@ -532,23 +384,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
-
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
      }
   }

@@ -568,18 +403,13 @@ int main(int argc, char *argv[])
   delete u;
   delete B;
   delete b;
+   delete K;
   delete k;
+   delete M;
   delete m;
   delete fes;
   delete pmesh;
   delete ode_solver;
-   delete pd;
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
   delete dc;

   MPI_Finalize();
@@ -588,43 +418,15 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
+FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
                           const Vector &_b)
   : TimeDependentOperator(_M.Height()),
-     b(_b),
-     M_solver(_M.ParFESpace()->GetComm()),
-     z(_M.Height())
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
-   {
-      M.Reset(&_M, false);
-      K.Reset(&_K, false);
-   }
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);

-   M_solver.SetOperator(*M);
-
-   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
-      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
-      M_prec = hypre_prec;
-
-      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
-   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }
-
-   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -632,28 +434,14 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
   M_solver.SetPrintLevel(0);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   K->Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
   // y = M^{-1} (K x + b)
-   K->Mult(x, z);
+   K.Mult(x, z);
   z += b;
   M_solver.Mult(z, y);
 }

-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}
-

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -1,62 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(SUPERLU_EXAMPLES_SRCS)
-if (MFEM_USE_MPI)
-  list(APPEND SUPERLU_EXAMPLES_SRCS
-    ex1p.cpp
-    )
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_superlu" target, see below.
-add_custom_target(test_superlu
-  ${CMAKE_CTEST_COMMAND} -R superlu USES_TERMINAL)
-
-# Add one executable per cpp file, adding "superlu_" as prefix. Sets
-# "test_superlu" as a target that depends on the given examples.
-set(PFX superlu_)
-add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
-
-# Testing.
-# The SuperLU tests can be run separately using the target "test_superlu"
-# which builds the examples and runs:
-#   ctest -R superlu
-
-# Command line options for the tests.
-# Example 1: Test SuperLU on the simple Poisson problem
-set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
-set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,321 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             SuperLU Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   int slu_colperm = 4;
-   int slu_rowperm = 1;
-   int slu_iterref = 2;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&slu_colperm, "-cp", "--colperm",
-                  "SuperLU Column Permutation Method:  0-NATURAL, 1-MMD-ATA "
-                  "2-MMD_AT_PLUS_A, 3-COLAMD, 4-METIS_AT_PLUS_A, 5-PARMETIS "
-                  "6-ZOLTAN");
-   args.AddOption(&slu_rowperm, "-rp", "--rowperm",
-                  "SuperLU Row Permutation Method:  0-NOROWPERM, 1-LargeDiag");
-   args.AddOption(&slu_iterref, "-rp", "--rowperm",
-                  "SuperLU Iterative Refinement:  0-NOREFINE, 1-Single, "
-                  "2-Double, 3-Extra");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B utilizing SuperLU.
-   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD);
-   Operator *SLU_A = new SuperLURowLocMatrix(*A.As<HypreParMatrix>());
-   superlu->SetPrintStatistics(true);
-   superlu->SetSymmetricPattern(false);
-
-   if (slu_colperm == 0)
-   {
-      superlu->SetColumnPermutation(superlu::NATURAL);
-   }
-   else if (slu_colperm == 1)
-   {
-      superlu->SetColumnPermutation(superlu::MMD_ATA);
-   }
-   else if (slu_colperm == 2)
-   {
-      superlu->SetColumnPermutation(superlu::MMD_AT_PLUS_A);
-   }
-   else if (slu_colperm == 3)
-   {
-      superlu->SetColumnPermutation(superlu::COLAMD);
-   }
-   else if (slu_colperm == 4)
-   {
-      superlu->SetColumnPermutation(superlu::METIS_AT_PLUS_A);
-   }
-   else if (slu_colperm == 5)
-   {
-      superlu->SetColumnPermutation(superlu::PARMETIS);
-   }
-   else if (slu_colperm == 6)
-   {
-      superlu->SetColumnPermutation(superlu::ZOLTAN);
-   }
-
-   if (slu_rowperm == 0)
-   {
-      superlu->SetRowPermutation(superlu::NOROWPERM);
-   }
-   else if (slu_rowperm == 1)
-   {
-#ifdef MFEM_USE_SUPERLU5
-      superlu->SetRowPermutation(superlu::LargeDiag);
-#else
-      superlu->SetRowPermutation(superlu::LargeDiag_MC64);
-#endif
-   }
-
-   if (slu_iterref == 0)
-   {
-      superlu->SetIterativeRefine(superlu::NOREFINE);
-   }
-   else if (slu_iterref == 1)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_SINGLE);
-   }
-   else if (slu_iterref == 2)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_DOUBLE);
-   }
-   else if (slu_iterref == 3)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_EXTRA);
-   }
-
-   superlu->SetOperator(*SLU_A);
-   superlu->SetPrintStatistics(true);
-   superlu->Mult(B, X);
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,79 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/superlu/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = 
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_SUPERLU),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with SuperLU)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-PARALLEL_NAME := Parallel SuperLU example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-
-# Testing: Specific execution options:
-# Example 1: Test SuperLU on the simple poisson problem
-EX1_COMMON_ARGS := -m ../../data/star.mesh
-EX1P_ARGS := $(EX1_COMMON_ARGS)
-ex1p-test-par: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX1P_ARGS))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f mesh.* sol.*
@@ -17,7 +17,6 @@ set(SRCS
  bilininteg_convection_ea.cpp
  bilininteg_dgtrace_pa.cpp
  bilininteg_dgtrace_ea.cpp
-  bilininteg_diffusion_mf.cpp
  bilininteg_diffusion_pa.cpp
  bilininteg_diffusion_ea.cpp
  bilininteg_divergence.cpp
@@ -25,17 +24,13 @@ set(SRCS
  bilininteg_hdiv.cpp
  bilininteg_vectorfe.cpp
  bilininteg_gradient.cpp
-  bilininteg_mass_mf.cpp
  bilininteg_mass_pa.cpp
  bilininteg_mass_ea.cpp
  bilininteg_transpose_ea.cpp
  bilininteg_vecdiffusion.cpp
-  bilininteg_vecdiffusion_mf.cpp
  bilininteg_vecmass.cpp
-  bilininteg_vecmass_mf.cpp
  coefficient.cpp
  complex_fem.cpp
-  convergence.cpp
  datacollection.cpp
  eltrans.cpp
  estimators.cpp
@@ -46,9 +41,6 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  libceed/ceed.cpp
-  libceed/diffusion.cpp
-  libceed/mass.cpp
  linearform.cpp
  lininteg.cpp
  multigrid.cpp
@@ -73,7 +65,6 @@ set(HDRS
  bilininteg.hpp
  coefficient.hpp
  complex_fem.hpp
-  convergence.hpp
  datacollection.hpp
  eltrans.hpp
  estimators.hpp
@@ -85,9 +76,6 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  libceed/ceed.hpp
-  libceed/diffusion.hpp
-  libceed/mass.hpp
  linearform.hpp
  lininteg.hpp
  multigrid.hpp
@@ -147,6 +135,17 @@ if (MFEM_USE_MPI)
    prestriction.hpp)
 endif()

+if (MFEM_USE_CEED)
+   list(APPEND SRCS
+     libceed/ceed.cpp
+     libceed/diffusion.cpp
+     libceed/mass.cpp)
+   list(APPEND HDRS
+     libceed/ceed.hpp
+     libceed/diffusion.hpp
+     libceed/mass.hpp)
+endif()
+
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)

@@ -133,7 +133,8 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
         ext = new PABilinearFormExtension(this);
         break;
      case AssemblyLevel::NONE:
-         ext = new MFBilinearFormExtension(this);
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         // ext = new MFBilinearFormExtension(this);
         break;
      default:
         mfem_error("Unknown assembly level");
@@ -157,8 +157,7 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
+       - AssemblyLevel::FULL  (default)
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE
@@ -416,7 +415,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
                         OpType &A, Vector &X, Vector &B,
@@ -438,7 +439,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
   {
@@ -756,7 +759,7 @@ public:
   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }

-   /// Set the desired assembly level. The default is AssemblyLevel::LEGACYFULL.
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

@@ -858,7 +861,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
                                    const Array<int> &test_tdof_list, OpType &A)
@@ -888,7 +893,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
                                    const Array<int> &test_tdof_list,
@@ -36,206 +36,6 @@ const Operator *BilinearFormExtension::GetRestriction() const
   return a->GetRestriction();
 }

-// Data and methods for partially-assembled bilinear forms
-MFBilinearFormExtension::MFBilinearFormExtension(BilinearForm *form)
-   : BilinearFormExtension(form),
-     trialFes(a->FESpace()),
-     testFes(a->FESpace())
-{
-   elem_restrict = NULL;
-   int_face_restrict_lex = NULL;
-   bdr_face_restrict_lex = NULL;
-}
-
-void MFBilinearFormExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssembleMF(*a->FESpace());
-   }
-}
-
-void MFBilinearFormExtension::AssembleDiagonal(Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
-   {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalMF(localY);
-      }
-      const ElementRestriction* H1elem_restrict =
-         dynamic_cast<const ElementRestriction*>(elem_restrict);
-      if (H1elem_restrict)
-      {
-         H1elem_restrict->MultTransposeUnsigned(localY, y);
-      }
-      else
-      {
-         elem_restrict->MultTranspose(localY, y);
-      }
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalMF(y);
-      }
-   }
-}
-
-void MFBilinearFormExtension::Update()
-{
-   FiniteElementSpace *fes = a->FESpace();
-   height = width = fes->GetVSize();
-   trialFes = fes;
-   testFes = fes;
-
-   elem_restrict = nullptr;
-   int_face_restrict_lex = nullptr;
-   bdr_face_restrict_lex = nullptr;
-}
-
-void MFBilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                               OperatorHandle &A)
-{
-   Operator *oper;
-   Operator::FormSystemOperator(ess_tdof_list, oper);
-   A.Reset(oper); // A will own oper
-}
-
-void MFBilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                               Vector &x, Vector &b,
-                                               OperatorHandle &A,
-                                               Vector &X, Vector &B,
-                                               int copy_interior)
-{
-   Operator *oper;
-   Operator::FormLinearSystem(ess_tdof_list, x, b, oper, X, B, copy_interior);
-   A.Reset(oper); // A will own oper
-}
-
-void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultMF(x, y);
-      }
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultMF(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-   if (elem_restrict)
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposeMF(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true);
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposeMF(x, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}

 // Data and methods for partially-assembled bilinear forms
 PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
@@ -510,12 +310,13 @@ void EABilinearFormExtension::Assemble()

   ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
   ea_data.UseDevice(true);
+   ea_data = 0.0;

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
   const int integratorCount = integrators.Size();
   for (int i = 0; i < integratorCount; ++i)
   {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);
+      integrators[i]->AssembleEA(*a->FESpace(), ea_data);
   }

   faceDofs = trialFes ->
@@ -532,13 +333,14 @@ void EABilinearFormExtension::Assemble()
      nf_int = trialFes->GetNFbyType(FaceType::Interior);
      ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
      ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
+      ea_data_int = 0.0;
+      ea_data_ext = 0.0;
   }
   for (int i = 0; i < intFaceIntegratorCount; ++i)
   {
      intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
                                                     ea_data_int,
-                                                     ea_data_ext,
-                                                     i);
+                                                     ea_data_ext);
   }

   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
@@ -551,7 +353,7 @@ void EABilinearFormExtension::Assemble()
   }
   for (int i = 0; i < boundFaceIntegratorCount; ++i)
   {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr,i);
+      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
   }

   if (factorize_face_terms && int_face_restrict_lex)
@@ -130,31 +130,24 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
 };

-/// Data and methods for matrix-free bilinear forms
+/// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
 class MFBilinearFormExtension : public BilinearFormExtension
 {
-protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
-   mutable Vector localX, localY;
-   mutable Vector faceIntX, faceIntY;
-   mutable Vector faceBdrX, faceBdrY;
-   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
-
 public:
-   MFBilinearFormExtension(BilinearForm *form);
+   MFBilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }

-   void Assemble();
-   void AssembleDiagonal(Vector &diag) const;
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
   void FormLinearSystem(const Array<int> &ess_tdof_list,
                         Vector &x, Vector &b,
                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-   void Update();
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~MFBilinearFormExtension() {}
 };

 /// Class extending the MixedBilinearForm class to support different AssemblyLevels.
@@ -52,8 +52,7 @@ void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 }

 void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                        Vector &emat,
-                                        const bool add)
+                                        Vector &emat)
 {
   mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
               "   is not implemented for this class.");
@@ -62,8 +61,7 @@ void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                     &fes,
                                                     Vector &ea_data_int,
-                                                     Vector &ea_data_ext,
-                                                     const bool add)
+                                                     Vector &ea_data_ext)
 {
   mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
               "   is not implemented for this class.");
@@ -71,8 +69,7 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace

 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                     &fes,
-                                                     Vector &ea_data_bdr,
-                                                     const bool add)
+                                                     Vector &ea_data_bdr)
 {
   mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
               "   is not implemented for this class.");
@@ -96,30 +93,6 @@ void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultMF(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::AddMultMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultTransposeMF(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
-               "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -157,22 +130,8 @@ void BilinearFormIntegrator::AssembleElementVector(
   const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
   Vector &elvect)
 {
-   // Note: This default implementation is general but not efficient
-   DenseMatrix elmat;
-   AssembleElementMatrix(el, Tr, elmat);
-   elvect.SetSize(elmat.Height());
-   elmat.Mult(elfun, elvect);
-}
-
-void BilinearFormIntegrator::AssembleFaceVector(
-   const FiniteElement &el1, const FiniteElement &el2,
-   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
-{
-   // Note: This default implementation is general but not efficient
-   DenseMatrix elmat;
-   AssembleFaceMatrix(el1, el2, Tr, elmat);
-   elvect.SetSize(elmat.Height());
-   elmat.Mult(elfun, elvect);
+   mfem_error("BilinearFormIntegrator::AssembleElementVector\n"
+              "   is not implemented for this class.");
 }


@@ -554,12 +513,10 @@ void DiffusionIntegrator::AssembleElementMatrix

 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), dshapedxt(nd,spaceDim), invdfdx(dim,spaceDim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd,dim);
   dshapedxt.SetSize(nd,spaceDim);
   invdfdx.SetSize(dim,spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   elmat.SetSize(nd);

@@ -577,20 +534,7 @@ void DiffusionIntegrator::AssembleElementMatrix
      // AdjugateJacobian = / adj(J),         if J is square
      //                    \ adj(J^t.J).J^t, otherwise
      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
-      if (MQ)
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(dshapedxt, invdfdx, dshape);
-         AddMultABt(dshape, dshapedxt, elmat);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(dshapedxt, D, elmat);
-      }
-      else
+      if (!MQ)
      {
         if (Q)
         {
@@ -598,6 +542,13 @@ void DiffusionIntegrator::AssembleElementMatrix
         }
         AddMult_a_AAt(w, dshapedxt, elmat);
      }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(dshapedxt, invdfdx, dshape);
+         AddMultABt(dshape, dshapedxt, elmat);
+      }
   }
 }

@@ -616,14 +567,12 @@ void DiffusionIntegrator::AssembleElementMatrix2(
   DenseMatrix dshape(tr_nd, dim), dshapedxt(tr_nd, spaceDim);
   DenseMatrix te_dshape(te_nd, dim), te_dshapedxt(te_nd, spaceDim);
   DenseMatrix invdfdx(dim, spaceDim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(tr_nd, dim);
   dshapedxt.SetSize(tr_nd, spaceDim);
   te_dshape.SetSize(te_nd, dim);
   te_dshapedxt.SetSize(te_nd, spaceDim);
   invdfdx.SetSize(dim, spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   elmat.SetSize(te_nd, tr_nd);

@@ -643,20 +592,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
      Mult(dshape, invdfdx, dshapedxt);
      Mult(te_dshape, invdfdx, te_dshapedxt);
      // invdfdx, dshape, and te_dshape no longer needed
-      if (MQ)
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(te_dshapedxt, invdfdx, te_dshape);
-         AddMultABt(te_dshape, dshapedxt, elmat);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(dshapedxt, D, elmat);
-      }
-      else
+      if (!MQ)
      {
         if (Q)
         {
@@ -665,6 +601,13 @@ void DiffusionIntegrator::AssembleElementMatrix2(
         dshapedxt *= w;
         AddMultABt(te_dshapedxt, dshapedxt, elmat);
      }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(te_dshapedxt, invdfdx, te_dshape);
+         AddMultABt(te_dshape, dshapedxt, elmat);
+      }
   }
 }

@@ -676,19 +619,12 @@ void DiffusionIntegrator::AssembleElementVector(
   int dim = el.GetDim();
   double w;

-   if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "Unexpected dimension for VectorCoefficient");
-   }
-
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), invdfdx(dim), mq(dim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd,dim);
   invdfdx.SetSize(dim);
   mq.SetSize(dim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   vec.SetSize(dim);
   pointflux.SetSize(dim);
@@ -707,7 +643,7 @@ void DiffusionIntegrator::AssembleElementVector(
      CalcAdjugate(Tr.Jacobian(), invdfdx); // invdfdx = adj(J)
      w = ip.weight / Tr.Weight();

-      if (!MQ && !VQ)
+      if (!MQ)
      {
         dshape.MultTranspose(elfun, vec);
         invdfdx.MultTranspose(vec, pointflux);
@@ -718,21 +654,11 @@ void DiffusionIntegrator::AssembleElementVector(
      }
      else
      {
+
         dshape.MultTranspose(elfun, pointflux);
         invdfdx.MultTranspose(pointflux, vec);
-         if (MQ)
-         {
-            MQ->Eval(mq, Tr, ip);
-            mq.Mult(vec, pointflux);
-         }
-         else
-         {
-            VQ->Eval(D, Tr, ip);
-            for (int j=0; j<dim; ++j)
-            {
-               pointflux[j] *= D[j];
-            }
-         }
+         MQ->Eval(mq, Tr, ip);
+         mq.Mult(vec, pointflux);
      }
      pointflux *= w;
      invdfdx.Mult(pointflux, vec);
@@ -1596,7 +1522,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
   double w;

 #ifdef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
   curlshape.SetSize(nd,dimc);
@@ -1604,7 +1529,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
   elmat.SetSize(nd);
   if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }

   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
@@ -1648,12 +1572,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
         Mult(curlshape_dFt, M, curlshape);
         AddMultABt(curlshape, curlshape_dFt, elmat);
      }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(curlshape_dFt, D, elmat);
-      }
      else if (Q)
      {
         w *= Q->Eval(Trans, ip);
@@ -2265,7 +2183,16 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      ir = &DiffusionIntegrator::GetRule(el,el);
+      // integrand is rational function if det(J) is not constant
+      int order = 2 * Trans.OrderGrad(&el); // order of the numerator
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
   }

   elmat = 0.0;
@@ -2316,7 +2243,11 @@ void VectorDiffusionIntegrator::AssembleElementVector(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      ir = &DiffusionIntegrator::GetRule(el,el);
+      // integrand is rational function if det(J) is not constant
+      int order = 2 * Tr.OrderGrad(&el); // order of the numerator
+      ir = (el.Space() == FunctionSpace::rQk) ?
+           &RefinedIntRules.Get(el.GetGeomType(), order) :
+           &IntRules.Get(el.GetGeomType(), order);
   }

   elvect = 0.0;
@@ -86,47 +86,20 @@ public:
   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;

   /// Method defining element assembly.
-   /** The result of the element assembly is added to the @a emat Vector if
-       @a add is true. Otherwise, if @a add is false, we set @a emat. */
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add = true);
+   /** The result of the element assembly is added and stored in the @a emat
+       Vector. */
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
   /** Used with BilinearFormIntegrators that have different spaces. */
   // virtual void AssembleEA(const FiniteElementSpace &trial_fes,
   //                         const FiniteElementSpace &test_fes,
   //                         Vector &emat);

-   /// Method defining matrix-free assembly.
-   /** The result of fully matrix-free assembly is stored internally so that it
-       can be used later in the methods AddMultMF() and AddMultTransposeMF(). */
-   virtual void AssembleMF(const FiniteElementSpace &fes);
-
-   /** Perform the action of integrator on the input @a x and add the result to
-       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
-       the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssembleMF() has been
-       called. */
-   virtual void AddMultMF(const Vector &x, Vector &y) const;
-
-   /** Perform the transpose action of integrator on the input @a x and add the
-       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
-       represent the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultTransposeMF(const Vector &x, Vector &y) const;
-
-   /// Assemble diagonal and add it to Vector @a diag.
-   virtual void AssembleDiagonalMF(Vector &diag);
-
   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add = true);
+                                        Vector &ea_data_ext);

   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add = true);
+                                        Vector &ea_data_bdr);

   /// Given a particular Finite Element computes the element matrix elmat.
   virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -155,22 +128,11 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

-   /// @brief Perform the local action of the BilinearFormIntegrator.
-   /// Note that the default implementation in the base class is general but not
-   /// efficient.
+   /// Perform the local action of the BilinearFormIntegrator
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &Tr,
                                      const Vector &elfun, Vector &elvect);

-   /// @brief Perform the local action of the BilinearFormIntegrator resulting
-   /// from a face integral term.
-   /// Note that the default implementation in the base class is general but not
-   /// efficient.
-   virtual void AssembleFaceVector(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Tr,
-                                   const Vector &elfun, Vector &elvect);
-
   virtual void AssembleElementGrad(const FiniteElement &el,
                                    ElementTransformation &Tr,
                                    const Vector &elfun, DenseMatrix &elmat)
@@ -300,17 +262,14 @@ public:
      bfi->AddMultTransposePA(x, y);
   }

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add);
+                                        Vector &ea_data_ext);

   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add);
+                                        Vector &ea_data_bdr);

   virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };
@@ -1898,7 +1857,6 @@ class DiffusionIntegrator: public BilinearFormIntegrator
 {
 protected:
   Coefficient *Q;
-   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;

 private:
@@ -1906,7 +1864,6 @@ private:
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix dshape, dshapedxt, invdfdx, mq;
   DenseMatrix te_dshape, te_dshapedxt;
-   Vector D;
 #endif

   // PA extension
@@ -1915,30 +1872,54 @@ private:
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, dofs1D, quad1D;
   Vector pa_data;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
+
+#ifdef MFEM_USE_CEED
   // CEED extension
   CeedData* ceedDataPtr;
+#endif

 public:
   /// Construct a diffusion integrator with coefficient Q = 1
   DiffusionIntegrator()
-      : Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+   {
+      Q = NULL;
+      MQ = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }

   /// Construct a diffusion integrator with a scalar coefficient q
   DiffusionIntegrator(Coefficient &q)
-      : Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
-
-   /// Construct a diffusion integrator with a vector coefficient q
-   DiffusionIntegrator(VectorCoefficient &q)
-      : Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : Q(&q)
+   {
+      MQ = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }

   /// Construct a diffusion integrator with a matrix coefficient q
   DiffusionIntegrator(MatrixCoefficient &q)
-      : Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL), ceedDataPtr(NULL) { }
+      : MQ(&q)
+   {
+      Q = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }

   virtual ~DiffusionIntegrator()
   {
+#ifdef MFEM_USE_CEED
      delete ceedDataPtr;
+#endif
   }

   /** Given a particular Finite Element computes the element stiffness matrix
@@ -1969,23 +1950,18 @@ public:

   using BilinearFormIntegrator::AssemblePA;

-   virtual void AssembleMF(const FiniteElementSpace &fes);
-
   virtual void AssemblePA(const FiniteElementSpace &fes);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AssembleDiagonalPA(Vector &diag);

-   virtual void AssembleDiagonalMF(Vector &diag);
-
-   virtual void AddMultMF(const Vector&, Vector&) const;
-
   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe);
+
+   void SetupPA(const FiniteElementSpace &fes);
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2003,22 +1979,39 @@ protected:
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;

+#ifdef MFEM_USE_CEED
   // CEED extension
   CeedData* ceedDataPtr;
+#endif

 public:
   MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL),
-        ceedDataPtr(NULL) { }
+      : BilinearFormIntegrator(ir)
+   {
+      Q = NULL;
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }

   /// Construct a mass integrator with coefficient q
   MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL),
-        ceedDataPtr(NULL) { }
+      : BilinearFormIntegrator(ir), Q(&q)
+   {
+      maps = NULL;
+      geom = NULL;
+#ifdef MFEM_USE_CEED
+      ceedDataPtr = NULL;
+#endif
+   }

   virtual ~MassIntegrator()
   {
+#ifdef MFEM_USE_CEED
      delete ceedDataPtr;
+#endif
   }
   /** Given a particular Finite Element computes the element mass matrix
       elmat. */
@@ -2032,19 +2025,12 @@ public:

   using BilinearFormIntegrator::AssemblePA;

-   virtual void AssembleMF(const FiniteElementSpace &fes);
-
   virtual void AssemblePA(const FiniteElementSpace &fes);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AssembleDiagonalPA(Vector &diag);

-   virtual void AssembleDiagonalMF(Vector &diag);
-
-   virtual void AddMultMF(const Vector&, Vector&) const;
-
   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
@@ -2097,8 +2083,7 @@ public:

   virtual void AssemblePA(const FiniteElementSpace&);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AddMultPA(const Vector&, Vector&) const;

@@ -2151,34 +2136,24 @@ protected:
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;

-   // CEED extension
-   CeedData* ceedDataPtr;
-
 public:
   /// Construct an integrator with coefficient 1.0
   VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL), ceedDataPtr(NULL) { }
+      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
   /** Construct an integrator with scalar coefficient q.  If possible, save
       memory by using a scalar integrator since the resulting matrix is block
       diagonal with the same diagonal block repeated. */
   VectorMassIntegrator(Coefficient &q, int qo = 0)
-      : vdim(-1), Q_order(qo), Q(&q), VQ(NULL), MQ(NULL), ceedDataPtr(NULL) { }
+      : vdim(-1), Q(&q) { VQ = NULL; MQ = NULL; Q_order = qo; }
   VectorMassIntegrator(Coefficient &q, const IntegrationRule *ir)
-      : BilinearFormIntegrator(ir), vdim(-1), Q_order(0), Q(&q), VQ(NULL),
-        MQ(NULL), ceedDataPtr(NULL) { }
+      : BilinearFormIntegrator(ir), vdim(-1), Q(&q)
+   { VQ = NULL; MQ = NULL; Q_order = 0; }
   /// Construct an integrator with diagonal coefficient q
   VectorMassIntegrator(VectorCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(&q), MQ(NULL),
-        ceedDataPtr(NULL) { }
+      : vdim(q.GetVDim()), VQ(&q) { Q = NULL; MQ = NULL; Q_order = qo; }
   /// Construct an integrator with matrix coefficient q
   VectorMassIntegrator(MatrixCoefficient &q, int qo = 0)
-      : vdim(q.GetVDim()), Q_order(qo), Q(NULL), VQ(NULL), MQ(&q),
-        ceedDataPtr(NULL) { }
-
-   virtual ~VectorMassIntegrator()
-   {
-      delete ceedDataPtr;
-   }
+      : vdim(q.GetVDim()), MQ(&q) { Q = NULL; VQ = NULL; Q_order = qo; }

   int GetVDim() const { return vdim; }
   void SetVDim(int vdim) { this->vdim = vdim; }
@@ -2192,11 +2167,8 @@ public:
                                       DenseMatrix &elmat);
   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssembleMF(const FiniteElementSpace &fes);
   virtual void AssembleDiagonalPA(Vector &diag);
-   virtual void AssembleDiagonalMF(Vector &diag);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultMF(const Vector &x, Vector &y) const;
 };


@@ -2328,14 +2300,12 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
   Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape, curlshape_dFt, M;
   DenseMatrix vshape, projcurl;
 #endif

 protected:
   Coefficient *Q;
-   VectorCoefficient *DQ;
   MatrixCoefficient *MQ;

   // PA extension
@@ -2344,17 +2314,12 @@ protected:
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
-   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
+   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2560,23 +2525,13 @@ protected:
   int dim, sdim, ne, dofs1D, quad1D;
   Vector pa_data;

-   // CEED extension
-   CeedData* ceedDataPtr;
-
 private:
   DenseMatrix dshape, dshapedxt, pelmat;
   DenseMatrix Jinv, gshape;

 public:
-   VectorDiffusionIntegrator()
-      : Q(NULL), ceedDataPtr(NULL) { }
-   VectorDiffusionIntegrator(Coefficient &q)
-      : Q(&q), ceedDataPtr(NULL) { }
-
-   virtual ~VectorDiffusionIntegrator()
-   {
-      delete ceedDataPtr;
-   }
+   VectorDiffusionIntegrator() { Q = NULL; }
+   VectorDiffusionIntegrator(Coefficient &q) { Q = &q; }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -2586,11 +2541,8 @@ public:
                                      const Vector &elfun, Vector &elvect);
   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssembleMF(const FiniteElementSpace &fes);
   virtual void AssembleDiagonalPA(Vector &diag);
-   virtual void AssembleDiagonalMF(Vector &diag);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultMF(const Vector &x, Vector &y) const;
 };

 /** Integrator for the linear elasticity form:
@@ -2701,12 +2653,10 @@ public:

   virtual void AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add);
+                                        Vector &ea_data_ext);

   virtual void AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add);
+                                        Vector &ea_data_bdr);

   static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                         FaceElementTransformations &T);
@@ -3061,17 +3011,6 @@ protected:
   VectorCoefficient *VQ;
 };

-
-
-// PA Diffusion Assemble 2D kernel
-template<const int T_SDIM>
-void PADiffusionSetup2D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &c,
-                        Vector &d);
-
 }
+
 #endif
@@ -22,7 +22,6 @@ static void EAConvectionAssemble1D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -55,14 +54,7 @@ static void EAConvectionAssemble1D(const int NE,
            {
               val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
            }
-            if (add)
-            {
-               A(i1, j1, e) += val;
-            }
-            else
-            {
-               A(i1, j1, e) = val;
-            }
+            A(i1, j1, e) += val;
         }
      }
   });
@@ -74,7 +66,6 @@ static void EAConvectionAssemble2D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -130,14 +121,7 @@ static void EAConvectionAssemble2D(const int NE,
                               * r_B[k1][j1]* r_B[k2][j2];
                     }
                  }
-                  if (add)
-                  {
-                     A(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     A(i1, i2, j1, j2, e) = val;
-                  }
+                  A(i1, i2, j1, j2, e) += val;
               }
            }
         }
@@ -151,7 +135,6 @@ static void EAConvectionAssemble3D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -208,14 +191,7 @@ static void EAConvectionAssemble3D(const int NE,
                              }
                           }
                        }
-                        if (add)
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        A(i1, i2, i3, j1, j2, j3, e) += val;
                     }
                  }
               }
@@ -226,8 +202,7 @@ static void EAConvectionAssemble3D(const int NE,
 }

 void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                      Vector &ea_data,
-                                      const bool add)
+                                      Vector &ea_data)
 {
   AssemblePA(fes);
   const int ne = fes.GetMesh()->GetNE();
@@ -237,47 +212,44 @@ void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
+         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
+         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
+         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
+         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
+         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
+         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -806,16 +806,16 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   {
      vel.SetSize(dim * nq * ne);
      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      DenseMatrix Q_ir;
+      Vector Vq(dim);
      for (int e = 0; e < ne; ++e)
      {
         ElementTransformation& T = *fes.GetElementTransformation(e);
-         Q->Eval(Q_ir, T, *ir);
         for (int q = 0; q < nq; ++q)
         {
+            Q->Eval(Vq, T, ir->IntPoint(q));
            for (int i = 0; i < dim; ++i)
            {
-               C(i,q,e) = Q_ir(i,q);
+               C(i,q,e) = Vq(i);
            }
         }
      }
@@ -20,8 +20,7 @@ static void EADGTraceAssemble1DInt(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const bool add)
+                                   Vector &eadata_ext)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -33,41 +32,23 @@ static void EADGTraceAssemble1DInt(const int NF,
      val_ext10 = D(1, 0, f);
      val_ext01 = D(0, 1, f);
      val_int1  = D(1, 1, f);
-      if (add)
-      {
-         A_int(0, f) += val_int0;
-         A_int(1, f) += val_int1;
-         A_ext(0, f) += val_ext01;
-         A_ext(1, f) += val_ext10;
-      }
-      else
-      {
-         A_int(0, f) = val_int0;
-         A_int(1, f) = val_int1;
-         A_ext(0, f) = val_ext01;
-         A_ext(1, f) = val_ext10;
-      }
+      A_int(0, f) += val_int0;
+      A_int(1, f) += val_int1;
+      A_ext(0, f) += val_ext01;
+      A_ext(1, f) += val_ext10;
   });
 }

 static void EADGTraceAssemble1DBdr(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const bool add)
+                                   Vector &eadata_bdr)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
   MFEM_FORALL(f, NF,
   {
-      if (add)
-      {
-         A_bdr(f) += D(0, 0, f);
-      }
-      else
-      {
-         A_bdr(f) = D(0, 0, f);
-      }
+      A_bdr(f) += D(0, 0, f);
   });
 }

@@ -77,7 +58,6 @@ static void EADGTraceAssemble2DInt(const int NF,
                                   const Vector &padata,
                                   Vector &eadata_int,
                                   Vector &eadata_ext,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -108,20 +88,10 @@ static void EADGTraceAssemble2DInt(const int NF,
               val_ext10 += B(k1,i1) * B(k1,j1) * D(k1, 1, 0, f);
               val_int1  += B(k1,i1) * B(k1,j1) * D(k1, 1, 1, f);
            }
-            if (add)
-            {
-               A_int(i1, j1, 0, f) += val_int0;
-               A_int(i1, j1, 1, f) += val_int1;
-               A_ext(i1, j1, 0, f) += val_ext01;
-               A_ext(i1, j1, 1, f) += val_ext10;
-            }
-            else
-            {
-               A_int(i1, j1, 0, f) = val_int0;
-               A_int(i1, j1, 1, f) = val_int1;
-               A_ext(i1, j1, 0, f) = val_ext01;
-               A_ext(i1, j1, 1, f) = val_ext10;
-            }
+            A_int(i1, j1, 0, f) += val_int0;
+            A_int(i1, j1, 1, f) += val_int1;
+            A_ext(i1, j1, 0, f) += val_ext01;
+            A_ext(i1, j1, 1, f) += val_ext10;
         }
      }
   });
@@ -132,7 +102,6 @@ static void EADGTraceAssemble2DBdr(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
                                   Vector &eadata_bdr,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -156,14 +125,7 @@ static void EADGTraceAssemble2DBdr(const int NF,
            {
               val_bdr  += B(k1,i1) * B(k1,j1) * D(k1, 0, 0, f);
            }
-            if (add)
-            {
-               A_bdr(i1, j1, f) += val_bdr;
-            }
-            else
-            {
-               A_bdr(i1, j1, f) = val_bdr;
-            }
+            A_bdr(i1, j1, f) += val_bdr;
         }
      }
   });
@@ -175,7 +137,6 @@ static void EADGTraceAssemble3DInt(const int NF,
                                   const Vector &padata,
                                   Vector &eadata_int,
                                   Vector &eadata_ext,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -246,20 +207,10 @@ static void EADGTraceAssemble3DInt(const int NF,
                                    * s_D[k1][k2][1][0];
                     }
                  }
-                  if (add)
-                  {
-                     A_int(i1, i2, j1, j2, 0, f) += val_int0;
-                     A_int(i1, i2, j1, j2, 1, f) += val_int1;
-                     A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
-                     A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
-                  }
-                  else
-                  {
-                     A_int(i1, i2, j1, j2, 0, f) = val_int0;
-                     A_int(i1, i2, j1, j2, 1, f) = val_int1;
-                     A_ext(i1, i2, j1, j2, 0, f) = val_ext01;
-                     A_ext(i1, i2, j1, j2, 1, f) = val_ext10;
-                  }
+                  A_int(i1, i2, j1, j2, 0, f) += val_int0;
+                  A_int(i1, i2, j1, j2, 1, f) += val_int1;
+                  A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
+                  A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
               }
            }
         }
@@ -272,7 +223,6 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
                                   Vector &eadata_bdr,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -330,14 +280,7 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                   * s_D[k1][k2][0][0];
                     }
                  }
-                  if (add)
-                  {
-                     A_bdr(i1, i2, j1, j2, f) += val_bdr;
-                  }
-                  else
-                  {
-                     A_bdr(i1, i2, j1, j2, f) = val_bdr;
-                  }
+                  A_bdr(i1, i2, j1, j2, f) += val_bdr;
               }
            }
         }
@@ -347,8 +290,7 @@ static void EADGTraceAssemble3DBdr(const int NF,

 void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                Vector &ea_data_int,
-                                                Vector &ea_data_ext,
-                                                const bool add)
+                                                Vector &ea_data_ext)
 {
   SetupPA(fes, FaceType::Interior);
   nf = fes.GetNFbyType(FaceType::Interior);
@@ -356,7 +298,7 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
   const Array<double> &B = maps->B;
   if (dim == 1)
   {
-      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext,add);
+      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext);
   }
   else if (dim == 2)
   {
@@ -364,31 +306,31 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
      {
         case 0x22:
            return EADGTraceAssemble2DInt<2,2>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x33:
            return EADGTraceAssemble2DInt<3,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x44:
            return EADGTraceAssemble2DInt<4,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x55:
            return EADGTraceAssemble2DInt<5,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x66:
            return EADGTraceAssemble2DInt<6,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x77:
            return EADGTraceAssemble2DInt<7,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x88:
            return EADGTraceAssemble2DInt<8,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x99:
            return EADGTraceAssemble2DInt<9,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         default:
            return EADGTraceAssemble2DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,add,dofs1D,quad1D);
+                                          ea_data_ext,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
@@ -397,36 +339,35 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
      {
         case 0x23:
            return EADGTraceAssemble3DInt<2,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x34:
            return EADGTraceAssemble3DInt<3,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x45:
            return EADGTraceAssemble3DInt<4,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x56:
            return EADGTraceAssemble3DInt<5,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x67:
            return EADGTraceAssemble3DInt<6,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x78:
            return EADGTraceAssemble3DInt<7,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x89:
            return EADGTraceAssemble3DInt<8,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         default:
            return EADGTraceAssemble3DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,add,dofs1D,quad1D);
+                                          ea_data_ext,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
 }

 void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                Vector &ea_data_bdr,
-                                                const bool add)
+                                                Vector &ea_data_bdr)
 {
   SetupPA(fes, FaceType::Boundary);
   nf = fes.GetNFbyType(FaceType::Boundary);
@@ -434,37 +375,37 @@ void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
   const Array<double> &B = maps->B;
   if (dim == 1)
   {
-      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr,add);
+      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr);
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr);
+         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr);
+         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr);
+         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr);
+         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr);
+         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr);
+         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr);
+         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr);
         default:
-            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
+            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr);
+         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr);
+         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr);
+         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr);
+         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr);
+         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr);
+         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr);
         default:
-            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
+            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -43,7 +43,7 @@ static void PADGTraceSetup2D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);

-   MFEM_FORALL(f, NF, // can be optimized with Q1D thread for NF blocks
+   MFEM_FORALL(f, NF,//can be optimized with Q1D thread for NF blocks
   {
      for (int q = 0; q < Q1D; ++q)
      {
@@ -85,7 +85,7 @@ static void PADGTraceSetup3D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);

-   MFEM_FORALL(f, NF, // can be optimized with Q1D*Q1D threads for NF blocks
+   MFEM_FORALL(f, NF,//can be optimized with Q1D*Q1D threads for NF blocks
   {
      for (int q1 = 0; q1 < Q1D; ++q1)
      {
@@ -156,6 +156,57 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
+   Vector r;
+   if (rho==nullptr)
+   {
+      r.SetSize(1);
+      r(0) = 1.0;
+   }
+   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
+   {
+      r.SetSize(1);
+      r(0) = c_rho->constant;
+   }
+   else if (QuadratureFunctionCoefficient* c_rho =
+               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
+   {
+      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      r.SetSize(nq * nf);
+      auto C = Reshape(r.HostWrite(), nq, nf);
+      int f_ind = 0;
+      for (int f = 0; f < fes.GetNF(); ++f)
+      {
+         int e1, e2;
+         int inf1, inf2;
+         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+         int face_id = inf1 / 64;
+         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
+             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         {
+            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
+            for (int q = 0; q < nq; ++q)
+            {
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               C(iq,f_ind) = rho->Eval(T, ir->IntPoint(q));
+            }
+            f_ind++;
+         }
+      }
+      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+   }
   Vector vel;
   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
                                        (u))
@@ -192,15 +243,12 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
             (type==FaceType::Boundary && e2<0 && inf2<0) )
         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
+            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
            for (int q = 0; q < nq; ++q)
            {
               // Convert to lexicographic ordering
               int iq = ToLexOrdering(dim, face_id, quad1D, q);
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               u->Eval(Vq, *T.Elem1, eip1);
+               u->Eval(Vq, T, ir->IntPoint(q));
               for (int i = 0; i < dim; ++i)
               {
                  C(i,iq,f_ind) = Vq(i);
@@ -211,80 +259,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      }
      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
   }
-   Vector r;
-   if (rho==nullptr)
-   {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
-   }
-   else if (QuadratureFunctionCoefficient* c_rho =
-               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
-   {
-      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      r.SetSize(nq * nf);
-      auto C_vel = Reshape(vel.HostRead(), dim, nq, nf);
-      auto n = Reshape(geom->normal.HostRead(), nq, dim, nf);
-      auto C = Reshape(r.HostWrite(), nq, nf);
-      int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
-      {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
-         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
-
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-               double r;
-
-               if (inf2 < 0)
-               {
-                  r = rho->Eval(*T.Elem1, eip1);
-               }
-               else
-               {
-                  double udotn = 0.0;
-                  for (int d=0; d<dim; ++d)
-                  {
-                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
-                  }
-                  if (udotn >= 0.0) { r = rho->Eval(*T.Elem2, eip2); }
-                  else { r = rho->Eval(*T.Elem1, eip1); }
-               }
-               C(iq,f_ind) = r;
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
-   }
   PADGTraceSetup(dim, dofs1D, quad1D, nf, ir->GetWeights(),
                  geom->detJ, geom->normal, r, vel,
                  alpha, beta, pa_data);
@@ -22,7 +22,6 @@ static void EADiffusionAssemble1D(const int NE,
                                  const Array<double> &g,
                                  const Vector &padata,
                                  Vector &eadata,
-                                  const bool add,
                                  const int d1d = 0,
                                  const int q1d = 0)
 {
@@ -54,14 +53,7 @@ static void EADiffusionAssemble1D(const int NE,
            {
               val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
            }
-            if (add)
-            {
-               A(i1, j1, e) += val;
-            }
-            else
-            {
-               A(i1, j1, e) = val;
-            }
+            A(i1, j1, e) += val;
         }
      }
   });
@@ -73,7 +65,6 @@ static void EADiffusionAssemble2D(const int NE,
                                  const Array<double> &g,
                                  const Vector &padata,
                                  Vector &eadata,
-                                  const bool add,
                                  const int d1d = 0,
                                  const int q1d = 0)
 {
@@ -129,14 +120,7 @@ static void EADiffusionAssemble2D(const int NE,
                               + gbi * D11 * gbj;
                     }
                  }
-                  if (add)
-                  {
-                     A(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     A(i1, i2, j1, j2, e) = val;
-                  }
+                  A(i1, i2, j1, j2, e) += val;
               }
            }
         }
@@ -150,7 +134,6 @@ static void EADiffusionAssemble3D(const int NE,
                                  const Array<double> &g,
                                  const Vector &padata,
                                  Vector &eadata,
-                                  const bool add,
                                  const int d1d = 0,
                                  const int q1d = 0)
 {
@@ -225,14 +208,7 @@ static void EADiffusionAssemble3D(const int NE,
                              }
                           }
                        }
-                        if (add)
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        A(i1, i2, i3, j1, j2, j3, e) += val;
                     }
                  }
               }
@@ -243,8 +219,7 @@ static void EADiffusionAssemble3D(const int NE,
 }

 void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data,
-                                     const bool add)
+                                     Vector &ea_data)
 {
   AssemblePA(fes);
   const int ne = fes.GetMesh()->GetNE();
@@ -254,47 +229,44 @@ void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
+         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
+         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
+         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
+         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
+         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
+         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -1,70 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void DiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFDiffusionAssemble(fes, *ir, *ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: DiffusionIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void DiffusionIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: DiffusionIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-void DiffusionIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: DiffusionIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-}
@@ -80,24 +80,28 @@ static void OccaPADiffusionSetup3D(const int D1D,
 }
 #endif // MFEM_USE_OCCA

+// PA Diffusion Assemble 2D kernel
+template<const int T_SDIM>
+static void PADiffusionSetup2D(const int Q1D,
+                               const int NE,
+                               const Array<double> &w,
+                               const Vector &j,
+                               const Vector &c,
+                               Vector &d);
 template<>
 void PADiffusionSetup2D<2>(const int Q1D,
-                           const int coeffDim,
                           const int NE,
                           const Array<double> &w,
                           const Vector &j,
                           const Vector &c,
                           Vector &d)
 {
-   const bool symmetric = (coeffDim != 4);
   const bool const_c = c.Size() == 1;
-   MFEM_VERIFY(coeffDim < 3 ||
-               !const_c, "Constant matrix coefficient not supported");
   const auto W = Reshape(w.Read(), Q1D,Q1D);
   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
-                  Reshape(c.Read(), coeffDim,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, symmetric ? 3 : 4, NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
   {
      MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -108,39 +112,11 @@ void PADiffusionSetup2D<2>(const int Q1D,
            const double J21 = J(qx,qy,1,0,e);
            const double J12 = J(qx,qy,0,1,e);
            const double J22 = J(qx,qy,1,1,e);
-            const double w_detJ = W(qx,qy) / ((J11*J22)-(J21*J12));
-            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
-            {
-               // First compute entries of R = MJ^{-T}, without det J factor.
-               const double M11 = C(0,qx,qy,e);
-               const double M12 = C(1,qx,qy,e);
-               const double M21 = symmetric ? M12 : C(2,qx,qy,e);
-               const double M22 = symmetric ? C(2,qx,qy,e) : C(3,qx,qy,e);
-               const double R11 = M11*J22 - M12*J12;
-               const double R21 = M21*J22 - M22*J12;
-               const double R12 = -M11*J21 + M12*J11;
-               const double R22 = -M21*J21 + M22*J11;
-
-               // Now set y to J^{-1}R.
-               D(qx,qy,0,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               D(qx,qy,1,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               D(qx,qy,2,e) = w_detJ * (symmetric ? (-J21*R12 + J11*R22) :
-               (J22*R12 - J12*R22)); // 2,2 or 1,2
-               if (!symmetric)
-               {
-                  D(qx,qy,3,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-               }
-            }
-            else // Vector or scalar coefficient
-            {
-               const double C1 = const_c ? C(0,0,0,0) : C(0,qx,qy,e);
-               const double C2 = const_c ? C(0,0,0,0) :
-                                 (coeffDim == 2 ? C(1,qx,qy,e) : C(0,qx,qy,e));
-
-               D(qx,qy,0,e) =  w_detJ * (C2*J12*J12 + C1*J22*J22); // 1,1
-               D(qx,qy,1,e) = -w_detJ * (C2*J12*J11 + C1*J22*J21); // 1,2
-               D(qx,qy,2,e) =  w_detJ * (C2*J11*J11 + C1*J21*J21); // 2,2
-            }
+            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
+            const double c_detJ = W(qx,qy) * coeff / ((J11*J22)-(J21*J12));
+            D(qx,qy,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+            D(qx,qy,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+            D(qx,qy,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
         }
      }
   });
@@ -149,14 +125,12 @@ void PADiffusionSetup2D<2>(const int Q1D,
 // PA Diffusion Assemble 2D kernel with 3D node coords
 template<>
 void PADiffusionSetup2D<3>(const int Q1D,
-                           const int coeffDim,
                           const int NE,
                           const Array<double> &w,
                           const Vector &j,
                           const Vector &c,
                           Vector &d)
 {
-   MFEM_VERIFY(coeffDim == 1, "Matrix and vector coefficients not supported");
   constexpr int DIM = 2;
   constexpr int SDIM = 3;
   const bool const_c = c.Size() == 1;
@@ -193,23 +167,19 @@ void PADiffusionSetup2D<3>(const int Q1D,
 }

 // PA Diffusion Assemble 3D kernel
-void PADiffusionSetup3D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &c,
-                        Vector &d)
+static void PADiffusionSetup3D(const int Q1D,
+                               const int NE,
+                               const Array<double> &w,
+                               const Vector &j,
+                               const Vector &c,
+                               Vector &d)
 {
-   const bool symmetric = (coeffDim != 9);
   const bool const_c = c.Size() == 1;
-   MFEM_VERIFY(coeffDim < 6 ||
-               !const_c, "Constant matrix coefficient not supported");
   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1,1) :
-                  Reshape(c.Read(), coeffDim,Q1D,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, symmetric ? 6 : 9, NE);
+   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
+                  Reshape(c.Read(), Q1D,Q1D,Q1D,NE);
+   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, 6, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
      MFEM_FOREACH_THREAD(qx,x,Q1D)
@@ -230,7 +200,8 @@ void PADiffusionSetup3D(const int Q1D,
               const double detJ = J11 * (J22 * J33 - J32 * J23) -
               /* */               J21 * (J12 * J33 - J32 * J13) +
               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double w_detJ = W(qx,qy,qz) / detJ;
+               const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
+               const double c_detJ = W(qx,qy,qz) * coeff / detJ;
               // adj(J)
               const double A11 = (J22 * J33) - (J23 * J32);
               const double A12 = (J32 * J13) - (J12 * J33);
@@ -241,69 +212,13 @@ void PADiffusionSetup3D(const int Q1D,
               const double A31 = (J21 * J32) - (J31 * J22);
               const double A32 = (J31 * J12) - (J11 * J32);
               const double A33 = (J11 * J22) - (J12 * J21);
-
-               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-               {
-                  // Compute entries of R = MJ^{-T} = M adj(J)^T, without det J.
-                  const double M11 = C(0, qx,qy,qz, e);
-                  const double M12 = C(1, qx,qy,qz, e);
-                  const double M13 = C(2, qx,qy,qz, e);
-                  const double M21 = (!symmetric) ? C(3, qx,qy,qz, e) : M12;
-                  const double M22 = (!symmetric) ? C(4, qx,qy,qz, e) : C(3, qx,qy,qz, e);
-                  const double M23 = (!symmetric) ? C(5, qx,qy,qz, e) : C(4, qx,qy,qz, e);
-                  const double M31 = (!symmetric) ? C(6, qx,qy,qz, e) : M13;
-                  const double M32 = (!symmetric) ? C(7, qx,qy,qz, e) : M23;
-                  const double M33 = (!symmetric) ? C(8, qx,qy,qz, e) : C(5, qx,qy,qz, e);
-
-                  const double R11 = M11*A11 + M12*A12 + M13*A13;
-                  const double R12 = M11*A21 + M12*A22 + M13*A23;
-                  const double R13 = M11*A31 + M12*A32 + M13*A33;
-                  const double R21 = M21*A11 + M22*A12 + M23*A13;
-                  const double R22 = M21*A21 + M22*A22 + M23*A23;
-                  const double R23 = M21*A31 + M22*A32 + M23*A33;
-                  const double R31 = M31*A11 + M32*A12 + M33*A13;
-                  const double R32 = M31*A21 + M32*A22 + M33*A23;
-                  const double R33 = M31*A31 + M32*A32 + M33*A33;
-
-                  // Now set D to J^{-1} R = adj(J) R
-                  D(qx,qy,qz,0,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-                  const double D12 = w_detJ * (A11*R12 + A12*R22 + A13*R32);
-                  D(qx,qy,qz,1,e) = D12; // 1,2
-                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-
-                  const double D21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
-                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
-                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
-
-                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
-
-                  D(qx,qy,qz,3,e) = symmetric ? D22 : D21; // 2,2 or 2,1
-                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
-                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3
-
-                  if (!symmetric)
-                  {
-                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-                     D(qx,qy,qz,8,e) = D33; // 3,3
-                  }
-               }
-               else  // Vector or scalar coefficient version
-               {
-                  const double C1 = const_c ? C(0,0,0,0,0) : C(0,qx,qy,qz,e);
-                  const double C2 = const_c ? C(0,0,0,0,0) :
-                                    (coeffDim == 3 ? C(1,qx,qy,qz,e) : C(0,qx,qy,qz,e));
-                  const double C3 = const_c ? C(0,0,0,0,0) :
-                                    (coeffDim == 3 ? C(2,qx,qy,qz,e) : C(0,qx,qy,qz,e));
-
-                  // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-                  D(qx,qy,qz,0,e) = w_detJ * (C1*A11*A11 + C2*A12*A12 + C3*A13*A13); // 1,1
-                  D(qx,qy,qz,1,e) = w_detJ * (C1*A11*A21 + C2*A12*A22 + C3*A13*A23); // 2,1
-                  D(qx,qy,qz,2,e) = w_detJ * (C1*A11*A31 + C2*A12*A32 + C3*A13*A33); // 3,1
-                  D(qx,qy,qz,3,e) = w_detJ * (C1*A21*A21 + C2*A22*A22 + C3*A23*A23); // 2,2
-                  D(qx,qy,qz,4,e) = w_detJ * (C1*A21*A31 + C2*A22*A32 + C3*A23*A33); // 3,2
-                  D(qx,qy,qz,5,e) = w_detJ * (C1*A31*A31 + C2*A32*A32 + C3*A33*A33); // 3,3
-               }
+               // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+               D(qx,qy,qz,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+               D(qx,qy,qz,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+               D(qx,qy,qz,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+               D(qx,qy,qz,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+               D(qx,qy,qz,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+               D(qx,qy,qz,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
            }
         }
      }
@@ -314,7 +229,6 @@ static void PADiffusionSetup(const int dim,
                             const int sdim,
                             const int D1D,
                             const int Q1D,
-                             const int coeffDim,
                             const int NE,
                             const Array<double> &W,
                             const Vector &J,
@@ -333,8 +247,8 @@ static void PADiffusionSetup(const int dim,
 #else
      MFEM_CONTRACT_VAR(D1D);
 #endif // MFEM_USE_OCCA
-      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, coeffDim, NE, W, J, C, D); }
-      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, coeffDim, NE, W, J, C, D); }
+      if (sdim == 2) { PADiffusionSetup2D<2>(Q1D, NE, W, J, C, D); }
+      if (sdim == 3) { PADiffusionSetup2D<3>(Q1D, NE, W, J, C, D); }
   }
   if (dim == 3)
   {
@@ -345,11 +259,11 @@ static void PADiffusionSetup(const int dim,
         return;
      }
 #endif // MFEM_USE_OCCA
-      PADiffusionSetup3D(Q1D, coeffDim, NE, W, J, C, D);
+      PADiffusionSetup3D(Q1D, NE, W, J, C, D);
   }
 }

-void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
 {
   // Assuming the same element type
   fespace = &fes;
@@ -357,13 +271,16 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (mesh->GetNE() == 0) { return; }
   const FiniteElement &el = *fes.GetFE(0);
   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPADiffusionAssemble(fes, *ir, *ceedDataPtr);
+      if (ceedDataPtr) { delete ceedDataPtr; }
+      CeedData* ptr = new CeedData();
+      ceedDataPtr = ptr;
+      InitCeedCoeff(Q, ptr);
+      return CeedPADiffusionAssemble(fes, *ir, *ptr);
   }
+#endif
   const int dims = el.GetDim();
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int nq = ir->GetNPoints();
@@ -374,80 +291,9 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   int coeffDim = 1;
+   pa_data.SetSize(symmDims * nq * ne, Device::GetDeviceMemoryType());
   Vector coeff;
-   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (MQ)
-   {
-      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-      const int MQsymmDim = MQ->GetWidth() * (MQ->GetWidth() + 1) / 2;
-
-      const int MQdim = MQ->IsSymmetric() ? MQsymmDim : MQfullDim;
-      coeffDim = MQdim;
-
-      coeff.SetSize(MQdim * nq * ne);
-      symmetric = MQ ? MQ->IsSymmetric() : true;
-
-      DenseMatrix M;
-      Vector Msymm;
-      if (symmetric)
-      {
-         Msymm.SetSize(MQsymmDim);
-      }
-      else
-      {
-         M.SetSize(dim);
-      }
-
-      auto C = Reshape(coeff.HostWrite(), MQdim, nq, ne);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            if (MQ->IsSymmetric())
-            {
-               MQ->EvalSymmetric(Msymm, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<MQsymmDim; ++i)
-               {
-                  C(i, p, e) = Msymm[i];
-               }
-            }
-            else
-            {
-               MQ->Eval(M, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dim; ++i)
-                  for (int j=0; j<dim; ++j)
-                  {
-                     C(j+(i*dim), p, e) = M(i,j);
-                  }
-            }
-         }
-      }
-   }
-   else if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "");
-      coeffDim = VQ->GetVDim();
-      coeff.SetSize(coeffDim * nq * ne);
-      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      Vector D(coeffDim);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            VQ->Eval(D, *tr, ir->IntPoint(p));
-            for (int i=0; i<coeffDim; ++i)
-            {
-               C(i, p, e) = D[i];
-            }
-         }
-      }
-   }
-   else if (Q == nullptr)
+   if (Q == nullptr)
   {
      coeff.SetSize(1);
      coeff(0) = 1.0;
@@ -483,15 +329,18 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
         }
      }
   }
-   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
-                   Device::GetDeviceMemoryType());
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
-                    geom->J, coeff, pa_data);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
+                    coeff, pa_data);
 }

+void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   SetupPA(fes);
+}
+
+
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionDiagonal2D(const int NE,
-                                  const bool symmetric,
                                  const Array<double> &b,
                                  const Array<double> &g,
                                  const Vector &d,
@@ -505,9 +354,9 @@ static void PADiffusionDiagonal2D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
-   // note the different shape for D, if this is a symmetric matrix we only
+   // note the different shape for D, this is a (symmetric) matrix so we only
   // store necessary entries
-   auto D = Reshape(d.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d.Read(), Q1D*Q1D, 3, NE);
   auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL(e, NE,
   {
@@ -529,13 +378,12 @@ static void PADiffusionDiagonal2D(const int NE,
            for (int qy = 0; qy < Q1D; ++qy)
            {
               const int q = qx + qy * Q1D;
-               const double D00 = D(q,0,e);
-               const double D10 = D(q,1,e);
-               const double D01 = symmetric ? D10 : D(q,2,e);
-               const double D11 = symmetric ? D(q,2,e) : D(q,3,e);
-               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D00;
-               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * (D01 + D10);
-               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D11;
+               const double D0 = D(q,0,e);
+               const double D1 = D(q,1,e);
+               const double D2 = D(q,2,e);
+               QD0[qx][dy] += B(qy, dy) * B(qy, dy) * D0;
+               QD1[qx][dy] += B(qy, dy) * G(qy, dy) * D1;
+               QD2[qx][dy] += G(qy, dy) * G(qy, dy) * D2;
            }
         }
      }
@@ -547,6 +395,7 @@ static void PADiffusionDiagonal2D(const int NE,
            {
               Y(dx,dy,e) += G(qx, dx) * G(qx, dx) * QD0[qx][dy];
               Y(dx,dy,e) += G(qx, dx) * B(qx, dx) * QD1[qx][dy];
+               Y(dx,dy,e) += B(qx, dx) * G(qx, dx) * QD1[qx][dy];
               Y(dx,dy,e) += B(qx, dx) * B(qx, dx) * QD2[qx][dy];
            }
         }
@@ -557,7 +406,6 @@ static void PADiffusionDiagonal2D(const int NE,
 // Shared memory PA Diffusion Diagonal 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionDiagonal2D(const int NE,
-                                      const bool symmetric,
                                      const Array<double> &b_,
                                      const Array<double> &g_,
                                      const Vector &d_,
@@ -574,7 +422,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
@@ -587,10 +435,10 @@ static void SmemPADiffusionDiagonal2D(const int NE,
      MFEM_SHARED double BG[2][MQ1*MD1];
      double (*B)[MD1] = (double (*)[MD1]) (BG+0);
      double (*G)[MD1] = (double (*)[MD1]) (BG+1);
-      MFEM_SHARED double QD[3][NBZ][MD1][MQ1];
+      MFEM_SHARED double QD[4][NBZ][MD1][MQ1];
      double (*QD0)[MD1] = (double (*)[MD1])(QD[0] + tidz);
      double (*QD1)[MD1] = (double (*)[MD1])(QD[1] + tidz);
-      double (*QD2)[MD1] = (double (*)[MD1])(QD[2] + tidz);
+      double (*QD2)[MD1] = (double (*)[MD1])(QD[3] + tidz);
      if (tidz == 0)
      {
         MFEM_FOREACH_THREAD(d,y,D1D)
@@ -613,18 +461,17 @@ static void SmemPADiffusionDiagonal2D(const int NE,
            for (int qy = 0; qy < Q1D; ++qy)
            {
               const int q = qx + qy * Q1D;
-               const double D00 = D(q,0,e);
-               const double D10 = D(q,1,e);
-               const double D01 = symmetric ? D10 : D(q,2,e);
-               const double D11 = symmetric ? D(q,2,e) : D(q,3,e);
+               const double D0 = D(q,0,e);
+               const double D1 = D(q,1,e);
+               const double D2 = D(q,2,e);
               const double By = B[qy][dy];
               const double Gy = G[qy][dy];
               const double BB = By * By;
               const double BG = By * Gy;
               const double GG = Gy * Gy;
-               QD0[qx][dy] += BB * D00;
-               QD1[qx][dy] += BG * (D01 + D10);
-               QD2[qx][dy] += GG * D11;
+               QD0[qx][dy] += BB * D0;
+               QD1[qx][dy] += BG * D1;
+               QD2[qx][dy] += GG * D2;
            }
         }
      }
@@ -642,6 +489,7 @@ static void SmemPADiffusionDiagonal2D(const int NE,
               const double GG = Gx * Gx;
               Y(dx,dy,e) += GG * QD0[qx][dy];
               Y(dx,dy,e) += BG * QD1[qx][dy];
+               Y(dx,dy,e) += BG * QD1[qx][dy];
               Y(dx,dy,e) += BB * QD2[qx][dy];
            }
         }
@@ -651,7 +499,6 @@ static void SmemPADiffusionDiagonal2D(const int NE,

 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionDiagonal3D(const int NE,
-                                  const bool symmetric,
                                  const Array<double> &b,
                                  const Array<double> &g,
                                  const Vector &d,
@@ -668,7 +515,7 @@ static void PADiffusionDiagonal3D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE);
   auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL(e, NE,
   {
@@ -693,10 +540,9 @@ static void PADiffusionDiagonal3D(const int NE,
                     for (int qz = 0; qz < Q1D; ++qz)
                     {
                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int ksym = j >= i ?
+                        const int k = j >= i ?
                        3 - (3-i)*(2-i)/2 + j:
                        3 - (3-j)*(2-j)/2 + i;
-                        const int k = symmetric ? ksym : (i*DIM) + j;
                        const double O = Q(q,k,e);
                        const double Bz = B(qz,dz);
                        const double Gz = G(qz,dz);
@@ -752,7 +598,6 @@ static void PADiffusionDiagonal3D(const int NE,
 // Shared memory PA Diffusion Diagonal 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionDiagonal3D(const int NE,
-                                      const bool symmetric,
                                      const Array<double> &b_,
                                      const Array<double> &g_,
                                      const Vector &d_,
@@ -769,7 +614,7 @@ static void SmemPADiffusionDiagonal3D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
@@ -810,10 +655,9 @@ static void SmemPADiffusionDiagonal3D(const int NE,
                     for (int qz = 0; qz < Q1D; ++qz)
                     {
                        const int q = qx + (qy + qz * Q1D) * Q1D;
-                        const int ksym = j >= i ?
-                                         3 - (3-i)*(2-i)/2 + j:
-                                         3 - (3-j)*(2-j)/2 + i;
-                        const int k = symmetric ? ksym : (i*DIM) + j;
+                        const int k = j >= i ?
+                                      3 - (3-i)*(2-i)/2 + j:
+                                      3 - (3-j)*(2-j)/2 + i;
                        const double O = D(q,k,e);
                        const double Bz = B[qz][dz];
                        const double Gz = G[qz][dz];
@@ -872,7 +716,6 @@ static void PADiffusionAssembleDiagonal(const int dim,
                                        const int D1D,
                                        const int Q1D,
                                        const int NE,
-                                        const bool symm,
                                        const Array<double> &B,
                                        const Array<double> &G,
                                        const Vector &D,
@@ -882,30 +725,30 @@ static void PADiffusionAssembleDiagonal(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,symm,B,G,D,Y);
-         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,symm,B,G,D,Y);
-         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,symm,B,G,D,Y);
-         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,symm,B,G,D,Y);
-         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,symm,B,G,D,Y);
-         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,symm,B,G,D,Y);
-         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,symm,B,G,D,Y);
-         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,symm,B,G,D,Y);
-         default: return PADiffusionDiagonal2D(NE,symm,B,G,D,Y,D1D,Q1D);
+         case 0x22: return SmemPADiffusionDiagonal2D<2,2,8>(NE,B,G,D,Y);
+         case 0x33: return SmemPADiffusionDiagonal2D<3,3,8>(NE,B,G,D,Y);
+         case 0x44: return SmemPADiffusionDiagonal2D<4,4,4>(NE,B,G,D,Y);
+         case 0x55: return SmemPADiffusionDiagonal2D<5,5,4>(NE,B,G,D,Y);
+         case 0x66: return SmemPADiffusionDiagonal2D<6,6,2>(NE,B,G,D,Y);
+         case 0x77: return SmemPADiffusionDiagonal2D<7,7,2>(NE,B,G,D,Y);
+         case 0x88: return SmemPADiffusionDiagonal2D<8,8,1>(NE,B,G,D,Y);
+         case 0x99: return SmemPADiffusionDiagonal2D<9,9,1>(NE,B,G,D,Y);
+         default: return PADiffusionDiagonal2D(NE,B,G,D,Y,D1D,Q1D);
      }
   }
   else if (dim == 3)
   {
      switch ((D1D << 4 ) | Q1D)
      {
-         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,symm,B,G,D,Y);
-         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,symm,B,G,D,Y);
-         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,symm,B,G,D,Y);
-         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,symm,B,G,D,Y);
-         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,symm,B,G,D,Y);
-         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,symm,B,G,D,Y);
-         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,symm,B,G,D,Y);
-         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,symm,B,G,D,Y);
-         default: return PADiffusionDiagonal3D(NE,symm,B,G,D,Y,D1D,Q1D);
+         case 0x23: return SmemPADiffusionDiagonal3D<2,3>(NE,B,G,D,Y);
+         case 0x34: return SmemPADiffusionDiagonal3D<3,4>(NE,B,G,D,Y);
+         case 0x45: return SmemPADiffusionDiagonal3D<4,5>(NE,B,G,D,Y);
+         case 0x56: return SmemPADiffusionDiagonal3D<5,6>(NE,B,G,D,Y);
+         case 0x67: return SmemPADiffusionDiagonal3D<6,7>(NE,B,G,D,Y);
+         case 0x78: return SmemPADiffusionDiagonal3D<7,8>(NE,B,G,D,Y);
+         case 0x89: return SmemPADiffusionDiagonal3D<8,9>(NE,B,G,D,Y);
+         case 0x9A: return SmemPADiffusionDiagonal3D<9,10>(NE,B,G,D,Y);
+         default: return PADiffusionDiagonal3D(NE,B,G,D,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -913,14 +756,15 @@ static void PADiffusionAssembleDiagonal(const int dim,

 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
+      CeedAssembleDiagonalPA(ceedDataPtr, diag);
   }
   else
+#endif
   {
-      if (pa_data.Size()==0) { AssemblePA(*fespace); }
-      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne, symmetric,
+      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
                                  maps->B, maps->G, pa_data, diag);
   }
 }
@@ -1029,7 +873,6 @@ static void OccaPADiffusionApply3D(const int D1D,
 // PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionApply2D(const int NE,
-                               const bool symmetric,
                               const Array<double> &b_,
                               const Array<double> &g_,
                               const Array<double> &bt_,
@@ -1048,7 +891,7 @@ static void PADiffusionApply2D(const int NE,
   auto G = Reshape(g_.Read(), Q1D, D1D);
   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
   auto Gt = Reshape(gt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
   auto X = Reshape(x_.Read(), D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL(e, NE,
@@ -1104,15 +947,14 @@ static void PADiffusionApply2D(const int NE,
            const int q = qx + qy * Q1D;

            const double O11 = D(q,0,e);
-            const double O21 = D(q,1,e);
-            const double O12 = symmetric ? O21 : D(q,2,e);
-            const double O22 = symmetric ? D(q,2,e) : D(q,3,e);
+            const double O12 = D(q,1,e);
+            const double O22 = D(q,2,e);

            const double gradX = grad[qy][qx][0];
            const double gradY = grad[qy][qx][1];

            grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-            grad[qy][qx][1] = (O21 * gradX) + (O22 * gradY);
+            grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
         }
      }
      for (int qy = 0; qy < Q1D; ++qy)
@@ -1151,7 +993,6 @@ static void PADiffusionApply2D(const int NE,
 // Shared memory PA Diffusion Apply 2D kernel
 template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
 static void SmemPADiffusionApply2D(const int NE,
-                                   const bool symmetric,
                                   const Array<double> &b_,
                                   const Array<double> &g_,
                                   const Vector &d_,
@@ -1169,7 +1010,7 @@ static void SmemPADiffusionApply2D(const int NE,
   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
@@ -1251,13 +1092,12 @@ static void SmemPADiffusionApply2D(const int NE,
         {
            const int q = (qx + ((qy) * Q1D));
            const double O11 = D(q,0,e);
-            const double O21 = D(q,1,e);
-            const double O12 = symmetric ? O21 : D(q,2,e);
-            const double O22 = symmetric ? D(q,2,e) : D(q,3,e);
+            const double O12 = D(q,1,e);
+            const double O22 = D(q,2,e);
            const double gX = QQ0[qy][qx];
            const double gY = QQ1[qy][qx];
            QQ0[qy][qx] = (O11 * gX) + (O12 * gY);
-            QQ1[qy][qx] = (O21 * gX) + (O22 * gY);
+            QQ1[qy][qx] = (O12 * gX) + (O22 * gY);
         }
      }
      MFEM_SYNC_THREAD;
@@ -1309,7 +1149,6 @@ static void SmemPADiffusionApply2D(const int NE,
 // PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PADiffusionApply3D(const int NE,
-                               const bool symmetric,
                               const Array<double> &b,
                               const Array<double> &g,
                               const Array<double> &bt,
@@ -1327,7 +1166,7 @@ static void PADiffusionApply3D(const int NE,
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto Bt = Reshape(bt.Read(), D1D, Q1D);
   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
+   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL(e, NE,
@@ -1418,18 +1257,15 @@ static void PADiffusionApply3D(const int NE,
               const double O11 = D(q,0,e);
               const double O12 = D(q,1,e);
               const double O13 = D(q,2,e);
-               const double O21 = symmetric ? O12 : D(q,3,e);
-               const double O22 = symmetric ? D(q,3,e) : D(q,4,e);
-               const double O23 = symmetric ? D(q,4,e) : D(q,5,e);
-               const double O31 = symmetric ? O13 : D(q,6,e);
-               const double O32 = symmetric ? O23 : D(q,7,e);
-               const double O33 = symmetric ? D(q,5,e) : D(q,8,e);
+               const double O22 = D(q,3,e);
+               const double O23 = D(q,4,e);
+               const double O33 = D(q,5,e);
               const double gradX = grad[qz][qy][qx][0];
               const double gradY = grad[qz][qy][qx][1];
               const double gradZ = grad[qz][qy][qx][2];
               grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
-               grad[qz][qy][qx][1] = (O21*gradX)+(O22*gradY)+(O23*gradZ);
-               grad[qz][qy][qx][2] = (O31*gradX)+(O32*gradY)+(O33*gradZ);
+               grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
+               grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
            }
         }
      }
@@ -1528,7 +1364,6 @@ static MFEM_HOST_DEVICE inline double sign(const int q, const int d)

 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
-                                   const bool symmetric,
                                   const Array<double> &b_,
                                   const Array<double> &g_,
                                   const Vector &d_,
@@ -1545,7 +1380,7 @@ static void SmemPADiffusionApply3D(const int NE,
   MFEM_VERIFY(Q1D <= M1Q, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
@@ -1692,18 +1527,15 @@ static void SmemPADiffusionApply3D(const int NE,
               const double O11 = d(qx,qy,qz,0,e);
               const double O12 = d(qx,qy,qz,1,e);
               const double O13 = d(qx,qy,qz,2,e);
-               const double O21 = symmetric ? O12 : d(qx,qy,qz,3,e);
-               const double O22 = symmetric ? d(qx,qy,qz,3,e) : d(qx,qy,qz,4,e);
-               const double O23 = symmetric ? d(qx,qy,qz,4,e) : d(qx,qy,qz,5,e);
-               const double O31 = symmetric ? O13 : d(qx,qy,qz,6,e);
-               const double O32 = symmetric ? O23 : d(qx,qy,qz,7,e);
-               const double O33 = symmetric ? d(qx,qy,qz,5,e) : d(qx,qy,qz,8,e);
+               const double O22 = d(qx,qy,qz,3,e);
+               const double O23 = d(qx,qy,qz,4,e);
+               const double O33 = d(qx,qy,qz,5,e);
               const double gX = u[qz];
               const double gY = v[qz];
               const double gZ = w[qz];
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
-               QQQ1[qz][qy][qx] = (O21*gX) + (O22*gY) + (O23*gZ);
-               QQQ2[qz][qy][qx] = (O31*gX) + (O32*gY) + (O33*gZ);
+               QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
+               QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
            }
         }
      }
@@ -1824,7 +1656,6 @@ static void PADiffusionApply(const int dim,
                             const int D1D,
                             const int Q1D,
                             const int NE,
-                             const bool symm,
                             const Array<double> &B,
                             const Array<double> &G,
                             const Array<double> &Bt,
@@ -1849,21 +1680,21 @@ static void PADiffusionApply(const int dim,
      MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
   }
 #endif // MFEM_USE_OCCA
-   const int ID = (D1D << 4) | Q1D;
+   const int ID = (D1D << 4 ) | Q1D;

   if (dim == 2)
   {
      switch (ID)
      {
-         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,symm,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,symm,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,symm,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,symm,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,symm,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
+         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
+         case 0x44: return SmemPADiffusionApply2D<4,4,8>(NE,B,G,D,X,Y);
+         case 0x55: return SmemPADiffusionApply2D<5,5,8>(NE,B,G,D,X,Y);
+         case 0x66: return SmemPADiffusionApply2D<6,6,4>(NE,B,G,D,X,Y);
+         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,B,G,D,X,Y);
+         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,B,G,D,X,Y);
+         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,B,G,D,X,Y);
+         default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }

@@ -1871,16 +1702,16 @@ static void PADiffusionApply(const int dim,
   {
      switch (ID)
      {
-         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,symm,B,G,D,X,Y);
-         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,symm,B,G,D,X,Y);
-         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,symm,B,G,D,X,Y);
-         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,symm,B,G,D,X,Y);
-         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,symm,B,G,D,X,Y);
-         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,symm,B,G,D,X,Y);
-         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
-         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
-         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
+         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
+         case 0x45: return SmemPADiffusionApply3D<4,5>(NE,B,G,D,X,Y);
+         case 0x46: return SmemPADiffusionApply3D<4,6>(NE,B,G,D,X,Y);
+         case 0x56: return SmemPADiffusionApply3D<5,6>(NE,B,G,D,X,Y);
+         case 0x58: return SmemPADiffusionApply3D<5,8>(NE,B,G,D,X,Y);
+         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,B,G,D,X,Y);
+         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,B,G,D,X,Y);
+         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,B,G,D,X,Y);
+         default:   return PADiffusionApply3D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -1889,13 +1720,15 @@ static void PADiffusionApply(const int dim,
 // PA Diffusion Apply kernel
 void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAddMult(ceedDataPtr, x, y);
+      CeedAddMultPA(ceedDataPtr, x, y);
   }
   else
+#endif
   {
-      PADiffusionApply(dim, dofs1D, quad1D, ne, symmetric,
+      PADiffusionApply(dim, dofs1D, quad1D, ne,
                       maps->B, maps->G, maps->Bt, maps->Gt,
                       pa_data, x, y);
   }
@@ -21,7 +21,6 @@ static void EAMassAssemble1D(const int NE,
                             const Array<double> &basis,
                             const Vector &padata,
                             Vector &eadata,
-                             const bool add,
                             const int d1d = 0,
                             const int q1d = 0)
 {
@@ -53,14 +52,7 @@ static void EAMassAssemble1D(const int NE,
            {
               val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
            }
-            if (add)
-            {
-               M(i1, j1, e) += val;
-            }
-            else
-            {
-               M(i1, j1, e) = val;
-            }
+            M(i1, j1, e) += val;
         }
      }
   });
@@ -71,7 +63,6 @@ static void EAMassAssemble2D(const int NE,
                             const Array<double> &basis,
                             const Vector &padata,
                             Vector &eadata,
-                             const bool add,
                             const int d1d = 0,
                             const int q1d = 0)
 {
@@ -123,14 +114,7 @@ static void EAMassAssemble2D(const int NE,
                               * s_D[k1][k2];
                     }
                  }
-                  if (add)
-                  {
-                     M(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     M(i1, i2, j1, j2, e) = val;
-                  }
+                  M(i1, i2, j1, j2, e) += val;
               }
            }
         }
@@ -143,7 +127,6 @@ static void EAMassAssemble3D(const int NE,
                             const Array<double> &basis,
                             const Vector &padata,
                             Vector &eadata,
-                             const bool add,
                             const int d1d = 0,
                             const int q1d = 0)
 {
@@ -206,14 +189,7 @@ static void EAMassAssemble3D(const int NE,
                              }
                           }
                        }
-                        if (add)
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        M(i1, i2, i3, j1, j2, j3, e) += val;
                     }
                  }
               }
@@ -224,8 +200,7 @@ static void EAMassAssemble3D(const int NE,
 }

 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                Vector &ea_data,
-                                const bool add)
+                                Vector &ea_data)
 {
   AssemblePA(fes);
   const int ne = fes.GetMesh()->GetNE();
@@ -234,47 +209,44 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data);
+         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data);
+         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data);
+         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data);
+         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data);
+         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data);
+         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data);
+         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data);
+         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data);
+         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data);
+         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data);
+         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data);
+         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data);
+         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data);
+         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data,add);
-         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data,add);
-         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data,add);
-         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data,add);
-         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data,add);
-         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data,add);
-         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data);
+         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data);
+         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data);
+         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data);
+         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data);
+         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data);
+         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -1,71 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/mass.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void MassIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assuming the same element type
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFMassAssemble(fes, *ir, *ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: MassIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void MassIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: MassIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-void MassIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: MassIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-} // namespace mfem
@@ -23,7 +23,7 @@ namespace mfem

 // PA Mass Assemble kernel

-void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
+void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
 {
   // Assuming the same element type
   fespace = &fes;
@@ -32,13 +32,16 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const FiniteElement &el = *fes.GetFE(0);
   ElementTransformation *T = mesh->GetElementTransformation(0);
   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
+#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPAMassAssemble(fes, *ir, *ceedDataPtr);
+      if (ceedDataPtr) { delete ceedDataPtr; }
+      CeedData* ptr = new CeedData();
+      ceedDataPtr = ptr;
+      InitCeedCoeff(Q, ptr);
+      return CeedPAMassAssemble(fes, *ir, *ptr);
   }
+#endif
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
@@ -152,6 +155,12 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   }
 }

+void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
+{
+   SetupPA(fes);
+}
+
+
 template<int T_D1D = 0, int T_Q1D = 0>
 static void PAMassAssembleDiagonal2D(const int NE,
                                     const Array<double> &b,
@@ -459,11 +468,13 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,

 void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
+#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
+      CeedAssembleDiagonalPA(ceedDataPtr, diag);
   }
   else
+#endif
   {
      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
   }
@@ -1216,11 +1227,13 @@ static void PAMassApply(const int dim,

 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+#ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAddMult(ceedDataPtr, x, y);
+      CeedAddMultPA(ceedDataPtr, x, y);
   }
   else
+#endif
   {
      PAMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
   }
@@ -16,171 +16,88 @@ namespace mfem
 {

 void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data, const bool add)
+                                     Vector &ea_data)
 {
-   if (add)
+   Vector ea_data_tmp(ea_data.Size());
+   ea_data_tmp = 0.0;
+   bfi->AssembleEA(fes, ea_data_tmp);
+   const int ne = fes.GetNE();
+   if (ne == 0) { return; }
+   const int dofs = fes.GetFE(0)->GetDof();
+   auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
+   auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
+   MFEM_FORALL(e, ne,
   {
-      Vector ea_data_tmp(ea_data.Size());
-      bfi->AssembleEA(fes, ea_data_tmp, false);
-      const int ne = fes.GetNE();
-      if (ne == 0) { return; }
-      const int dofs = fes.GetFE(0)->GetDof();
-      auto A = Reshape(ea_data_tmp.Read(), dofs, dofs, ne);
-      auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
+      for (int i = 0; i < dofs; i++)
      {
-         for (int i = 0; i < dofs; i++)
+         for (int j = 0; j < dofs; j++)
         {
-            for (int j = 0; j < dofs; j++)
-            {
-               const double a = A(i, j, e);
-               AT(j, i, e) += a;
-            }
+            const double a = A(i, j, e);
+            AT(j, i, e) += a;
         }
-      });
-   }
-   else
-   {
-      bfi->AssembleEA(fes, ea_data, false);
-      const int ne = fes.GetNE();
-      if (ne == 0) { return; }
-      const int dofs = fes.GetFE(0)->GetDof();
-      auto A = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
-      {
-         for (int i = 0; i < dofs; i++)
-         {
-            for (int j = i+1; j < dofs; j++)
-            {
-               const double aij = A(i, j, e);
-               const double aji = A(j, i, e);
-               A(j, i, e) = aij;
-               A(i, j, e) = aji;
-            }
-         }
-      });
-   }
+      }
+   });
 }

 void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                  Vector &ea_data_int,
-                                                  Vector &ea_data_ext,
-                                                  const bool add)
+                                                  Vector &ea_data_ext)
 {
   const int nf = fes.GetNFbyType(FaceType::Interior);
   if (nf == 0) { return; }
-   if (add)
+   Vector ea_data_int_tmp(ea_data_int.Size());
+   Vector ea_data_ext_tmp(ea_data_ext.Size());
+   ea_data_int_tmp = 0.0;
+   ea_data_ext_tmp = 0.0;
+   bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp);
+   const int faceDofs = fes.GetTraceElement(0,
+                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+   auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
+   auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
+   auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
+   auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
+   MFEM_FORALL(f, nf,
   {
-      Vector ea_data_int_tmp(ea_data_int.Size());
-      Vector ea_data_ext_tmp(ea_data_ext.Size());
-      bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
-      auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
-      auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
+      for (int i = 0; i < faceDofs; i++)
      {
-         for (int i = 0; i < faceDofs; i++)
+         for (int j = 0; j < faceDofs; j++)
         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double a_int0 = A_int(i, j, 0, f);
-               const double a_int1 = A_int(i, j, 1, f);
-               const double a_ext0 = A_ext(i, j, 0, f);
-               const double a_ext1 = A_ext(i, j, 1, f);
-               AT_int(j, i, 0, f) += a_int0;
-               AT_int(j, i, 1, f) += a_int1;
-               AT_ext(j, i, 0, f) += a_ext1;
-               AT_ext(j, i, 1, f) += a_ext0;
-            }
+            const double a_int0 = A_int(i, j, 0, f);
+            const double a_int1 = A_int(i, j, 1, f);
+            const double a_ext0 = A_ext(i, j, 0, f);
+            const double a_ext1 = A_ext(i, j, 1, f);
+            AT_int(j, i, 0, f) += a_int0;
+            AT_int(j, i, 1, f) += a_int1;
+            AT_ext(j, i, 0, f) += a_ext1;
+            AT_ext(j, i, 1, f) += a_ext0;
         }
-      });
-   }
-   else
-   {
-      bfi->AssembleEAInteriorFaces(fes, ea_data_int, ea_data_ext, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
-      {
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = i+1; j < faceDofs; j++)
-            {
-               const double aij_int0 = A_int(i, j, 0, f);
-               const double aij_int1 = A_int(i, j, 1, f);
-               const double aji_int0 = A_int(j, i, 0, f);
-               const double aji_int1 = A_int(j, i, 1, f);
-               A_int(j, i, 0, f) = aij_int0;
-               A_int(j, i, 1, f) = aij_int1;
-               A_int(i, j, 0, f) = aji_int0;
-               A_int(i, j, 1, f) = aji_int1;
-            }
-         }
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double aij_ext0 = A_ext(i, j, 0, f);
-               const double aji_ext1 = A_ext(j, i, 1, f);
-               A_ext(j, i, 1, f) = aij_ext0;
-               A_ext(i, j, 0, f) = aji_ext1;
-            }
-         }
-      });
-   }
+      }
+   });
 }

 void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                  Vector &ea_data_bdr,
-                                                  const bool add)
+                                                  Vector &ea_data_bdr)
 {
   const int nf = fes.GetNFbyType(FaceType::Boundary);
   if (nf == 0) { return; }
-   if (add)
+   Vector ea_data_bdr_tmp(ea_data_bdr.Size());
+   ea_data_bdr_tmp = 0.0;
+   bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp);
+   const int faceDofs = fes.GetTraceElement(0,
+                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+   auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
+   auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
+   MFEM_FORALL(f, nf,
   {
-      Vector ea_data_bdr_tmp(ea_data_bdr.Size());
-      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
-      auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
+      for (int i = 0; i < faceDofs; i++)
      {
-         for (int i = 0; i < faceDofs; i++)
+         for (int j = 0; j < faceDofs; j++)
         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double a_bdr = A_bdr(i, j, f);
-               AT_bdr(j, i, f) += a_bdr;
-            }
+            const double a_bdr = A_bdr(i, j, f);
+            AT_bdr(j, i, f) += a_bdr;
         }
-      });
-   }
-   else
-   {
-      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
-      {
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = i+1; j < faceDofs; j++)
-            {
-               const double aij_bdr = A_bdr(i, j, f);
-               const double aji_bdr = A_bdr(j, i, f);
-               A_bdr(j, i, f) = aij_bdr;
-               A_bdr(i, j, f) = aji_bdr;
-            }
-         }
-      });
-   }
+      }
+   });
 }

 }
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "libceed/diffusion.hpp"

 using namespace std;

@@ -130,13 +129,6 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   const FiniteElement &el = *fes.GetFE(0);
   const IntegrationRule *ir
      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPADiffusionAssemble(fes, *ir, * ceedDataPtr);
-   }
   const int dims = el.GetDim();
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int nq = ir->GetNPoints();
@@ -514,40 +506,33 @@ void PAVectorDiffusionApply3D(const int NE,
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-   {
-      const int D1D = dofs1D;
-      const int Q1D = quad1D;
-      const Array<double> &B = maps->B;
-      const Array<double> &G = maps->G;
-      const Array<double> &Bt = maps->Bt;
-      const Array<double> &Gt = maps->Gt;
-      const Vector &D = pa_data;
+   const int D1D = dofs1D;
+   const int Q1D = quad1D;
+   const Array<double> &B = maps->B;
+   const Array<double> &G = maps->G;
+   const Array<double> &Bt = maps->Bt;
+   const Array<double> &Gt = maps->Gt;
+   const Vector &D = pa_data;

-      if (dim == 2 && sdim == 3)
+   if (dim == 2 && sdim == 3)
+   {
+      switch ((dofs1D << 4 ) | quad1D)
      {
-         switch ((dofs1D << 4 ) | quad1D)
-         {
-            case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
-            case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
-            case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
-            case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
-            default:
-               return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
-         }
+         case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
+         case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
+         case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
+         case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
+         default:
+            return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
      }
-      if (dim == 2 && sdim == 2)
-      { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
-
-      if (dim == 3 && sdim == 3)
-      { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
-
-      MFEM_ABORT("Unknown kernel.");
   }
+   if (dim == 2 && sdim == 2)
+   { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
+
+   if (dim == 3 && sdim == 3)
+   { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
+
+   MFEM_ABORT("Unknown kernel.");
 }

 template<int T_D1D = 0, int T_Q1D = 0>
@@ -741,21 +726,14 @@ static void PAVectorDiffusionAssembleDiagonal(const int dim,

 void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-   {
-      PAVectorDiffusionAssembleDiagonal(dim,
-                                        dofs1D,
-                                        quad1D,
-                                        ne,
-                                        maps->B,
-                                        maps->G,
-                                        pa_data,
-                                        diag);
-   }
+   PAVectorDiffusionAssembleDiagonal(dim,
+                                     dofs1D,
+                                     quad1D,
+                                     ne,
+                                     maps->B,
+                                     maps->G,
+                                     pa_data,
+                                     diag);
 }

 } // namespace mfem
@@ -1,69 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/diffusion.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-void VectorDiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement &el = *fes.GetFE(0);
-   const IntegrationRule *ir
-      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFDiffusionAssemble(fes, *ir, * ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: VectorDiffusionIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void VectorDiffusionIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorDiffusionIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-void VectorDiffusionIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorDiffusionIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-} // namespace mfem
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "libceed/mass.hpp"

 using namespace std;

@@ -31,13 +30,6 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   ElementTransformation *T = mesh->GetElementTransformation(0);
   const IntegrationRule *ir
      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedPAMassAssemble(fes, *ir, *ceedDataPtr);
-   }
   dim = mesh->Dimension();
   ne = fes.GetMesh()->GetNE();
   nq = ir->GetNPoints();
@@ -369,14 +361,7 @@ static void PAVectorMassApply(const int dim,

 void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-   {
-      PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-   }
+   PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
 }

 template<const int T_D1D = 0, const int T_Q1D = 0>
@@ -529,21 +514,14 @@ static void PAVectorMassAssembleDiagonal(const int dim,

 void VectorMassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-   {
-      PAVectorMassAssembleDiagonal(dim,
-                                   dofs1D,
-                                   quad1D,
-                                   ne,
-                                   maps->B,
-                                   maps->Bt,
-                                   pa_data,
-                                   diag);
-   }
+   PAVectorMassAssembleDiagonal(dim,
+                                dofs1D,
+                                quad1D,
+                                ne,
+                                maps->B,
+                                maps->Bt,
+                                pa_data,
+                                diag);
 }

 } // namespace mfem
@@ -1,74 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-#include "libceed/mass.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// MF Mass Integrator
-
-// MF Mass Assemble kernel
-void VectorMassIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-#ifdef MFEM_USE_CEED
-   // Assuming the same element type
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir
-      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
-   if (DeviceCanUseCeed())
-   {
-      delete ceedDataPtr;
-      ceedDataPtr = new CeedData;
-      InitCeedCoeff(Q, *mesh, *ir, ceedDataPtr);
-      return CeedMFMassAssemble(fes, *ir, *ceedDataPtr);
-   }
-#endif
-   mfem_error("Error: VectorMassIntegrator::AssembleMF only implemented with libCEED");
-}
-
-void VectorMassIntegrator::AddMultMF(const Vector &x, Vector &y) const
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAddMult(ceedDataPtr, x, y);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorMassIntegrator::AssembleDiagonalMF only implemented with libCEED");
-   }
-}
-
-void VectorMassIntegrator::AssembleDiagonalMF(Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonal(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      mfem_error("Error: VectorMassIntegrator::AddMultMF only implemented with libCEED");
-   }
-}
-
-} // namespace mfem
@@ -15,78 +15,63 @@
 namespace mfem
 {

-void PADiffusionSetup3D(const int Q1D,
-                        const int coeffDim,
-                        const int NE,
-                        const Array<double> &w,
-                        const Vector &j,
-                        const Vector &_coeff,
-                        Vector &op);
+void PAHcurlSetup2D(const int Q1D,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    const Vector &j,
+                    Vector &_coeff,
+                    Vector &op);
+
+void PAHcurlSetup3D(const int Q1D,
+                    const int coeffDim,
+                    const int NE,
+                    const Array<double> &w,
+                    const Vector &j,
+                    Vector &_coeff,
+                    Vector &op);

 void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
                                   const int NE,
                                   const bool symmetric,
-                                   const Array<double> &bo,
-                                   const Array<double> &bc,
-                                   const Vector &pa_data,
-                                   Vector &diag);
+                                   const Array<double> &_Bo,
+                                   const Array<double> &_Bc,
+                                   const Vector &_op,
+                                   Vector &_diag);

 void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                   const int Q1D,
                                   const int NE,
                                   const bool symmetric,
-                                   const Array<double> &bo,
-                                   const Array<double> &bc,
-                                   const Vector &pa_data,
-                                   Vector &diag);
-
-template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAHcurlMassAssembleDiagonal3D(const int D1D,
-                                       const int Q1D,
-                                       const int NE,
-                                       const bool symmetric,
-                                       const Array<double> &bo,
-                                       const Array<double> &bc,
-                                       const Vector &pa_data,
-                                       Vector &diag);
+                                   const Array<double> &_Bo,
+                                   const Array<double> &_Bc,
+                                   const Vector &_op,
+                                   Vector &_diag);

 void PAHcurlMassApply2D(const int D1D,
                        const int Q1D,
                        const int NE,
                        const bool symmetric,
-                        const Array<double> &bo,
-                        const Array<double> &bc,
-                        const Array<double> &bot,
-                        const Array<double> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
+                        const Array<double> &_Bo,
+                        const Array<double> &_Bc,
+                        const Array<double> &_Bot,
+                        const Array<double> &_Bct,
+                        const Vector &_op,
+                        const Vector &_x,
+                        Vector &_y);

 void PAHcurlMassApply3D(const int D1D,
                        const int Q1D,
                        const int NE,
                        const bool symmetric,
-                        const Array<double> &bo,
-                        const Array<double> &bc,
-                        const Array<double> &bot,
-                        const Array<double> &bct,
-                        const Vector &pa_data,
-                        const Vector &x,
-                        Vector &y);
-
-template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAHcurlMassApply3D(const int D1D,
-                            const int Q1D,
-                            const int NE,
-                            const bool symmetric,
-                            const Array<double> &bo,
-                            const Array<double> &bc,
-                            const Array<double> &bot,
-                            const Array<double> &bct,
-                            const Vector &pa_data,
-                            const Vector &x,
-                            Vector &y);
+                        const Array<double> &_Bo,
+                        const Array<double> &_Bc,
+                        const Array<double> &_Bot,
+                        const Array<double> &_Bct,
+                        const Vector &_op,
+                        const Vector &_x,
+                        Vector &_y);

 void PAHdivSetup2D(const int Q1D,
                   const int NE,
@@ -105,24 +90,24 @@ void PAHdivSetup3D(const int Q1D,
 void PAHcurlH1Apply2D(const int D1D,
                      const int Q1D,
                      const int NE,
-                      const Array<double> &bc,
-                      const Array<double> &gc,
-                      const Array<double> &bot,
-                      const Array<double> &bct,
-                      const Vector &pa_data,
-                      const Vector &x,
-                      Vector &y);
+                      const Array<double> &_Bc,
+                      const Array<double> &_Gc,
+                      const Array<double> &_Bot,
+                      const Array<double> &_Bct,
+                      const Vector &_op,
+                      const Vector &_x,
+                      Vector &_y);

 void PAHcurlH1Apply3D(const int D1D,
                      const int Q1D,
                      const int NE,
-                      const Array<double> &bc,
-                      const Array<double> &gc,
-                      const Array<double> &bot,
-                      const Array<double> &bct,
-                      const Vector &pa_data,
-                      const Vector &x,
-                      Vector &y);
+                      const Array<double> &_Bc,
+                      const Array<double> &_Gc,
+                      const Array<double> &_Bot,
+                      const Array<double> &_Bct,
+                      const Vector &_op,
+                      const Vector &_x,
+                      Vector &_y);

 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  const int Q1D,
@@ -181,11 +166,12 @@ void PAHcurlHdivSetup3D(const int Q1D,
                        Vector &_coeff,
                        Vector &op)
 {
+   const int NQ = Q1D*Q1D*Q1D;
   const bool symmetric = (coeffDim != 9);
-   auto W = Reshape(_w.Read(), Q1D, Q1D, Q1D);
-   auto J = Reshape(j.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, Q1D, Q1D, Q1D, NE);
-   auto y = Reshape(op.Write(), 9, Q1D, Q1D, Q1D, NE);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 9, NQ, NE);

   const int i11 = 0;
   const int i12 = transpose ? 3 : 1;
@@ -197,89 +183,83 @@ void PAHcurlHdivSetup3D(const int Q1D,
   const int i32 = transpose ? 5 : 7;
   const int i33 = 8;

-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   MFEM_FORALL(e, NE,
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double J13 = J(q,0,2,e);
+         const double J23 = J(q,1,2,e);
+         const double J33 = J(q,2,2,e);
+         const double detJ = J11 * (J22 * J33 - J32 * J23) -
+         /* */               J21 * (J12 * J33 - J32 * J13) +
+         /* */               J31 * (J12 * J23 - J22 * J13);
+         const double w_detJ = W[q] / detJ;
+         // adj(J)
+         const double A11 = (J22 * J33) - (J23 * J32);
+         const double A12 = (J32 * J13) - (J12 * J33);
+         const double A13 = (J12 * J23) - (J22 * J13);
+         const double A21 = (J31 * J23) - (J21 * J33);
+         const double A22 = (J11 * J33) - (J13 * J31);
+         const double A23 = (J21 * J13) - (J11 * J23);
+         const double A31 = (J21 * J32) - (J31 * J22);
+         const double A32 = (J31 * J12) - (J11 * J32);
+         const double A33 = (J11 * J22) - (J12 * J21);
+
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               const double J11 = J(qx,qy,qz,0,0,e);
-               const double J21 = J(qx,qy,qz,1,0,e);
-               const double J31 = J(qx,qy,qz,2,0,e);
-               const double J12 = J(qx,qy,qz,0,1,e);
-               const double J22 = J(qx,qy,qz,1,1,e);
-               const double J32 = J(qx,qy,qz,2,1,e);
-               const double J13 = J(qx,qy,qz,0,2,e);
-               const double J23 = J(qx,qy,qz,1,2,e);
-               const double J33 = J(qx,qy,qz,2,2,e);
-               const double detJ = J11 * (J22 * J33 - J32 * J23) -
-               /* */               J21 * (J12 * J33 - J32 * J13) +
-               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double w_detJ = W(qx,qy,qz) / detJ;
-               // adj(J)
-               const double A11 = (J22 * J33) - (J23 * J32);
-               const double A12 = (J32 * J13) - (J12 * J33);
-               const double A13 = (J12 * J23) - (J22 * J13);
-               const double A21 = (J31 * J23) - (J21 * J33);
-               const double A22 = (J11 * J33) - (J13 * J31);
-               const double A23 = (J21 * J13) - (J11 * J23);
-               const double A31 = (J21 * J32) - (J31 * J22);
-               const double A32 = (J31 * J12) - (J11 * J32);
-               const double A33 = (J11 * J22) - (J12 * J21);
+            // First compute entries of R = MJ
+            const double M11 = (!symmetric) ? coeff(i11, q, e) : coeff(0, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M13 = (!symmetric) ? coeff(i13, q, e) : coeff(2, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(3, q, e);
+            const double M23 = (!symmetric) ? coeff(i23, q, e) : coeff(4, q, e);
+            const double M31 = (!symmetric) ? coeff(i31, q, e) : M13;
+            const double M32 = (!symmetric) ? coeff(i32, q, e) : M23;
+            const double M33 = (!symmetric) ? coeff(i33, q, e) : coeff(5, q, e);

-               if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-               {
-                  // First compute entries of R = MJ
-                  const double M11 = (!symmetric) ? coeff(i11,qx,qy,qz,e) : coeff(0,qx,qy,qz,e);
-                  const double M12 = (!symmetric) ? coeff(i12,qx,qy,qz,e) : coeff(1,qx,qy,qz,e);
-                  const double M13 = (!symmetric) ? coeff(i13,qx,qy,qz,e) : coeff(2,qx,qy,qz,e);
-                  const double M21 = (!symmetric) ? coeff(i21,qx,qy,qz,e) : M12;
-                  const double M22 = (!symmetric) ? coeff(i22,qx,qy,qz,e) : coeff(3,qx,qy,qz,e);
-                  const double M23 = (!symmetric) ? coeff(i23,qx,qy,qz,e) : coeff(4,qx,qy,qz,e);
-                  const double M31 = (!symmetric) ? coeff(i31,qx,qy,qz,e) : M13;
-                  const double M32 = (!symmetric) ? coeff(i32,qx,qy,qz,e) : M23;
-                  const double M33 = (!symmetric) ? coeff(i33,qx,qy,qz,e) : coeff(5,qx,qy,qz,e);
+            const double R11 = M11*J11 + M12*J12 + M13*J13;
+            const double R12 = M11*J21 + M12*J22 + M13*J23;
+            const double R13 = M11*J31 + M12*J32 + M13*J33;
+            const double R21 = M21*J11 + M22*J12 + M23*J13;
+            const double R22 = M21*J21 + M22*J22 + M23*J23;
+            const double R23 = M21*J31 + M22*J32 + M23*J33;
+            const double R31 = M31*J11 + M32*J12 + M33*J13;
+            const double R32 = M31*J21 + M32*J22 + M33*J23;
+            const double R33 = M31*J31 + M32*J32 + M33*J33;

-                  const double R11 = M11*J11 + M12*J12 + M13*J13;
-                  const double R12 = M11*J21 + M12*J22 + M13*J23;
-                  const double R13 = M11*J31 + M12*J32 + M13*J33;
-                  const double R21 = M21*J11 + M22*J12 + M23*J13;
-                  const double R22 = M21*J21 + M22*J22 + M23*J23;
-                  const double R23 = M21*J31 + M22*J32 + M23*J33;
-                  const double R31 = M31*J11 + M32*J12 + M33*J13;
-                  const double R32 = M31*J21 + M32*J22 + M33*J23;
-                  const double R33 = M31*J31 + M32*J32 + M33*J33;
-
-                  // Now set y to detJ J^{-1} R = adj(J) R
-                  y(i11,qx,qy,qz,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
-                  y(i12,qx,qy,qz,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
-                  y(i13,qx,qy,qz,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
-                  y(i21,qx,qy,qz,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
-                  y(i22,qx,qy,qz,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
-                  y(i23,qx,qy,qz,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
-                  y(i31,qx,qy,qz,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
-                  y(i32,qx,qy,qz,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
-                  y(i33,qx,qy,qz,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
-               }
-               else if (coeffDim == 3)  // Vector coefficient version
-               {
-                  const double D1 = coeff(0,qx,qy,qz,e);
-                  const double D2 = coeff(1,qx,qy,qz,e);
-                  const double D3 = coeff(2,qx,qy,qz,e);
-                  // detJ J^{-1} DJ = adj(J) DJ
-                  y(i11,qx,qy,qz,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
-                  y(i12,qx,qy,qz,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
-                  y(i13,qx,qy,qz,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
-                  y(i21,qx,qy,qz,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
-                  y(i22,qx,qy,qz,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
-                  y(i23,qx,qy,qz,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
-                  y(i31,qx,qy,qz,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
-                  y(i32,qx,qy,qz,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
-                  y(i33,qx,qy,qz,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
-               }
-            }
+            // Now set y to detJ J^{-1} R = adj(J) R
+            y(i11,q,e) = w_detJ * (A11*R11 + A12*R21 + A13*R31); // 1,1
+            y(i12,q,e) = w_detJ * (A11*R12 + A12*R22 + A13*R32); // 1,2
+            y(i13,q,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
+            y(i21,q,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
+            y(i22,q,e) = w_detJ * (A21*R12 + A22*R22 + A23*R32); // 2,2
+            y(i23,q,e) = w_detJ * (A21*R13 + A22*R23 + A23*R33); // 2,3
+            y(i31,q,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
+            y(i32,q,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
+            y(i33,q,e) = w_detJ * (A31*R13 + A32*R23 + A33*R33); // 3,3
+         }
+         else if (coeffDim == 3)  // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double D3 = coeff(2, q, e);
+            // detJ J^{-1} DJ = adj(J) DJ
+            y(i11,q,e) = w_detJ * (D1*A11*J11 + D2*A12*J21 + D3*A13*J31); // 1,1
+            y(i12,q,e) = w_detJ * (D1*A11*J12 + D2*A12*J22 + D3*A13*J32); // 1,2
+            y(i13,q,e) = w_detJ * (D1*A11*J13 + D2*A12*J23 + D3*A13*J33); // 1,3
+            y(i21,q,e) = w_detJ * (D1*A21*J11 + D2*A22*J21 + D3*A23*J31); // 2,1
+            y(i22,q,e) = w_detJ * (D1*A21*J12 + D2*A22*J22 + D3*A23*J32); // 2,2
+            y(i23,q,e) = w_detJ * (D1*A21*J13 + D2*A22*J23 + D3*A23*J33); // 2,3
+            y(i31,q,e) = w_detJ * (D1*A31*J11 + D2*A32*J21 + D3*A33*J31); // 3,1
+            y(i32,q,e) = w_detJ * (D1*A31*J12 + D2*A32*J22 + D3*A33*J32); // 3,2
+            y(i33,q,e) = w_detJ * (D1*A31*J13 + D2*A32*J23 + D3*A33*J33); // 3,3
         }
      }
   });
@@ -297,61 +277,59 @@ void PAHcurlHdivSetup2D(const int Q1D,
                        Vector &_coeff,
                        Vector &op)
 {
+   const int NQ = Q1D*Q1D;
   const bool symmetric = (coeffDim != 4);
-   auto W = Reshape(_w.Read(), Q1D, Q1D);
-   auto J = Reshape(j.Read(), Q1D, Q1D, 2, 2, NE);
-   auto coeff = Reshape(_coeff.Read(), coeffDim, Q1D, Q1D, NE);
-   auto y = Reshape(op.Write(), 4, Q1D, Q1D, NE);
+   auto W = _w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto coeff = Reshape(_coeff.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), 4, NQ, NE);

   const int i11 = 0;
   const int i12 = transpose ? 2 : 1;
   const int i21 = transpose ? 1 : 2;
   const int i22 = 3;

-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL(e, NE,
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double w_detJ = W[q] / (J11*J22) - (J21*J12);
+
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
         {
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double w_detJ = W(qx,qy) / (J11*J22) - (J21*J12);
+            // First compute entries of R = MJ
+            const double M11 = coeff(i11, q, e);
+            const double M12 = (!symmetric) ? coeff(i12, q, e) : coeff(1, q, e);
+            const double M21 = (!symmetric) ? coeff(i21, q, e) : M12;
+            const double M22 = (!symmetric) ? coeff(i22, q, e) : coeff(2, q, e);

-            if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient version
-            {
-               // First compute entries of R = MJ
-               const double M11 = coeff(i11,qx,qy,e);
-               const double M12 = (!symmetric) ? coeff(i12,qx,qy,e) : coeff(1,qx,qy,e);
-               const double M21 = (!symmetric) ? coeff(i21,qx,qy,e) : M12;
-               const double M22 = (!symmetric) ? coeff(i22,qx,qy,e) : coeff(2,qx,qy,e);
+            const double R11 = M11*J11 + M12*J21;
+            const double R12 = M11*J12 + M12*J22;
+            const double R21 = M21*J11 + M22*J21;
+            const double R22 = M21*J12 + M22*J22;

-               const double R11 = M11*J11 + M12*J21;
-               const double R12 = M11*J12 + M12*J22;
-               const double R21 = M21*J11 + M22*J21;
-               const double R22 = M21*J12 + M22*J22;
-
-               // Now set y to J^{-1} R
-               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-            }
-            else if (coeffDim == 2) // Vector coefficient version
-            {
-               const double D1 = coeff(0,qx,qy,e);
-               const double D2 = coeff(1,qx,qy,e);
-               const double R11 = D1*J11;
-               const double R12 = D1*J12;
-               const double R21 = D2*J21;
-               const double R22 = D2*J22;
-               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
-               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
-            }
+            // Now set y to J^{-1} R
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
+         }
+         else if (coeffDim == 2) // Vector coefficient version
+         {
+            const double D1 = coeff(0, q, e);
+            const double D2 = coeff(1, q, e);
+            const double R11 = D1*J11;
+            const double R12 = D1*J12;
+            const double R21 = D2*J21;
+            const double R22 = D2*J22;
+            y(i11,q,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
+            y(i12,q,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+            y(i21,q,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+            y(i22,q,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
         }
      }
   });
@@ -855,13 +833,13 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   if (trial_curl && test_curl && dim == 3)
   {
-      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
-                         coeff, pa_data);
+      PAHcurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
   }
   else if (trial_curl && test_curl && dim == 2)
   {
-      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
-                            coeff, pa_data);
+      PAHcurlSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 3)
   {
@@ -903,30 +881,8 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
   {
      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23: return SmemPAHcurlMassAssembleDiagonal3D<2,3>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               case 0x34: return SmemPAHcurlMassAssembleDiagonal3D<3,4>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               case 0x45: return SmemPAHcurlMassAssembleDiagonal3D<4,5>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               case 0x56: return SmemPAHcurlMassAssembleDiagonal3D<5,6>(dofs1D, quad1D, ne,
-                                                                           symmetric,
-                                                                           mapsO->B, mapsC->B, pa_data, diag);
-               default: return SmemPAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
-                                                                    mapsO->B, mapsC->B, pa_data, diag);
-            }
-         }
-         else
-            PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
-                                          mapsO->B, mapsC->B, pa_data, diag);
+         PAHcurlMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
+                                       mapsO->B, mapsC->B, pa_data, diag);
      }
      else if (trial_fetype == mfem::FiniteElement::DIV &&
               test_fetype == trial_fetype)
@@ -970,35 +926,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   {
      if (trial_curl && test_curl)
      {
-         if (Device::Allows(Backend::DEVICE_MASK))
-         {
-            const int ID = (dofs1D << 4) | quad1D;
-            switch (ID)
-            {
-               case 0x23: return SmemPAHcurlMassApply3D<2,3>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               case 0x34: return SmemPAHcurlMassApply3D<3,4>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               case 0x45: return SmemPAHcurlMassApply3D<4,5>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               case 0x56: return SmemPAHcurlMassApply3D<5,6>(dofs1D, quad1D, ne, symmetric,
-                                                                mapsO->B,
-                                                                mapsC->B, mapsO->Bt,
-                                                                mapsC->Bt, pa_data, x, y);
-               default: return SmemPAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B,
-                                                         mapsC->B,
-                                                         mapsO->Bt, mapsC->Bt, pa_data, x, y);
-            }
-         }
-         else
-            PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
-                               mapsC->Bt, pa_data, x, y);
+         PAHcurlMassApply3D(dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B,
+                            mapsO->Bt, mapsC->Bt, pa_data, x, y);
      }
      else if (trial_div && test_div)
      {
@@ -1108,13 +1037,13 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
   // Use the same setup functions as VectorFEMassIntegrator.
   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
   {
-      PADiffusionSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                         coeff, pa_data);
+      PAHcurlSetup3D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
   }
   else if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 2)
   {
-      PADiffusionSetup2D<2>(quad1D, 1, ne, ir->GetWeights(), geom->J,
-                            coeff, pa_data);
+      PAHcurlSetup2D(quad1D, 1, ne, ir->GetWeights(), geom->J,
+                     coeff, pa_data);
   }
   else
   {
@@ -38,11 +38,11 @@ double FunctionCoefficient::Eval(ElementTransformation & T,

   if (Function)
   {
-      return Function(transip);
+      return ((*Function)(transip));
   }
   else
   {
-      return TDFunction(transip, GetTime());
+      return (*TDFunction)(transip, GetTime());
   }
 }

@@ -112,11 +112,11 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
   V.SetSize(vdim);
   if (Function)
   {
-      Function(transip, V);
+      (*Function)(transip, V);
   }
   else
   {
-      TDFunction(transip, GetTime(), V);
+      (*TDFunction)(transip, GetTime(), V);
   }
   if (Q)
   {
@@ -301,45 +301,18 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,

   K.SetSize(height, width);

-   if (symmetric) // Use SymmFunction
+   if (Function)
   {
-      MFEM_VERIFY(height == width && SymmFunction,
-                  "MatrixFunctionCoefficient is not symmetric");
-
-      Vector Ksym((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
-
-      SymmFunction(transip, Ksym);
-
-      // Copy upper triangular values from Ksym to the full matrix K
-      int os = 0;
-      for (int i=0; i<height; ++i)
-      {
-         for (int j=i; j<width; ++j)
-         {
-            const double Kij = Ksym[j - i + os];
-            K(i,j) = Kij;
-            if (j != i) { K(j,i) = Kij; }
-         }
-
-         os += width - i;
-      }
+      (*Function)(transip, K);
+   }
+   else if (TDFunction)
+   {
+      (*TDFunction)(transip, GetTime(), K);
   }
   else
   {
-      if (Function)
-      {
-         Function(transip, K);
-      }
-      else if (TDFunction)
-      {
-         TDFunction(transip, GetTime(), K);
-      }
-      else
-      {
-         K = mat;
-      }
+      K = mat;
   }
-
   if (Q)
   {
      K *= Q->Eval(T, ip, GetTime());
@@ -350,7 +323,7 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
                                              ElementTransformation &T,
                                              const IntegrationPoint &ip)
 {
-   MFEM_VERIFY(symmetric && height == width && SymmFunction,
+   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
               "MatrixFunctionCoefficient is not symmetric");

   double x[3];
@@ -362,7 +335,7 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,

   if (SymmFunction)
   {
-      SymmFunction(transip, K);
+      (*SymmFunction)(transip, K);
   }

   if (Q)
@@ -12,8 +12,6 @@
 #ifndef MFEM_COEFFICIENT
 #define MFEM_COEFFICIENT

-#include <functional>
-
 #include "../config/config.hpp"
 #include "../linalg/linalg.hpp"
 #include "intrules.hpp"
@@ -125,25 +123,28 @@ public:
                       const IntegrationPoint &ip);
 };

-/// A general function coefficient
+
+/// A general C-function coefficient
 class FunctionCoefficient : public Coefficient
 {
 protected:
-   std::function<double(const Vector &)> Function;
-   std::function<double(const Vector &, double)> TDFunction;
+   double (*Function)(const Vector &);
+   double (*TDFunction)(const Vector &, double);

 public:
-   /// Define a time-independent coefficient from a std function
-   /** \param F time-independent std::function */
-   FunctionCoefficient(std::function<double(const Vector &)> F)
-      : Function(std::move(F))
-   { }
+   /// Define a time-independent coefficient from a pointer to a C-function
+   FunctionCoefficient(double (*f)(const Vector &))
+   {
+      Function = f;
+      TDFunction = NULL;
+   }

-   /// Define a time-dependent coefficient from a std function
-   /** \param TDF time-dependent function */
-   FunctionCoefficient(std::function<double(const Vector &, double)> TDF)
-      : TDFunction(std::move(TDF))
-   { }
+   /// Define a time-dependent coefficient from a pointer to a C-function
+   FunctionCoefficient(double (*tdf)(const Vector &, double))
+   {
+      Function = NULL;
+      TDFunction = tdf;
+   }

   /// (DEPRECATED) Define a time-independent coefficient from a C-function
   /** @deprecated Use the method where the C-function, @a f, uses a const
@@ -404,34 +405,33 @@ public:
   const Vector& GetVec() { return vec; }
 };

-/// A general vector function coefficient
+/// A general C-function vector coefficient
 class VectorFunctionCoefficient : public VectorCoefficient
 {
 private:
-   std::function<void(const Vector &, Vector &)> Function;
-   std::function<void(const Vector &, double, Vector &)> TDFunction;
+   void (*Function)(const Vector &, Vector &);
+   void (*TDFunction)(const Vector &, double, Vector &);
   Coefficient *Q;

 public:
-   /// Define a time-independent vector coefficient from a std function
-   /** \param dim - the size of the vector
-       \param F - time-independent function
-       \param q - optional scalar Coefficient to scale the vector coefficient */
-   VectorFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, Vector &)> F,
-                             Coefficient *q = nullptr)
-      : VectorCoefficient(dim), Function(std::move(F)), Q(q)
-   { }
+   /// Construct a time-independent vector coefficient from a C-function
+   VectorFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
+                             Coefficient *q = NULL)
+      : VectorCoefficient(dim), Q(q)
+   {
+      Function = F;
+      TDFunction = NULL;
+   }

-   /// Define a time-dependent vector coefficient from a std function
-   /** \param dim - the size of the vector
-       \param TDF - time-dependent function
-       \param q - optional scalar Coefficient to scale the vector coefficient */
+   /// Construct a time-dependent vector coefficient from a C-function
   VectorFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, double, Vector &)> TDF,
-                             Coefficient *q = nullptr)
-      : VectorCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
-   { }
+                             void (*TDF)(const Vector &, double, Vector &),
+                             Coefficient *q = NULL)
+      : VectorCoefficient(dim), Q(q)
+   {
+      Function = NULL;
+      TDFunction = TDF;
+   }

   using VectorCoefficient::Eval;
   /// Evaluate the vector coefficient at @a ip.
@@ -721,6 +721,7 @@ public:
   /// For backward compatibility get the width of the matrix.
   int GetVDim() const { return width; }

+   void SetSymmetric(bool s) { symmetric = s; }
   bool IsSymmetric() const { return symmetric; }

   /** @brief Evaluate the matrix coefficient in the element described by @a T
@@ -761,56 +762,61 @@ public:


 /** @brief A matrix coefficient with an optional scalar coefficient multiplier
-    \a q.  The matrix function can either be represented by a std function or
-    a constant matrix provided when constructing this object.  */
+    \a q.  The matrix function can either be represented by a C-function or a
+    constant matrix provided when constructing this object.  */
 class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
-   std::function<void(const Vector &, DenseMatrix &)> Function;
-   std::function<void(const Vector &, Vector &)> SymmFunction;
-   std::function<void(const Vector &, double, DenseMatrix &)> TDFunction;
-
+   void (*Function)(const Vector &, DenseMatrix &);
+   void (*SymmFunction)(const Vector &, Vector &);
+   void (*TDFunction)(const Vector &, double, DenseMatrix &);
   Coefficient *Q;
   DenseMatrix mat;

 public:
-   /// Define a time-independent square matrix coefficient from a std function
-   /** \param dim - the size of the matrix
-       \param F - time-independent function
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, DenseMatrix &)> F,
-                             Coefficient *q = nullptr)
-      : MatrixCoefficient(dim), Function(std::move(F)), Q(q), mat(0)
-   { }
-
-   /// Define a constant matrix coefficient times a scalar Coefficient
-   /** \param m - constant matrix
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(const DenseMatrix &m, Coefficient &q)
-      : MatrixCoefficient(m.Height(), m.Width()), Q(&q), mat(m)
-   { }
-
-   /// Define a time-dependent square matrix coefficient from a std function
-   /** \param dim - the size of the matrix
-       \param TDF - time-dependent function
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, double, DenseMatrix &)> TDF,
-                             Coefficient *q = nullptr)
-      : MatrixCoefficient(dim), TDFunction(std::move(TDF)), Q(q)
-   { }
-
-   /** @brief Define a time-independent symmetric square matrix coefficient from
-       a std function */
-   /** \param dim - the size of the matrix
-       \param SymmF - function used in EvalSymmetric
-       \param q - optional scalar Coefficient to scale the matrix coefficient */
-   MatrixFunctionCoefficient(int dim,
-                             std::function<void(const Vector &, Vector &)> SymmF,
+   /// Construct a square matrix coefficient from a C-function without time
+   /// dependence.
+   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, DenseMatrix &),
                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim, true), SymmFunction(std::move(SymmF)), Q(q), mat(0)
-   { }
+      : MatrixCoefficient(dim), Q(q)
+   {
+      Function = F;
+      TDFunction = NULL;
+      mat.SetSize(0);
+   }
+
+   /// Construct a constant matrix coefficient times a scalar Coefficient
+   MatrixFunctionCoefficient(const DenseMatrix &m, Coefficient &q)
+      : MatrixCoefficient(m.Height(), m.Width()), Q(&q)
+   {
+      Function = NULL;
+      TDFunction = NULL;
+      mat = m;
+   }
+
+   /// Construct a square matrix coefficient from a C-function with
+   /// time-dependence.
+   MatrixFunctionCoefficient(int dim,
+                             void (*TDF)(const Vector &, double, DenseMatrix &),
+                             Coefficient *q = NULL)
+      : MatrixCoefficient(dim), Q(q)
+   {
+      Function = NULL;
+      TDFunction = TDF;
+      mat.SetSize(0);
+   }
+
+   /// Construct a symmetric square matrix coefficient from a C-function
+   /// defining a vector function used by EvalSymmetric
+   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
+                             Coefficient *q = NULL)
+      : MatrixCoefficient(dim, true), Q(q)
+   {
+      SymmFunction = F;
+      Function = NULL;
+      TDFunction = NULL;
+      mat.SetSize(0);
+   }

   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
@@ -883,7 +889,7 @@ public:

 /// Coefficients based on sums, products, or other functions of coefficients.
 ///@{
-/** @brief Scalar coefficient defined as the linear combination of two scalar
+/** Scalar coefficient defined as the linear combination of two scalar
    coefficients or a scalar and a scalar coefficient */
 class SumCoefficient : public Coefficient
 {
@@ -940,8 +946,8 @@ public:
   }
 };

-/** @brief Scalar coefficient defined as the product of two scalar coefficients
-    or a scalar and a scalar coefficient. */
+/** Scalar coefficient defined as the product of two scalar coefficients or
+    a scalar and a scalar coefficient. */
 class ProductCoefficient : public Coefficient
 {
 private:
@@ -979,8 +985,8 @@ public:
   { return ((a == NULL ) ? aConst : a->Eval(T, ip) ) * b->Eval(T, ip); }
 };

-/** @brief Scalar coefficient defined as the ratio of two scalars where one or
-    both scalars are scalar coefficients. */
+/** Scalar coefficient defined as the ratio of two scalars where one or both
+    scalars are scalar coefficients. */
 class RatioCoefficient : public Coefficient
 {
 private:
@@ -10,7 +10,6 @@
 // CONTRIBUTING.md for details.

 #include "complex_fem.hpp"
-#include "../general/forall.hpp"

 using namespace std;

@@ -20,21 +19,16 @@ namespace mfem
 ComplexGridFunction::ComplexGridFunction(FiniteElementSpace *fes)
   : Vector(2*(fes->GetVSize()))
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   gfr = new GridFunction();
-   gfr->MakeRef(fes, *this, 0);
-
-   gfi = new GridFunction();
-   gfi->MakeRef(fes, *this, fes->GetVSize());
+   gfr = new GridFunction(fes, data);
+   gfi = new GridFunction(fes, &data[fes->GetVSize()]);
 }

 void
 ComplexGridFunction::Update()
 {
-   FiniteElementSpace *fes = gfr->FESpace();
-   const int vsize = fes->GetVSize();
+   FiniteElementSpace * fes = gfr->FESpace();
+
+   int vsize = fes->GetVSize();

   const Operator *T = fes->GetUpdateOperator();
   if (T)
@@ -46,36 +40,30 @@ ComplexGridFunction::Update()

      // Our data array now contains old data as well as being the wrong size so
      // reallocate it.
-      UseDevice(true);
      this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);

      // Create temporary vectors which point to the new data array
-      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
-      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
+      Vector gf_r(data, vsize);
+      Vector gf_i((data) ? &data[vsize] : data, vsize);

      // Copy the updated GridFunctions into the new data array
      gf_r = *gfr;
      gf_i = *gfi;
-      gf_r.SyncAliasMemory(*this);
-      gf_i.SyncAliasMemory(*this);

      // Replace the individual data arrays with pointers into the new data
      // array
-      gfr->MakeRef(*this, 0, vsize);
-      gfi->MakeRef(*this, vsize, vsize);
+      gfr->NewDataAndSize(data, vsize);
+      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
   }
   else
   {
      // The existing data will not be transferred to the new GridFunctions so
-      // delete it and allocate a new array
-      UseDevice(true);
+      // delete it a allocate a new array
      this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);

      // Point the individual GridFunctions to the new data array
-      gfr->MakeRef(*this, 0, vsize);
-      gfi->MakeRef(*this, vsize, vsize);
+      gfr->NewDataAndSize(data, vsize);
+      gfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);

      // These updates will only set the proper 'sequence' value within the
      // individual GridFunction objects because their sizes are already correct
@@ -88,24 +76,16 @@ void
 ComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                        Coefficient &imag_coeff)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
   gfr->ProjectCoefficient(real_coeff);
   gfi->ProjectCoefficient(imag_coeff);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }

 void
 ComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                        VectorCoefficient &imag_vcoeff)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
   gfr->ProjectCoefficient(real_vcoeff);
   gfi->ProjectCoefficient(imag_vcoeff);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }

 void
@@ -113,12 +93,8 @@ ComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                           Coefficient &imag_coeff,
                                           Array<int> &attr)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
   gfr->ProjectBdrCoefficient(real_coeff, attr);
   gfi->ProjectBdrCoefficient(imag_coeff, attr);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }

 void
@@ -126,12 +102,8 @@ ComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient &real_vcoeff,
                                                 VectorCoefficient &imag_vcoeff,
                                                 Array<int> &attr)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
   gfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
   gfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }

 void
@@ -141,28 +113,18 @@ ComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                  &imag_vcoeff,
                                                  Array<int> &attr)
 {
-   gfr->SyncMemory(*this);
-   gfi->SyncMemory(*this);
   gfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
   gfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
-   gfr->SyncAliasMemory(*this);
-   gfi->SyncAliasMemory(*this);
 }


-ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
+ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *f,
                                     ComplexOperator::Convention convention)
-   : Vector(2*(fes->GetVSize())),
+   : Vector(2*(f->GetVSize())),
     conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   lfr = new LinearForm();
-   lfr->MakeRef(fes, *this, 0);
-
-   lfi = new LinearForm();
-   lfi->MakeRef(fes, *this, fes->GetVSize());
+   lfr = new LinearForm(f, data);
+   lfi = new LinearForm(f, &data[f->GetVSize()]);
 }

 ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
@@ -171,14 +133,8 @@ ComplexLinearForm::ComplexLinearForm(FiniteElementSpace *fes,
   : Vector(2*(fes->GetVSize())),
     conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   lfr = new LinearForm(fes, lf_r);
-   lfi = new LinearForm(fes, lf_i);
-
-   lfr->MakeRef(fes, *this, 0);
-   lfi->MakeRef(fes, *this, fes->GetVSize());
+   lfr = new LinearForm(fes, lf_r);  lfr->SetData(data);
+   lfi = new LinearForm(fes, lf_i);  lfi->SetData(&data[fes->GetVSize()]);
 }

 ComplexLinearForm::~ComplexLinearForm()
@@ -233,43 +189,42 @@ void
 ComplexLinearForm::Update()
 {
   FiniteElementSpace *fes = lfr->FESpace();
+
   this->Update(fes);
 }

 void
 ComplexLinearForm::Update(FiniteElementSpace *fes)
 {
-   UseDevice(true);
-   SetSize(2 * fes->GetVSize());
-   this->Vector::operator=(0.0);
+   int vsize = fes->GetVSize();
+   SetSize(2 * vsize);

-   lfr->MakeRef(fes, *this, 0);
-   lfi->MakeRef(fes, *this, fes->GetVSize());
+   Vector vlfr(data, vsize);
+   Vector vlfi((data) ? &data[vsize] : data, vsize);
+
+   lfr->Update(fes, vlfr, 0);
+   lfi->Update(fes, vlfi, 0);
 }

 void
 ComplexLinearForm::Assemble()
 {
-   lfr->SyncMemory(*this);
-   lfi->SyncMemory(*this);
   lfr->Assemble();
   lfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *lfi *= -1.0; }
-   lfr->SyncAliasMemory(*this);
-   lfi->SyncAliasMemory(*this);
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      *lfi *= -1.0;
+   }
 }

 complex<double>
 ComplexLinearForm::operator()(const ComplexGridFunction &gf) const
 {
-   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
-   lfr->SyncMemory(*this);
-   lfi->SyncMemory(*this);
+   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
   return complex<double>((*lfr)(gf.real()) - s * (*lfi)(gf.imag()),
                          (*lfr)(gf.imag()) + s * (*lfi)(gf.real()));
 }

-
 bool SesquilinearForm::RealInteg()
 {
   int nint = blfr->GetFBFI()->Size() + blfr->GetDBFI()->Size() +
@@ -386,45 +341,34 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                   Vector &X, Vector &B,
                                   int ci)
 {
-   FiniteElementSpace *fes = blfr->FESpace();
-   const int vsize = fes->GetVSize();
+   FiniteElementSpace * fes = blfr->FESpace();
+   int vsize  = fes->GetVSize();

-   // Allocate temporary vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize);
-   b_0 = 0.0;
+   // Allocate temporary vectors
+   Vector b_0(vsize); b_0 = 0.0;

   // Extract the real and imaginary parts of the input vectors
   MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);

   MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r; b_r.MakeRef(b, 0, vsize);
-   Vector b_i; b_i.MakeRef(b, vsize, vsize);
+   Vector b_r(b.GetData(), vsize);
+   Vector b_i(&(b.GetData())[vsize], vsize);

   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }

-   const int tvsize = fes->GetTrueVSize();
+   int tvsize = fes->GetTrueVSize();
   OperatorHandle A_r, A_i;

-   X.UseDevice(true);
   X.SetSize(2 * tvsize);
-   X = 0.0;
-
-   B.UseDevice(true);
   B.SetSize(2 * tvsize);
-   B = 0.0;

-   Vector X_r; X_r.MakeRef(X, 0, tvsize);
-   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
-   Vector B_r; B_r.MakeRef(B, 0, tvsize);
-   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
-
-   Vector X_0, B_0;
+   Vector X_0(tvsize), B_0(tvsize);
+   Vector X_r(X.GetData(),tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);

   if (RealInteg())
   {
@@ -474,18 +418,13 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
      // conform with standard essential BC treatment
      if (A_i.Is<ConstrainedOperator>())
      {
-         const int n = ess_tdof_list.Size();
-         auto d_B_r = B_r.Write();
-         auto d_B_i = B_i.Write();
-         auto d_X_r = X_r.Read();
-         auto d_X_i = X_i.Read();
-         auto d_idx = ess_tdof_list.Read();
-         MFEM_FORALL(i, n,
+         int n = ess_tdof_list.Size();
+         for (int k = 0; k < n; k++)
         {
-            const int j = d_idx[i];
-            d_B_r[j] = d_X_r[j];
-            d_B_i[j] = d_X_i[j];
-         });
+            int j = ess_tdof_list[k];
+            B_r(j) = X_r(j);
+            B_i(j) = X_i(j);
+         }
         A_i.As<ConstrainedOperator>()->SetDiagonalPolicy
         (mfem::Operator::DiagonalPolicy::DIAG_ZERO);
      }
@@ -497,16 +436,6 @@ SesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
      b_i *= -1.0;
   }

-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
   // A = A_r + i A_i
   A.Clear();
   if ( A_r.Type() == Operator::MFEM_SPARSEMAT ||
@@ -599,32 +528,29 @@ void
 SesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                     Vector &x)
 {
-   FiniteElementSpace *fes = blfr->FESpace();
+   FiniteElementSpace * fes = blfr->FESpace();

   const SparseMatrix *P = fes->GetConformingProlongation();
+
+   int vsize  = fes->GetVSize();
+   int tvsize = X.Size() / 2;
+
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);
+
   if (!P)
   {
      x = X;
-      return;
   }
-
-   const int vsize  = fes->GetVSize();
-   const int tvsize = X.Size() / 2;
-
-   X.Read();
-   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
-   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
-
-   x.Write();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
-
-   // Apply conforming prolongation
-   P->Mult(X_r, x_r);
-   P->Mult(X_i, x_i);
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
+   else
+   {
+      // Apply conforming prolongation
+      P->Mult(X_r, x_r);
+      P->Mult(X_i, x_i);
+   }
 }

 void
@@ -640,21 +566,16 @@ SesquilinearForm::Update(FiniteElementSpace *nfes)
 ParComplexGridFunction::ParComplexGridFunction(ParFiniteElementSpace *pfes)
   : Vector(2*(pfes->GetVSize()))
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
-   pgfr = new ParGridFunction();
-   pgfr->MakeRef(pfes, *this, 0);
-
-   pgfi = new ParGridFunction();
-   pgfi->MakeRef(pfes, *this, pfes->GetVSize());
+   pgfr = new ParGridFunction(pfes, data);
+   pgfi = new ParGridFunction(pfes, (data) ? &data[pfes->GetVSize()]:data);
 }

 void
 ParComplexGridFunction::Update()
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
-   const int vsize = pfes->GetVSize();
+   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
+
+   int vsize = pfes->GetVSize();

   const Operator *T = pfes->GetUpdateOperator();
   if (T)
@@ -666,34 +587,30 @@ ParComplexGridFunction::Update()

      // Our data array now contains old data as well as being the wrong size so
      // reallocate it.
-      UseDevice(true);
      this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);

      // Create temporary vectors which point to the new data array
-      Vector gf_r; gf_r.MakeRef(*this, 0, vsize);
-      Vector gf_i; gf_i.MakeRef(*this, vsize, vsize);
+      Vector gf_r(data, vsize);
+      Vector gf_i((data) ? &data[vsize] : data, vsize);

      // Copy the updated GridFunctions into the new data array
-      gf_r = *pgfr; gf_r.SyncAliasMemory(*this);
-      gf_i = *pgfi; gf_i.SyncAliasMemory(*this);
+      gf_r = *pgfr;
+      gf_i = *pgfi;

      // Replace the individual data arrays with pointers into the new data
      // array
-      pgfr->MakeRef(*this, 0, vsize);
-      pgfi->MakeRef(*this, vsize, vsize);
+      pgfr->NewDataAndSize(data, vsize);
+      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);
   }
   else
   {
      // The existing data will not be transferred to the new GridFunctions so
-      // delete it and allocate a new array
-      UseDevice(true);
+      // delete it a allocate a new array
      this->SetSize(2 * vsize);
-      this->Vector::operator=(0.0);

      // Point the individual GridFunctions to the new data array
-      pgfr->MakeRef(*this, 0, vsize);
-      pgfi->MakeRef(*this, vsize, vsize);
+      pgfr->NewDataAndSize(data, vsize);
+      pgfi->NewDataAndSize((data) ? &data[vsize] : data, vsize);

      // These updates will only set the proper 'sequence' value within the
      // individual GridFunction objects because their sizes are already correct
@@ -706,24 +623,16 @@ void
 ParComplexGridFunction::ProjectCoefficient(Coefficient &real_coeff,
                                           Coefficient &imag_coeff)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->ProjectCoefficient(real_coeff);
   pgfi->ProjectCoefficient(imag_coeff);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }

 void
 ParComplexGridFunction::ProjectCoefficient(VectorCoefficient &real_vcoeff,
                                           VectorCoefficient &imag_vcoeff)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->ProjectCoefficient(real_vcoeff);
   pgfi->ProjectCoefficient(imag_vcoeff);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }

 void
@@ -731,12 +640,8 @@ ParComplexGridFunction::ProjectBdrCoefficient(Coefficient &real_coeff,
                                              Coefficient &imag_coeff,
                                              Array<int> &attr)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->ProjectBdrCoefficient(real_coeff, attr);
   pgfi->ProjectBdrCoefficient(imag_coeff, attr);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }

 void
@@ -746,12 +651,8 @@ ParComplexGridFunction::ProjectBdrCoefficientNormal(VectorCoefficient
                                                    &imag_vcoeff,
                                                    Array<int> &attr)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->ProjectBdrCoefficientNormal(real_vcoeff, attr);
   pgfi->ProjectBdrCoefficientNormal(imag_vcoeff, attr);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }

 void
@@ -761,51 +662,36 @@ ParComplexGridFunction::ProjectBdrCoefficientTangent(VectorCoefficient
                                                     &imag_vcoeff,
                                                     Array<int> &attr)
 {
-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->ProjectBdrCoefficientTangent(real_vcoeff, attr);
   pgfi->ProjectBdrCoefficientTangent(imag_vcoeff, attr);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }

 void
 ParComplexGridFunction::Distribute(const Vector *tv)
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
-   const int tvsize = pfes->GetTrueVSize();
+   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
+   HYPRE_Int size = pfes->GetTrueVSize();

-   tv->Read();
-   Vector tvr; tvr.MakeRef(const_cast<Vector&>(*tv), 0, tvsize);
-   Vector tvi; tvi.MakeRef(const_cast<Vector&>(*tv), tvsize, tvsize);
+   double * tvd = tv->GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);

-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->Distribute(tvr);
   pgfi->Distribute(tvi);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
 }

 void
 ParComplexGridFunction::ParallelProject(Vector &tv) const
 {
-   ParFiniteElementSpace *pfes = pgfr->ParFESpace();
-   const int tvsize = pfes->GetTrueVSize();
+   ParFiniteElementSpace * pfes = pgfr->ParFESpace();
+   HYPRE_Int size = pfes->GetTrueVSize();

-   tv.Write();
-   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
-   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
+   double * tvd = tv.GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);

-   pgfr->SyncMemory(*this);
-   pgfi->SyncMemory(*this);
   pgfr->ParallelProject(tvr);
   pgfi->ParallelProject(tvi);
-   pgfr->SyncAliasMemory(*this);
-   pgfi->SyncAliasMemory(*this);
-
-   tvr.SyncAliasMemory(tv);
-   tvi.SyncAliasMemory(tv);
 }


@@ -815,16 +701,10 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
   : Vector(2*(pfes->GetVSize())),
     conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
+   plfr = new ParLinearForm(pfes, data);
+   plfi = new ParLinearForm(pfes, (data) ? &data[pfes->GetVSize()]:data);

-   plfr = new ParLinearForm();
-   plfr->MakeRef(pfes, *this, 0);
-
-   plfi = new ParLinearForm();
-   plfi->MakeRef(pfes, *this, pfes->GetVSize());
-
-   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();

   int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
   tdof_offsets = new HYPRE_Int[n+1];
@@ -844,16 +724,12 @@ ParComplexLinearForm::ParComplexLinearForm(ParFiniteElementSpace *pfes,
   : Vector(2*(pfes->GetVSize())),
     conv(convention)
 {
-   UseDevice(true);
-   this->Vector::operator=(0.0);
-
   plfr = new ParLinearForm(pfes, plf_r);
+   plfr->SetData(data);
   plfi = new ParLinearForm(pfes, plf_i);
+   plfi->SetData((data) ? &data[pfes->GetVSize()]:data);

-   plfr->MakeRef(pfes, *this, 0);
-   plfi->MakeRef(pfes, *this, pfes->GetVSize());
-
-   HYPRE_Int *tdof_offsets_fes = pfes->GetTrueDofOffsets();
+   HYPRE_Int * tdof_offsets_fes = pfes->GetTrueDofOffsets();

   int n = (HYPRE_AssumedPartitionCheck()) ? 2 : pfes->GetNRanks();
   tdof_offsets = new HYPRE_Int[n+1];
@@ -916,71 +792,58 @@ ParComplexLinearForm::AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
 void
 ParComplexLinearForm::Update(ParFiniteElementSpace *pf)
 {
-   ParFiniteElementSpace *pfes = (pf != NULL) ? pf : plfr->ParFESpace();
+   ParFiniteElementSpace *pfes = (pf!=NULL)?pf:plfr->ParFESpace();
+   int vsize = pfes->GetVSize();
+   SetSize(2 * vsize);

-   UseDevice(true);
-   SetSize(2 * pfes->GetVSize());
-   this->Vector::operator=(0.0);
+   Vector vplfr(data, vsize);
+   Vector vplfi((data) ? &data[vsize] : data, vsize);

-   plfr->MakeRef(pfes, *this, 0);
-   plfi->MakeRef(pfes, *this, pfes->GetVSize());
+   plfr->Update(pfes, vplfr, 0);
+   plfi->Update(pfes, vplfi, 0);
 }

 void
 ParComplexLinearForm::Assemble()
 {
-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
   plfr->Assemble();
   plfi->Assemble();
-   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { *plfi *= -1.0; }
-   plfr->SyncAliasMemory(*this);
-   plfi->SyncAliasMemory(*this);
+   if (conv == ComplexOperator::BLOCK_SYMMETRIC)
+   {
+      *plfi *= -1.0;
+   }
 }

 void
 ParComplexLinearForm::ParallelAssemble(Vector &tv)
 {
-   const int tvsize = plfr->ParFESpace()->GetTrueVSize();
+   HYPRE_Int size = plfr->ParFESpace()->GetTrueVSize();

-   tv.Write();
-   Vector tvr; tvr.MakeRef(tv, 0, tvsize);
-   Vector tvi; tvi.MakeRef(tv, tvsize, tvsize);
+   double * tvd = tv.GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);

-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
   plfr->ParallelAssemble(tvr);
   plfi->ParallelAssemble(tvi);
-   plfr->SyncAliasMemory(*this);
-   plfi->SyncAliasMemory(*this);
-
-   tvr.SyncAliasMemory(tv);
-   tvi.SyncAliasMemory(tv);
 }

 HypreParVector *
 ParComplexLinearForm::ParallelAssemble()
 {
-   const ParFiniteElementSpace *pfes = plfr->ParFESpace();
-   const int tvsize = pfes->GetTrueVSize();
+   const ParFiniteElementSpace * pfes = plfr->ParFESpace();

-   HypreParVector *tv = new HypreParVector(pfes->GetComm(),
-                                           2*(pfes->GlobalTrueVSize()),
-                                           tdof_offsets);
+   HypreParVector * tv = new HypreParVector(pfes->GetComm(),
+                                            2*(pfes->GlobalTrueVSize()),
+                                            tdof_offsets);

-   tv->Write();
-   Vector tvr; tvr.MakeRef(*tv, 0, tvsize);
-   Vector tvi; tvi.MakeRef(*tv, tvsize, tvsize);
+   HYPRE_Int size = pfes->GetTrueVSize();
+
+   double * tvd = tv->GetData();
+   Vector tvr(tvd, size);
+   Vector tvi((tvd) ? &tvd[size] : tvd, size);

-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
   plfr->ParallelAssemble(tvr);
   plfi->ParallelAssemble(tvi);
-   plfr->SyncAliasMemory(*this);
-   plfi->SyncAliasMemory(*this);
-
-   tvr.SyncAliasMemory(*tv);
-   tvi.SyncAliasMemory(*tv);

   return tv;
 }
@@ -988,14 +851,13 @@ ParComplexLinearForm::ParallelAssemble()
 complex<double>
 ParComplexLinearForm::operator()(const ParComplexGridFunction &gf) const
 {
-   plfr->SyncMemory(*this);
-   plfi->SyncMemory(*this);
-   double s = (conv == ComplexOperator::HERMITIAN) ? 1.0 : -1.0;
+   double s = (conv == ComplexOperator::HERMITIAN)?1.0:-1.0;
   return complex<double>((*plfr)(gf.real()) - s * (*plfi)(gf.imag()),
                          (*plfr)(gf.imag()) + s * (*plfi)(gf.real()));
 }


+
 bool ParSesquilinearForm::RealInteg()
 {
   int nint = pblfr->GetFBFI()->Size() + pblfr->GetDBFI()->Size() +
@@ -1102,6 +964,7 @@ ParSesquilinearForm::ParallelAssemble()
   return new ComplexHypreParMatrix(pblfr->ParallelAssemble(),
                                    pblfi->ParallelAssemble(),
                                    true, true, conv);
+
 }

 void
@@ -1111,45 +974,35 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
                                      Vector &X, Vector &B,
                                      int ci)
 {
-   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
-   const int vsize = pfes->GetVSize();
+   ParFiniteElementSpace * pfes = pblfr->ParFESpace();
+   int vsize = pfes->GetVSize();

-   // Allocate temporary vector
-   Vector b_0;
-   b_0.UseDevice(true);
-   b_0.SetSize(vsize);
-   b_0 = 0.0;
+   // Allocate temporary vectors
+   Vector b_0(vsize); b_0 = 0.0;

   // Extract the real and imaginary parts of the input vectors
   MFEM_ASSERT(x.Size() == 2 * vsize, "Input GridFunction of incorrect size!");
-   x.Read();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);

   MFEM_ASSERT(b.Size() == 2 * vsize, "Input LinearForm of incorrect size!");
-   b.Read();
-   Vector b_r; b_r.MakeRef(b, 0, vsize);
-   Vector b_i; b_i.MakeRef(b, vsize, vsize);
+   Vector b_r(b.GetData(), vsize);
+   Vector b_i(&(b.GetData())[vsize], vsize);

   if (conv == ComplexOperator::BLOCK_SYMMETRIC) { b_i *= -1.0; }

-   const int tvsize = pfes->GetTrueVSize();
+   int tvsize = pfes->GetTrueVSize();
+
   OperatorHandle A_r, A_i;

-   X.UseDevice(true);
   X.SetSize(2 * tvsize);
-   X = 0.0;
-
-   B.UseDevice(true);
   B.SetSize(2 * tvsize);
-   B = 0.0;

-   Vector X_r; X_r.MakeRef(X, 0, tvsize);
-   Vector X_i; X_i.MakeRef(X, tvsize, tvsize);
-   Vector B_r; B_r.MakeRef(B, 0, tvsize);
-   Vector B_i; B_i.MakeRef(B, tvsize, tvsize);
-
-   Vector X_0, B_0;
+   Vector X_0(tvsize), B_0(tvsize);
+   Vector X_r(X.GetData(),tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);
+   Vector B_r(B.GetData(), tvsize);
+   Vector B_i(&(B.GetData())[tvsize], tvsize);

   if (RealInteg())
   {
@@ -1189,29 +1042,24 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,

   if (RealInteg() && ImagInteg())
   {
+      int n = ess_tdof_list.Size();
      // Modify RHS to conform with standard essential BC treatment
-      const int n = ess_tdof_list.Size();
-      auto d_B_r = B_r.Write();
-      auto d_B_i = B_i.Write();
-      auto d_X_r = X_r.Read();
-      auto d_X_i = X_i.Read();
-      auto d_idx = ess_tdof_list.Read();
-      MFEM_FORALL(i, n,
+      for (int k = 0; k < n; k++)
      {
-         const int j = d_idx[i];
-         d_B_r[j] = d_X_r[j];
-         d_B_i[j] = d_X_i[j];
-      });
+         int j=ess_tdof_list[k];
+         B_r(j) = X_r(j);
+         B_i(j) = X_i(j);
+      }
      // Modify offdiagonal blocks (imaginary parts of the matrix) to conform
      // with standard essential BC treatment
-      if (A_i.Type() == Operator::Hypre_ParCSR)
+      if ( A_i.Type() == Operator::Hypre_ParCSR )
      {
         HypreParMatrix * Ah;
         A_i.Get(Ah);
         hypre_ParCSRMatrix *Aih = *Ah;
         for (int k = 0; k < n; k++)
         {
-            const int j = ess_tdof_list[k];
+            int j = ess_tdof_list[k];
            Aih->diag->data[Aih->diag->i[j]] = 0.0;
         }
      }
@@ -1228,16 +1076,6 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
      b_i *= -1.0;
   }

-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
-   b_r.SyncAliasMemory(b);
-   b_i.SyncAliasMemory(b);
-
-   X_r.SyncAliasMemory(X);
-   X_i.SyncAliasMemory(X);
-   B_r.SyncAliasMemory(B);
-   B_i.SyncAliasMemory(B);
-
   // A = A_r + i A_i
   A.Clear();
   if ( A_r.Type() == Operator::Hypre_ParCSR ||
@@ -1337,27 +1175,22 @@ void
 ParSesquilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
                                        Vector &x)
 {
-   ParFiniteElementSpace *pfes = pblfr->ParFESpace();
+   ParFiniteElementSpace * pfes = pblfr->ParFESpace();

   const Operator &P = *pfes->GetProlongationMatrix();

-   const int vsize  = pfes->GetVSize();
-   const int tvsize = X.Size() / 2;
+   int vsize  = pfes->GetVSize();
+   int tvsize = X.Size() / 2;

-   X.Read();
-   Vector X_r; X_r.MakeRef(const_cast<Vector&>(X), 0, tvsize);
-   Vector X_i; X_i.MakeRef(const_cast<Vector&>(X), tvsize, tvsize);
+   Vector X_r(X.GetData(), tvsize);
+   Vector X_i(&(X.GetData())[tvsize], tvsize);

-   x.Write();
-   Vector x_r; x_r.MakeRef(x, 0, vsize);
-   Vector x_i; x_i.MakeRef(x, vsize, vsize);
+   Vector x_r(x.GetData(), vsize);
+   Vector x_i(&(x.GetData())[vsize], vsize);

   // Apply conforming prolongation
   P.Mult(X_r, x_r);
   P.Mult(X_i, x_i);
-
-   x_r.SyncAliasMemory(x);
-   x_i.SyncAliasMemory(x);
 }

 void
@@ -38,8 +38,8 @@ protected:
   void Destroy() { delete gfr; delete gfi; }

 public:
-   /** @brief Construct a ComplexGridFunction associated with the
-       FiniteElementSpace @a *f. */
+   /* @brief Construct a ComplexGridFunction associated with the
+      FiniteElementSpace @a *f. */
   ComplexGridFunction(FiniteElementSpace *f);

   void Update();
@@ -71,14 +71,6 @@ public:
   const GridFunction & real() const { return *gfr; }
   const GridFunction & imag() const { return *gfi; }

-   /// Update the memory location of the real and imaginary GridFunction @a gfr
-   /// and @a gfi to match the ComplexGridFunction.
-   void Sync() { gfr->SyncMemory(*this); gfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary GridFunction
-   /// @a gfr and @a gfi to match the ComplexGridFunction.
-   void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); }
-
   /// Destroys the grid function.
   virtual ~ComplexGridFunction() { Destroy(); }

@@ -107,8 +99,8 @@ public:
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);

-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
-       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .

       The pointer @a fes is not owned by the newly constructed object.

@@ -165,14 +157,6 @@ public:
   const LinearForm & real() const { return *lfr; }
   const LinearForm & imag() const { return *lfi; }

-   /// Update the memory location of the real and imaginary LinearForm @a lfr
-   /// and @a lfi to match the ComplexLinearForm.
-   void Sync() { lfr->SyncMemory(*this); lfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary LinearForm @a
-   /// lfr and @a lfi to match the ComplexLinearForm.
-   void SyncAlias() { lfr->SyncAliasMemory(*this); lfi->SyncAliasMemory(*this); }
-
   void Update();
   void Update(FiniteElementSpace *f);

@@ -211,8 +195,8 @@ private:
   BilinearForm *blfr;
   BilinearForm *blfi;

-   /* These methods check if the real/imag parts of the sesquilinear form are
-      not empty */
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
   bool RealInteg();
   bool ImagInteg();

@@ -220,7 +204,7 @@ public:
   SesquilinearForm(FiniteElementSpace *fes,
                    ComplexOperator::Convention
                    convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
       the same integrators as the BilinearForms @a bfr and @a bfi .

       The pointer @a fes is not owned by the newly constructed object.
@@ -238,8 +222,7 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
+       - AssemblyLevel::FULL  (default)
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE
@@ -340,8 +323,8 @@ protected:

 public:

-   /** @brief Construct a ParComplexGridFunction associated with the
-       ParFiniteElementSpace @a *pf. */
+   /* @brief Construct a ParComplexGridFunction associated with the
+      ParFiniteElementSpace @a *f. */
   ParComplexGridFunction(ParFiniteElementSpace *pf);

   void Update();
@@ -382,15 +365,6 @@ public:
   const ParGridFunction & real() const { return *pgfr; }
   const ParGridFunction & imag() const { return *pgfi; }

-   /// Update the memory location of the real and imaginary ParGridFunction @a
-   /// pgfr and @a pgfi to match the ParComplexGridFunction.
-   void Sync() { pgfr->SyncMemory(*this); pgfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary
-   /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction.
-   void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
-
-
   virtual double ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
                                 const IntegrationRule *irs[] = NULL) const
   {
@@ -442,8 +416,8 @@ public:
                        convention = ComplexOperator::HERMITIAN);

   /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plf_r (real) and
-       @a plf_i (imag).
+       using the same integrators as the LinearForms @a plfr (real) and @a plfi
+       (imag) .

      The pointer @a fes is not owned by the newly constructed object.

@@ -501,14 +475,6 @@ public:
   const ParLinearForm & real() const { return *plfr; }
   const ParLinearForm & imag() const { return *plfi; }

-   /// Update the memory location of the real and imaginary ParLinearForm @a lfr
-   /// and @a lfi to match the ParComplexLinearForm.
-   void Sync() { plfr->SyncMemory(*this); plfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary ParLinearForm
-   /// @a plfr and @a plfi to match the ParComplexLinearForm.
-   void SyncAlias() { plfr->SyncAliasMemory(*this); plfi->SyncAliasMemory(*this); }
-
   void Update(ParFiniteElementSpace *pf = NULL);

   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
@@ -576,8 +542,7 @@ public:
   /// Set the desired assembly level.
   /** Valid choices are:

-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
+       - AssemblyLevel::FULL  (default)
       - AssemblyLevel::PARTIAL
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE
@@ -1,297 +0,0 @@
-#include "convergence.hpp"
-
-using namespace std;
-
-
-namespace mfem
-{
-
-void ConvergenceStudy::Reset()
-{
-   counter=0;
-   dcounter=0;
-   fcounter=0;
-   cont_type=-1;
-   print_flag=1;
-   L2Errors.SetSize(0);
-   L2Rates.SetSize(0);
-   DErrors.SetSize(0);
-   DRates.SetSize(0);
-   EnErrors.SetSize(0);
-   EnRates.SetSize(0);
-   DGFaceErrors.SetSize(0);
-   DGFaceRates.SetSize(0);
-   ndofs.SetSize(0);
-}
-
-double ConvergenceStudy::GetNorm(GridFunction *gf, Coefficient *scalar_u,
-                                 VectorCoefficient *vector_u)
-{
-   bool norm_set = false;
-   double norm=0.0;
-   int order = gf->FESpace()->GetOrder(0);
-   int order_quad = std::max(2, 2*order+1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-#ifdef MFEM_USE_MPI
-   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
-   if (pgf)
-   {
-      ParMesh *pmesh = pgf->ParFESpace()->GetParMesh();
-      if (scalar_u)
-      {
-         norm = ComputeGlobalLpNorm(2.0,*scalar_u,*pmesh,irs);
-      }
-      else if (vector_u)
-      {
-         norm = ComputeGlobalLpNorm(2.0,*vector_u,*pmesh,irs);
-      }
-      norm_set = true;
-   }
-#endif
-   if (!norm_set)
-   {
-      Mesh *mesh = gf->FESpace()->GetMesh();
-      if (scalar_u)
-      {
-         norm = ComputeLpNorm(2.0,*scalar_u,*mesh,irs);
-      }
-      else if (vector_u)
-      {
-         norm = ComputeLpNorm(2.0,*vector_u,*mesh,irs);
-      }
-   }
-   return norm;
-}
-
-void ConvergenceStudy::AddL2Error(GridFunction *gf,
-                                  Coefficient *scalar_u, VectorCoefficient *vector_u)
-{
-   int tdofs=0;
-#ifdef MFEM_USE_MPI
-   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
-   if (pgf)
-   {
-      MPI_Comm comm = pgf->ParFESpace()->GetComm();
-      int rank;
-      MPI_Comm_rank(comm, &rank);
-      print_flag = 0;
-      if (rank==0) { print_flag = 1; }
-      tdofs = pgf->ParFESpace()->GlobalTrueVSize();
-   }
-#endif
-   if (!tdofs) { tdofs = gf->FESpace()->GetTrueVSize(); }
-   ndofs.Append(tdofs);
-   double L2Err;
-   if (scalar_u)
-   {
-      L2Err = gf->ComputeL2Error(*scalar_u);
-      CoeffNorm = GetNorm(gf,scalar_u,nullptr);
-   }
-   else if (vector_u)
-   {
-      L2Err = gf->ComputeL2Error(*vector_u);
-      CoeffNorm = GetNorm(gf,nullptr,vector_u);
-   }
-   else
-   {
-      MFEM_ABORT("Exact Solution Coefficient pointer is NULL");
-   }
-   L2Errors.Append(L2Err);
-   // Compute the rate of convergence by:
-   // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-   double val = (counter) ? log(L2Errors[counter-1]/L2Err)/log(2.0) : 0.0;
-   L2Rates.Append(val);
-   counter++;
-}
-
-void ConvergenceStudy::AddGf(GridFunction *gf, Coefficient *scalar_u,
-                             VectorCoefficient *grad,
-                             Coefficient *ell_coeff, double Nu)
-{
-   cont_type = gf->FESpace()->FEColl()->GetContType();
-
-   MFEM_VERIFY((cont_type == mfem::FiniteElementCollection::CONTINUOUS) ||
-               (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS),
-               "This constructor is intended for H1 or L2 Elements")
-
-   AddL2Error(gf,scalar_u, nullptr);
-
-   if (grad)
-   {
-      double GradErr = gf->ComputeGradError(grad);
-      DErrors.Append(GradErr);
-      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1]+GradErr*GradErr);
-      EnErrors.Append(err);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val = (dcounter) ? log(DErrors[dcounter-1]/GradErr)/log(2.0) : 0.0;
-      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
-      DRates.Append(val);
-      EnRates.Append(eval);
-      CoeffDNorm = GetNorm(gf,nullptr,grad);
-      dcounter++;
-      MFEM_VERIFY(counter == dcounter,
-                  "Number of added solutions and derivatives do not match")
-   }
-
-   if (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS && ell_coeff)
-   {
-      double DGErr = gf->ComputeDGFaceJumpError(scalar_u,ell_coeff,Nu);
-      DGFaceErrors.Append(DGErr);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val=(fcounter) ? log(DGFaceErrors[fcounter-1]/DGErr)/log(2.0):0.;
-      DGFaceRates.Append(val);
-      fcounter++;
-      MFEM_VERIFY(fcounter == counter, "Number of added solutions mismatch");
-   }
-}
-
-void ConvergenceStudy::AddGf(GridFunction *gf, VectorCoefficient *vector_u,
-                             VectorCoefficient *curl, Coefficient *div)
-{
-   cont_type = gf->FESpace()->FEColl()->GetContType();
-
-   AddL2Error(gf,nullptr,vector_u);
-   double DErr = 0.0;
-   bool derivative = false;
-   if (curl)
-   {
-      DErr = gf->ComputeCurlError(curl);
-      CoeffDNorm = GetNorm(gf,nullptr,curl);
-      derivative = true;
-   }
-   else if (div)
-   {
-      DErr = gf->ComputeDivError(div);
-      // update coefficient norm
-      CoeffDNorm = GetNorm(gf,div,nullptr);
-      derivative = true;
-   }
-   if (derivative)
-   {
-      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1] + DErr*DErr);
-      DErrors.Append(DErr);
-      EnErrors.Append(err);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val = (dcounter) ? log(DErrors[dcounter-1]/DErr)/log(2.0) : 0.0;
-      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
-      DRates.Append(val);
-      EnRates.Append(eval);
-      dcounter++;
-      MFEM_VERIFY(counter == dcounter,
-                  "Number of added solutions and derivatives do not match")
-   }
-}
-
-void ConvergenceStudy::Print(bool relative, std::ostream &out)
-{
-   if (print_flag)
-   {
-      std::string title = (relative) ? "Relative " : "Absolute ";
-      out << "\n";
-      out << " -------------------------------------------" << "\n";
-      out <<  std::setw(21) << title << "L2 Error " << "\n";
-      out << " -------------------------------------------"
-          << "\n";
-      out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13) << "Error ";
-      out <<  std::setw(15) << "Rate " << "\n";
-      out << " -------------------------------------------"
-          << "\n";
-      out << std::setprecision(4);
-      double d = (relative) ? CoeffNorm : 1.0;
-      for (int i =0; i<counter; i++)
-      {
-         out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-             << std::scientific << L2Errors[i]/d << std::setw(13)
-             << std::fixed << L2Rates[i] << "\n";
-      }
-      out << "\n";
-      if (dcounter == counter)
-      {
-         std::string dname;
-         switch (cont_type)
-         {
-            case 0: dname = "Grad"; break;
-            case 1: dname = "Curl"; break;
-            case 2: dname = "Div";  break;
-            case 3: dname = "DG Grad";  break;
-            default: break;
-         }
-         out << " -------------------------------------------" << "\n";
-         out <<  std::setw(21) << title << dname << " Error  " << "\n";
-         out << " -------------------------------------------" << "\n";
-         out << std::right<<std::setw(11)<< "DOFs "<< std::setw(13) << "Error";
-         out << std::setw(15) << "Rate " << "\n";
-         out << " -------------------------------------------"
-             << "\n";
-         out << std::setprecision(4);
-         d = (relative) ? CoeffDNorm : 1.0;
-         for (int i =0; i<dcounter; i++)
-         {
-            out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                << std::scientific << DErrors[i]/d << std::setw(13)
-                << std::fixed << DRates[i] << "\n";
-         }
-         out << "\n";
-         switch (cont_type)
-         {
-            case 0: dname = "H1"; break;
-            case 1: dname = "H(Curl)"; break;
-            case 2: dname = "H(Div)";  break;
-            case 3: dname = "DG H1";  break;
-            default: break;
-         }
-
-         if (dcounter)
-         {
-            d = (relative) ?
-                sqrt(CoeffNorm*CoeffNorm + CoeffDNorm*CoeffDNorm):1.0;
-
-            out << " -------------------------------------------" << "\n";
-            out << std::setw(21) << title << dname << " Error   " << "\n";
-            out << " -------------------------------------------" << "\n";
-            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
-            out << "Error ";
-            out << std::setw(15) << "Rate " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::setprecision(4);
-            for (int i =0; i<dcounter; i++)
-            {
-               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                   << std::scientific << EnErrors[i]/d << std::setw(13)
-                   << std::fixed << EnRates[i] << "\n";
-            }
-            out << "\n";
-         }
-         if (cont_type == 3 && fcounter)
-         {
-            out << " -------------------------------------------" << "\n";
-            out << "            DG Face Jump Error          " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
-            out << "Error ";
-            out << std::setw(15) << "Rate " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::setprecision(4);
-            for (int i =0; i<fcounter; i++)
-            {
-               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                   << std::scientific << DGFaceErrors[i] << std::setw(13)
-                   << std::fixed << DGFaceRates[i] << "\n";
-            }
-            out << "\n";
-         }
-      }
-   }
-}
-
-} // namespace mfem
@@ -1,149 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_CONVERGENCE
-#define MFEM_CONVERGENCE
-
-#include "../linalg/linalg.hpp"
-#include "gridfunc.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#endif
-
-namespace mfem
-{
-
-/** @brief Class to compute error and convergence rates.
-    It supports H1, H(curl) (ND elements), H(div) (RT elements) and L2 (DG).
-
-    For "smooth enough" solutions the Galerkin error measured in the appropriate
-    norm satisfies || u - u_h || ~ h^k
-
-    Here, k is called the asymptotic rate of convergence
-
-    For successive uniform h-refinements the rate can be estimated by
-    k = log(||u - u_h|| / ||u - u_{h/2}||)/log(2)
-*/
-class ConvergenceStudy
-{
-private:
-   // counters for solutions/derivatives
-   int counter=0;
-   int dcounter=0;
-   int fcounter=0;
-
-   // space continuity type
-   int cont_type=-1;
-
-   // printing flag for helpful for MPI calls
-   int print_flag=1;
-
-   // exact solution and derivatives
-   double CoeffNorm;
-   double CoeffDNorm;
-
-   // Arrays to store error/rates
-   Array<double> L2Errors, DGFaceErrors, DErrors, EnErrors;
-   Array<double> L2Rates, DGFaceRates, DRates, EnRates;
-   Array<int> ndofs;
-
-   void AddL2Error(GridFunction *gf, Coefficient *scalar_u,
-                   VectorCoefficient *vector_u);
-   void AddGf(GridFunction *gf, Coefficient *scalar_u,
-              VectorCoefficient *grad=nullptr,
-              Coefficient *ell_coeff=nullptr, double Nu=1.0);
-   void AddGf(GridFunction *gf, VectorCoefficient *vector_u,
-              VectorCoefficient *curl, Coefficient *div);
-   // returns the L2-norm of scalar_u or vector_u
-   double GetNorm(GridFunction *gf, Coefficient *scalar_u,
-                  VectorCoefficient *vector_u);
-
-public:
-
-   /// Clear any internal data
-   void Reset();
-
-   /// Add L2 GridFunction, the exact solution and possibly its gradient and/or
-   /// DG face jumps parameters
-   void AddL2GridFunction(GridFunction *gf, Coefficient *scalar_u,
-                          VectorCoefficient *grad=nullptr,
-                          Coefficient *ell_coeff=nullptr, double Nu=1.0)
-   {
-      AddGf(gf, scalar_u, grad, ell_coeff, Nu);
-   }
-
-   /// Add H1 GridFunction, the exact solution and possibly its gradient
-   void AddH1GridFunction(GridFunction *gf, Coefficient *scalar_u,
-                          VectorCoefficient *grad=nullptr)
-   {
-      AddGf(gf, scalar_u, grad);
-   }
-
-   /// Add H(curl) GridFunction, the exact solution and possibly its curl
-   void AddHcurlGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
-                             VectorCoefficient *curl=nullptr)
-   {
-      AddGf(gf, vector_u, curl, nullptr);
-   }
-
-   /// Add H(div) GridFunction, the exact solution and possibly its div
-   void AddHdivGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
-                            Coefficient *div=nullptr)
-   {
-      AddGf(gf,vector_u, nullptr, div);
-   }
-
-   /// Get the L2 error at step n
-   double GetL2Error(int n)
-   {
-      MFEM_VERIFY( n <= counter,"Step out of bounds")
-      return L2Errors[n];
-   }
-
-   /// Get all L2 errors
-   void GetL2Errors(Array<double> & L2Errors_)
-   {
-      L2Errors_ = L2Errors;
-   }
-
-   /// Get the Grad/Curl/Div error at step n
-   double GetDError(int n)
-   {
-      MFEM_VERIFY(n <= dcounter,"Step out of bounds")
-      return DErrors[n];
-   }
-
-   /// Get all Grad/Curl/Div errors
-   void GetDErrors(Array<double> & DErrors_)
-   {
-      DErrors_ = DErrors;
-   }
-
-   /// Get the DGFaceJumps error at step n
-   double GetDGFaceJumpsError(int n)
-   {
-      MFEM_VERIFY(n<= fcounter,"Step out of bounds")
-      return DGFaceErrors[n];
-   }
-
-   /// Get all DGFaceJumps errors
-   void GetDGFaceJumpsErrors(Array<double> & DGFaceErrors_)
-   {
-      DGFaceErrors_ = DGFaceErrors;
-   }
-
-   /// Print rates and errors
-   void Print(bool relative = false, std::ostream &out = mfem::out);
-};
-
-} // namespace mfem
-
-#endif // MFEM_CONVERGENCE
@@ -563,8 +563,6 @@ void VisItDataCollection::LoadVisItRootFile(const std::string& root_name)

 void VisItDataCollection::LoadMesh()
 {
-   // GetMeshFileName() uses 'serial', so we need to set it in advance.
-   serial = (format == SERIAL_FORMAT);
   std::string mesh_fname = GetMeshFileName();
   named_ifgzstream file(mesh_fname);
   // TODO: in parallel, check for errors on all processors
@@ -908,7 +906,7 @@ void ParaViewDataCollection::Save()

      // CELL DATA
      out << "<PCellData>\n";
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "attribute"
+      out << "\t<PDataArray type=\"Int32\" Name=\"" << "material"
          << "\" NumberOfComponents=\"1\""
          << " format=\"" << GetDataFormatString() << "\"/>\n";
      out << "</PCellData>\n";
@@ -382,10 +382,6 @@ public:
   int Error() const { return error; }
   /// Reset the error state
   void ResetError(int err = NO_ERROR) { error = err; }
-
-#ifdef MFEM_USE_MPI
-   friend class ParMesh;
-#endif
 };


@@ -139,12 +139,6 @@ void FiniteElement::Project (
   mfem_error ("FiniteElement::Project (...) (vector) is not overloaded !");
 }

-void FiniteElement::ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                     Vector &dofs) const
-{
-   mfem_error ("FiniteElement::ProjectFromNodes() (vector) is not overloaded!");
-}
-
 void FiniteElement::ProjectMatrixCoefficient(
   MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
 {
@@ -931,23 +925,6 @@ void VectorFiniteElement::Project_RT(
   }
 }

-void VectorFiniteElement::Project_RT(
-   const double *nk, const Array<int> &d2n,
-   Vector &vc, ElementTransformation &Trans, Vector &dofs) const
-{
-   const int sdim = Trans.GetSpaceDim();
-   const bool square_J = (dim == sdim);
-
-   for (int k = 0; k < dof; k++)
-   {
-      Trans.SetIntPoint(&Nodes.IntPoint(k));
-      // dof_k = nk^t adj(J) xk
-      Vector vk(vc.GetData()+k*sdim, sdim);
-      dofs(k) = Trans.AdjugateJacobian().InnerProduct(vk, nk + d2n[k]*dim);
-      if (!square_J) { dofs(k) /= Trans.Weight(); }
-   }
-}
-
 void VectorFiniteElement::ProjectMatrixCoefficient_RT(
   const double *nk, const Array<int> &d2n,
   MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -1124,19 +1101,6 @@ void VectorFiniteElement::Project_ND(
   }
 }

-void VectorFiniteElement::Project_ND(
-   const double *tk, const Array<int> &d2t,
-   Vector &vc, ElementTransformation &Trans, Vector &dofs) const
-{
-   for (int k = 0; k < dof; k++)
-   {
-      Trans.SetIntPoint(&Nodes.IntPoint(k));
-      Vector vk(vc.GetData()+k*dim, dim);
-      // dof_k = xk^t J tk
-      dofs(k) = Trans.Jacobian().InnerProduct(tk + d2t[k]*dim, vk);
-   }
-}
-
 void VectorFiniteElement::ProjectMatrixCoefficient_ND(
   const double *tk, const Array<int> &d2t,
   MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
@@ -8031,7 +7995,7 @@ void H1_HexahedronElement::CalcHessian(const IntegrationPoint &ip,
 #ifdef MFEM_THREAD_SAFE
   Vector shape_x(p+1),  shape_y(p+1),  shape_z(p+1);
   Vector dshape_x(p+1), dshape_y(p+1), dshape_z(p+1);
-   Vector d2shape_x(p+1), d2shape_y(p+1), d2shape_z(p+1);
+   Vector d2shape_x(p+1), d2shape_y(p+1), ds2hape_z(p+1);
 #endif

   basis1d.Eval(ip.x, shape_x, dshape_x, d2shape_x);
@@ -504,21 +504,14 @@ public:
   /** @brief Given a coefficient and a transformation, compute its projection
       (approximation) in the local finite dimensional space in terms
       of the degrees of freedom. */
-   virtual void Project(Coefficient &coeff,
-                        ElementTransformation &Trans, Vector &dofs) const;
+   virtual void Project (Coefficient &coeff,
+                         ElementTransformation &Trans, Vector &dofs) const;

   /** @brief Given a vector coefficient and a transformation, compute its
       projection (approximation) in the local finite dimensional space
       in terms of the degrees of freedom. (VectorFiniteElements) */
-   virtual void Project(VectorCoefficient &vc,
-                        ElementTransformation &Trans, Vector &dofs) const;
-
-   /** @brief Given a vector of values at the finite element nodes and a
-       transformation, compute its projection (approximation) in the local
-       finite dimensional space in terms of the degrees of freedom. Valid for
-       VectorFiniteElements. */
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const;
+   virtual void Project (VectorCoefficient &vc,
+                         ElementTransformation &Trans, Vector &dofs) const;

   /** @brief Given a matrix coefficient and a transformation, compute an
       approximation ("projection") in the local finite dimensional space in
@@ -804,12 +797,7 @@ protected:
                   VectorCoefficient &vc, ElementTransformation &Trans,
                   Vector &dofs) const;

-   /// Projects the vector of values given at FE nodes to RT space
-   void Project_RT(const double *nk, const Array<int> &d2n,
-                   Vector &vc, ElementTransformation &Trans,
-                   Vector &dofs) const;
-
-   /// Project the rows of the matrix coefficient in an RT space
+   // project the rows of the matrix coefficient in an RT space
   void ProjectMatrixCoefficient_RT(
      const double *nk, const Array<int> &d2n,
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
@@ -837,12 +825,7 @@ protected:
                   VectorCoefficient &vc, ElementTransformation &Trans,
                   Vector &dofs) const;

-   /// Projects the vector of values given at FE nodes to ND space
-   void Project_ND(const double *tk, const Array<int> &d2t,
-                   Vector &vc, ElementTransformation &Trans,
-                   Vector &dofs) const;
-
-   /// Project the rows of the matrix coefficient in an ND space
+   /// project the rows of the matrix coefficient in an ND space
   void ProjectMatrixCoefficient_ND(
      const double *tk, const Array<int> &d2t,
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const;
@@ -2706,9 +2689,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2767,9 +2747,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2821,9 +2798,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2881,9 +2855,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_RT(nk, dof2nk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_RT(nk, dof2nk, mc, T, dofs); }
@@ -2943,10 +2914,6 @@ public:
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_ND(tk, dof2tk, vc, Trans, dofs); }

-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3006,9 +2973,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3060,9 +3024,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -3119,9 +3080,6 @@ public:
   virtual void Project(VectorCoefficient &vc,
                        ElementTransformation &Trans, Vector &dofs) const
   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
-   virtual void ProjectFromNodes(Vector &vc, ElementTransformation &Trans,
-                                 Vector &dofs) const
-   { Project_ND(tk, dof2tk, vc, Trans, dofs); }
   virtual void ProjectMatrixCoefficient(
      MatrixCoefficient &mc, ElementTransformation &T, Vector &dofs) const
   { ProjectMatrixCoefficient_ND(tk, dof2tk, mc, T, dofs); }
@@ -19,7 +19,6 @@
 #include "eltrans.hpp"
 #include "coefficient.hpp"
 #include "complex_fem.hpp"
-#include "convergence.hpp"
 #include "lininteg.hpp"
 #include "nonlininteg.hpp"
 #include "bilininteg.hpp"
@@ -14,7 +14,6 @@
 #include "../general/text.hpp"
 #include "../general/forall.hpp"
 #include "../mesh/mesh_headers.hpp"
-#include "../fem/libceed/ceed.hpp"
 #include "fem.hpp"

 #include <cmath>
@@ -441,7 +440,6 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
      if (marker[i]) { num_marked++; }
   }
   list.SetSize(0);
-   list.HostWrite();
   list.Reserve(num_marked);
   for (int i = 0; i < marker.Size(); i++)
   {
@@ -453,9 +451,7 @@ void FiniteElementSpace::MarkerToList(const Array<int> &marker,
 void FiniteElementSpace::ListToMarker(const Array<int> &list, int marker_size,
                                      Array<int> &marker, int mark_val)
 {
-   list.HostRead(); // make sure we can read the array on host
   marker.SetSize(marker_size);
-   marker.HostWrite();
   marker = 0;
   for (int i = 0; i < list.Size(); i++)
   {
@@ -699,7 +695,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
   for (int entity = 1; entity <= 2; entity++)
   {
      const NCMesh::NCList &list = mesh->ncmesh->GetNCList(entity);
-      if (!list.masters.Size()) { continue; }
+      if (!list.masters.size()) { continue; }

      Array<int> master_dofs, slave_dofs;

@@ -707,7 +703,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
      DenseMatrix I;

      // loop through all master edges/faces, constrain their slave edges/faces
-      for (int mi = 0; mi < list.masters.Size(); mi++)
+      for (unsigned mi = 0; mi < list.masters.size(); mi++)
      {
         const NCMesh::Master &master = list.masters[mi];

@@ -731,7 +727,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
            GetEntityDofs(entity, slave.index, slave_dofs, master.Geom());
            if (!slave_dofs.Size()) { continue; }

-            list.OrientedPointMatrix(slave, T.GetPointMat());
+            slave.OrientedPointMatrix(T.GetPointMat());
            fe->GetLocalInterpolation(T, I);

            // make each slave DOF dependent on all master DOFs
@@ -2161,7 +2157,6 @@ void FiniteElementSpace::Destroy()
      delete [] bdofs;
      delete [] fdofs;
   }
-   RemoveCeedBasisAndRestriction(this);
 }

 void FiniteElementSpace::GetTransferOperator(
@@ -199,7 +199,8 @@ void GridFunction::MakeRef(FiniteElementSpace *f, Vector &v, int v_offset)
   if (f != fes) { Destroy(); }
   fes = f;
   v.UseDevice(true);
-   this->Vector::MakeRef(v, v_offset, fes->GetVSize());
+   NewMemoryAndSize(Memory<double>(v.GetMemory(), v_offset, fes->GetVSize()),
+                    fes->GetVSize(), true);
   sequence = fes->GetSequence();
 }

@@ -1833,19 +1834,6 @@ void GridFunction::ImposeBounds(int i, const Vector &weights,
   ImposeBounds(i, weights, minv, maxv);
 }

-void GridFunction::RestrictConforming()
-{
-   const SparseMatrix *R = fes->GetRestrictionMatrix();
-   const Operator *P = fes->GetProlongationMatrix();
-
-   if (P && R)
-   {
-      Vector tmp(R->Height());
-      R->Mult(*this, tmp);
-      P->Mult(tmp, *this);
-   }
-}
-
 void GridFunction::GetNodalValues(Vector &nval, int vdim) const
 {
   int i, j;
@@ -2614,7 +2602,11 @@ double GridFunction::ComputeL2Error(
      }
   }

-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+   if (error < 0.0)
+   {
+      return -sqrt(-error);
+   }
+   return sqrt(error);
 }

 double GridFunction::ComputeL2Error(
@@ -2655,199 +2647,94 @@ double GridFunction::ComputeL2Error(
      }
   }

-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeGradError(VectorCoefficient *exgrad,
-                                      const IntegrationRule *irs[]) const
-{
-   double error = 0.0;
-   const FiniteElement *fe;
-   ElementTransformation *Tr;
-   Array<int> dofs;
-   Vector grad;
-   int intorder;
-   int dim = fes->GetMesh()->SpaceDimension();
-   Vector vec(dim);
-
-   for (int i = 0; i < fes->GetNE(); i++)
+   if (error < 0.0)
   {
-      fe = fes->GetFE(i);
-      Tr = fes->GetElementTransformation(i);
-      intorder = 2*fe->GetOrder() + 3; // <--------
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      fes->GetElementDofs(i, dofs);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         Tr->SetIntPoint(&ip);
-         GetGradient(*Tr,grad);
-         exgrad->Eval(vec,*Tr,ip);
-         vec-=grad;
-         error += ip.weight * Tr->Weight() * (vec * vec);
-      }
+      return -sqrt(-error);
   }
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
+   return sqrt(error);
 }

-double GridFunction::ComputeCurlError(VectorCoefficient *excurl,
-                                      const IntegrationRule *irs[]) const
+double GridFunction::ComputeH1Error(
+   Coefficient *exsol, VectorCoefficient *exgrad,
+   Coefficient *ell_coeff, double Nu, int norm_type) const
 {
-   double error = 0.0;
-   const FiniteElement *fe;
-   ElementTransformation *Tr;
-   Array<int> dofs;
-   Vector curl;
-   int intorder;
-   int dim = fes->GetMesh()->SpaceDimension();
-   int n = (dim == 3) ? dim : 1;
-   Vector vec(n);
-
-   for (int i = 0; i < fes->GetNE(); i++)
-   {
-      fe = fes->GetFE(i);
-      Tr = fes->GetElementTransformation(i);
-      intorder = 2*fe->GetOrder() + 3;
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      fes->GetElementDofs(i, dofs);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         Tr->SetIntPoint(&ip);
-         GetCurl(*Tr,curl);
-         excurl->Eval(vec,*Tr,ip);
-         vec-=curl;
-         error += ip.weight * Tr->Weight() * ( vec * vec );
-      }
-   }
-
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeDivError(
-   Coefficient *exdiv, const IntegrationRule *irs[]) const
-{
-   double error = 0.0, a;
-   const FiniteElement *fe;
-   ElementTransformation *Tr;
-   Array<int> dofs;
-   int intorder;
-
-   for (int i = 0; i < fes->GetNE(); i++)
-   {
-      fe = fes->GetFE(i);
-      Tr = fes->GetElementTransformation(i);
-      intorder = 2*fe->GetOrder() + 3;
-      const IntegrationRule *ir;
-      if (irs)
-      {
-         ir = irs[fe->GetGeomType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(fe->GetGeomType(), intorder));
-      }
-      fes->GetElementDofs(i, dofs);
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         Tr->SetIntPoint (&ip);
-         a = GetDivergence(*Tr) - exdiv->Eval(*Tr, ip);
-         error += ip.weight * Tr->Weight() * a * a;
-      }
-   }
-
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
-                                            Coefficient *ell_coeff, double Nu,
-                                            const IntegrationRule *irs[])  const
-{
-   int fdof, dim, intorder, k;
+   // assuming vdim is 1
+   int i, fdof, dim, intorder, j, k;
   Mesh *mesh;
   const FiniteElement *fe;
   ElementTransformation *transf;
   FaceElementTransformations *face_elem_transf;
-   Vector shape, el_dofs, err_val, ell_coeff_val;
+   Vector e_grad, a_grad, shape, el_dofs, err_val, ell_coeff_val;
+   DenseMatrix dshape, dshapet, Jinv;
   Array<int> vdofs;
   IntegrationPoint eip;
   double error = 0.0;

   mesh = fes->GetMesh();
   dim = mesh->Dimension();
+   e_grad.SetSize(dim);
+   a_grad.SetSize(dim);
+   Jinv.SetSize(dim);

-   for (int i = 0; i < mesh->GetNumFaces(); i++)
-   {
-      face_elem_transf = mesh->GetFaceElementTransformations(i, 5);
-      int i1 = face_elem_transf->Elem1No;
-      int i2 = face_elem_transf->Elem2No;
-      intorder = fes->GetFE(i1)->GetOrder();
-      if (i2 >= 0)
-         if ( (k = fes->GetFE(i2)->GetOrder()) > intorder )
-         {
-            intorder = k;
-         }
-      intorder = 2 * intorder;  // <-------------
-      const IntegrationRule *ir;
-      if (irs)
+   if (norm_type & 1)
+      for (i = 0; i < mesh->GetNE(); i++)
      {
-         ir = irs[face_elem_transf->GetGeometryType()];
-      }
-      else
-      {
-         ir = &(IntRules.Get(face_elem_transf->GetGeometryType(), intorder));
-      }
-      err_val.SetSize(ir->GetNPoints());
-      ell_coeff_val.SetSize(ir->GetNPoints());
-      // side 1
-      transf = face_elem_transf->Elem1;
-      fe = fes->GetFE(i1);
-      fdof = fe->GetDof();
-      fes->GetElementVDofs(i1, vdofs);
-      shape.SetSize(fdof);
-      el_dofs.SetSize(fdof);
-      for (k = 0; k < fdof; k++)
-         if (vdofs[k] >= 0)
-         {
-            el_dofs(k) =   (*this)(vdofs[k]);
-         }
-         else
-         {
-            el_dofs(k) = - (*this)(-1-vdofs[k]);
-         }
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         face_elem_transf->Loc1.Transform(ir->IntPoint(j), eip);
-         fe->CalcShape(eip, shape);
-         transf->SetIntPoint(&eip);
-         ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
-         err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
-      }
-      if (i2 >= 0)
-      {
-         // side 2
-         face_elem_transf = mesh->GetFaceElementTransformations(i, 10);
-         transf = face_elem_transf->Elem2;
-         fe = fes->GetFE(i2);
+         fe = fes->GetFE(i);
         fdof = fe->GetDof();
-         fes->GetElementVDofs(i2, vdofs);
+         transf = mesh->GetElementTransformation(i);
+         el_dofs.SetSize(fdof);
+         dshape.SetSize(fdof, dim);
+         dshapet.SetSize(fdof, dim);
+         intorder = 2 * fe->GetOrder(); // <----------
+         const IntegrationRule &ir = IntRules.Get(fe->GetGeomType(), intorder);
+         fes->GetElementVDofs(i, vdofs);
+         for (k = 0; k < fdof; k++)
+            if (vdofs[k] >= 0)
+            {
+               el_dofs(k) =   (*this)(vdofs[k]);
+            }
+            else
+            {
+               el_dofs(k) = - (*this)(-1-vdofs[k]);
+            }
+         for (j = 0; j < ir.GetNPoints(); j++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(j);
+            fe->CalcDShape(ip, dshape);
+            transf->SetIntPoint(&ip);
+            exgrad->Eval(e_grad, *transf, ip);
+            CalcInverse(transf->Jacobian(), Jinv);
+            Mult(dshape, Jinv, dshapet);
+            dshapet.MultTranspose(el_dofs, a_grad);
+            e_grad -= a_grad;
+            error += (ip.weight * transf->Weight() *
+                      ell_coeff->Eval(*transf, ip) *
+                      (e_grad * e_grad));
+         }
+      }
+
+   if (norm_type & 2)
+      for (i = 0; i < mesh->GetNFaces(); i++)
+      {
+         face_elem_transf = mesh->GetFaceElementTransformations(i, 5);
+         int i1 = face_elem_transf->Elem1No;
+         int i2 = face_elem_transf->Elem2No;
+         intorder = fes->GetFE(i1)->GetOrder();
+         if (i2 >= 0)
+            if ( (k = fes->GetFE(i2)->GetOrder()) > intorder )
+            {
+               intorder = k;
+            }
+         intorder = 2 * intorder;  // <-------------
+         const IntegrationRule &ir =
+            IntRules.Get(face_elem_transf->GetGeometryType(), intorder);
+         err_val.SetSize(ir.GetNPoints());
+         ell_coeff_val.SetSize(ir.GetNPoints());
+         // side 1
+         transf = face_elem_transf->Elem1;
+         fe = fes->GetFE(i1);
+         fdof = fe->GetDof();
+         fes->GetElementVDofs(i1, vdofs);
         shape.SetSize(fdof);
         el_dofs.SetSize(fdof);
         for (k = 0; k < fdof; k++)
@@ -2859,69 +2746,60 @@ double GridFunction::ComputeDGFaceJumpError(Coefficient *exsol,
            {
               el_dofs(k) = - (*this)(-1-vdofs[k]);
            }
-         for (int j = 0; j < ir->GetNPoints(); j++)
+         for (j = 0; j < ir.GetNPoints(); j++)
         {
-            face_elem_transf->Loc2.Transform(ir->IntPoint(j), eip);
+            face_elem_transf->Loc1.Transform(ir.IntPoint(j), eip);
            fe->CalcShape(eip, shape);
            transf->SetIntPoint(&eip);
-            ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
-            ell_coeff_val(j) *= 0.5;
-            err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
+            ell_coeff_val(j) = ell_coeff->Eval(*transf, eip);
+            err_val(j) = exsol->Eval(*transf, eip) - (shape * el_dofs);
+         }
+         if (i2 >= 0)
+         {
+            // side 2
+            face_elem_transf = mesh->GetFaceElementTransformations(i, 10);
+            transf = face_elem_transf->Elem2;
+            fe = fes->GetFE(i2);
+            fdof = fe->GetDof();
+            fes->GetElementVDofs(i2, vdofs);
+            shape.SetSize(fdof);
+            el_dofs.SetSize(fdof);
+            for (k = 0; k < fdof; k++)
+               if (vdofs[k] >= 0)
+               {
+                  el_dofs(k) =   (*this)(vdofs[k]);
+               }
+               else
+               {
+                  el_dofs(k) = - (*this)(-1-vdofs[k]);
+               }
+            for (j = 0; j < ir.GetNPoints(); j++)
+            {
+               face_elem_transf->Loc2.Transform(ir.IntPoint(j), eip);
+               fe->CalcShape(eip, shape);
+               transf->SetIntPoint(&eip);
+               ell_coeff_val(j) += ell_coeff->Eval(*transf, eip);
+               ell_coeff_val(j) *= 0.5;
+               err_val(j) -= (exsol->Eval(*transf, eip) - (shape * el_dofs));
+            }
+         }
+         face_elem_transf = mesh->GetFaceElementTransformations(i, 16);
+         transf = face_elem_transf;
+         for (j = 0; j < ir.GetNPoints(); j++)
+         {
+            const IntegrationPoint &ip = ir.IntPoint(j);
+            transf->SetIntPoint(&ip);
+            error += (ip.weight * Nu * ell_coeff_val(j) *
+                      pow(transf->Weight(), 1.0-1.0/(dim-1)) *
+                      err_val(j) * err_val(j));
         }
      }
-      face_elem_transf = mesh->GetFaceElementTransformations(i, 16);
-      transf = face_elem_transf;
-      for (int j = 0; j < ir->GetNPoints(); j++)
-      {
-         const IntegrationPoint &ip = ir->IntPoint(j);
-         transf->SetIntPoint(&ip);
-         error += (ip.weight * Nu * ell_coeff_val(j) *
-                   pow(transf->Weight(), 1.0-1.0/(dim-1)) *
-                   err_val(j) * err_val(j));
-      }
+
+   if (error < 0.0)
+   {
+      return -sqrt(-error);
   }
-
-   return (error < 0.0) ? -sqrt(-error) : sqrt(error);
-}
-
-double GridFunction::ComputeH1Error(Coefficient *exsol,
-                                    VectorCoefficient *exgrad,
-                                    Coefficient *ell_coef, double Nu,
-                                    int norm_type) const
-{
-   double error1 = 0.0;
-   double error2 = 0.0;
-   if (norm_type & 1) { error1 = GridFunction::ComputeGradError(exgrad); }
-   if (norm_type & 2) { error2 = GridFunction::ComputeDGFaceJumpError(exsol,ell_coef,Nu); }
-
-   return sqrt(error1 * error1 + error2 * error2);
-}
-
-double GridFunction::ComputeH1Error(Coefficient *exsol,
-                                    VectorCoefficient *exgrad,
-                                    const IntegrationRule *irs[]) const
-{
-   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,irs);
-   double GradError = ComputeGradError(exgrad,irs);
-   return sqrt(L2error*L2error + GradError*GradError);
-}
-
-double GridFunction::ComputeHDivError(VectorCoefficient *exsol,
-                                      Coefficient *exdiv,
-                                      const IntegrationRule *irs[]) const
-{
-   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,NULL,irs);
-   double DivError = ComputeDivError(exdiv,irs);
-   return sqrt(L2error*L2error + DivError*DivError);
-}
-
-double GridFunction::ComputeHCurlError(VectorCoefficient *exsol,
-                                       VectorCoefficient *excurl,
-                                       const IntegrationRule *irs[]) const
-{
-   double L2error = GridFunction::ComputeLpError(2.0,*exsol,NULL,NULL,irs);
-   double CurlError = ComputeCurlError(excurl,irs);
-   return sqrt(L2error*L2error + CurlError*CurlError);
+   return sqrt(error);
 }

 double GridFunction::ComputeMaxError(
@@ -2977,6 +2855,7 @@ double GridFunction::ComputeMaxError(
         }
      }
   }
+
   return error;
 }

@@ -334,11 +334,6 @@ public:
   void ImposeBounds(int i, const Vector &weights,
                     double _min = 0.0, double _max = infinity());

-   /** On a non-conforming mesh, make sure the function lies in the conforming
-       space by multiplying with R and then with P, the conforming restriction
-       and prolongation matrices of the space, respectively. */
-   void RestrictConforming();
-
   /** @brief Project the @a src GridFunction to @a this GridFunction, both of
       which must be on the same mesh. */
   /** The current implementation assumes that all elements use the same
@@ -427,7 +422,6 @@ public:
   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &vcoeff,
                                             Array<int> &bdr_attr);

-
   virtual double ComputeL2Error(Coefficient &exsol,
                                 const IntegrationRule *irs[] = NULL) const
   { return ComputeLpError(2.0, exsol, NULL, irs); }
@@ -439,50 +433,10 @@ public:
                                 const IntegrationRule *irs[] = NULL,
                                 Array<int> *elems = NULL) const;

-   /// Returns ||grad u_ex - grad u_h||_L2 for H1 or L2 elements
-   virtual double ComputeGradError(VectorCoefficient *exgrad,
-                                   const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns ||curl u_ex - curl u_h||_L2 for ND elements
-   virtual double ComputeCurlError(VectorCoefficient *excurl,
-                                   const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns ||div u_ex - div u_h||_L2 for RT elements
-   virtual double ComputeDivError(Coefficient *exdiv,
-                                  const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns the Face Jumps error for L2 elements
-   virtual double ComputeDGFaceJumpError(Coefficient *exsol,
-                                         Coefficient *ell_coeff,
-                                         double Nu,
-                                         const IntegrationRule *irs[] = NULL)
-   const;
-
-   /** This method is kept for backward compatibility.
-
-       Returns either the H1-seminorm, or the DG face jumps error, or both
-       depending on norm_type = 1, 2, 3. Additional arguments for the DG face
-       jumps norm: ell_coeff: mesh-depended coefficient (weight) Nu: scalar
-       constant weight */
   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
                                 Coefficient *ell_coef, double Nu,
                                 int norm_type) const;

-   /// Returns the error measured in H1-norm for H1 elements or in "broken"
-   /// H1-norm for L2 elements
-   virtual double ComputeH1Error(Coefficient *exsol, VectorCoefficient *exgrad,
-                                 const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns the error measured in H(div)-norm for RT elements
-   virtual double ComputeHDivError(VectorCoefficient *exsol,
-                                   Coefficient *exdiv,
-                                   const IntegrationRule *irs[] = NULL) const;
-
-   /// Returns the error measured in H(curl)-norm for ND elements
-   virtual double ComputeHCurlError(VectorCoefficient *exsol,
-                                    VectorCoefficient *excurl,
-                                    const IntegrationRule *irs[] = NULL) const;
-
   virtual double ComputeMaxError(Coefficient &exsol,
                                  const IntegrationRule *irs[] = NULL) const
   {
@@ -29,13 +29,10 @@ namespace mfem
 {

 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(NULL), meshsplit(NULL), ir_simplex(NULL),
-     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
-     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC)
+   : mesh(NULL), ir_simplex(NULL), fdata2D(NULL), fdata3D(NULL),
+     dim(-1), gsl_mesh(), gsl_ref(), gsl_dist(), setupflag(false)
 {
   gsl_comm = new comm;
-   cr       = new crystal;
 #ifdef MFEM_USE_MPI
   int initialized;
   MPI_Initialized(&initialized);
@@ -50,20 +47,15 @@ FindPointsGSLIB::FindPointsGSLIB()
 FindPointsGSLIB::~FindPointsGSLIB()
 {
   delete gsl_comm;
-   delete cr;
   delete ir_simplex;
-   delete meshsplit;
 }

 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm _comm)
-   : mesh(NULL), meshsplit(NULL), ir_simplex(NULL),
-     fdata2D(NULL), fdata3D(NULL), cr(NULL), gsl_comm(NULL),
-     dim(-1), points_cnt(0), setupflag(false), default_interp_value(0),
-     avgtype(AvgType::ARITHMETIC)
+   : mesh(NULL), ir_simplex(NULL), fdata2D(NULL), fdata3D(NULL),
+     dim(-1), gsl_mesh(), gsl_ref(), gsl_dist(), setupflag(false)
 {
   gsl_comm = new comm;
-   cr      = new crystal;
   comm_init(gsl_comm, _comm);
 }
 #endif
@@ -78,7 +70,6 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
   // call FreeData if FindPointsGSLIB::Setup has been called already
   if (setupflag) { FreeData(); }

-   crystal_init(cr, gsl_comm);
   mesh = &m;
   dim  = mesh->Dimension();
   const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
@@ -122,16 +113,14 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
   setupflag = true;
 }

-void FindPointsGSLIB::FindPoints(const Vector &point_pos)
+void FindPointsGSLIB::FindPoints(const Vector &point_pos,
+                                 Array<unsigned int> &codes,
+                                 Array<unsigned int> &proc_ids,
+                                 Array<unsigned int> &elem_ids,
+                                 Vector &ref_pos, Vector &dist)
 {
   MFEM_VERIFY(setupflag, "Use FindPointsGSLIB::Setup before finding points.");
-   points_cnt = point_pos.Size() / dim;
-   gsl_code.SetSize(points_cnt);
-   gsl_proc.SetSize(points_cnt);
-   gsl_elem.SetSize(points_cnt);
-   gsl_ref.SetSize(points_cnt * dim);
-   gsl_dist.SetSize(points_cnt);
-
+   const int points_cnt = point_pos.Size() / dim;
   if (dim == 2)
   {
      const double *xv_base[2];
@@ -140,11 +129,11 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
      unsigned xv_stride[2];
      xv_stride[0] = sizeof(double);
      xv_stride[1] = sizeof(double);
-      findpts_2(gsl_code.GetData(), sizeof(unsigned int),
-                gsl_proc.GetData(), sizeof(unsigned int),
-                gsl_elem.GetData(), sizeof(unsigned int),
-                gsl_ref.GetData(),  sizeof(double) * dim,
-                gsl_dist.GetData(), sizeof(double),
+      findpts_2(codes.GetData(), sizeof(unsigned int),
+                proc_ids.GetData(), sizeof(unsigned int),
+                elem_ids.GetData(), sizeof(unsigned int),
+                ref_pos.GetData(), sizeof(double) * dim,
+                dist.GetData(), sizeof(double),
                xv_base, xv_stride, points_cnt, fdata2D);
   }
   else
@@ -157,27 +146,25 @@ void FindPointsGSLIB::FindPoints(const Vector &point_pos)
      xv_stride[0] = sizeof(double);
      xv_stride[1] = sizeof(double);
      xv_stride[2] = sizeof(double);
-      findpts_3(gsl_code.GetData(), sizeof(unsigned int),
-                gsl_proc.GetData(), sizeof(unsigned int),
-                gsl_elem.GetData(), sizeof(unsigned int),
-                gsl_ref.GetData(),  sizeof(double) * dim,
-                gsl_dist.GetData(), sizeof(double),
+      findpts_3(codes.GetData(), sizeof(unsigned int),
+                proc_ids.GetData(), sizeof(unsigned int),
+                elem_ids.GetData(), sizeof(unsigned int),
+                ref_pos.GetData(), sizeof(double) * dim,
+                dist.GetData(), sizeof(double),
                xv_base, xv_stride, points_cnt, fdata3D);
   }
+}

-   // Set the element number and reference position to 0 for points not found
-   for (int i = 0; i < points_cnt; i++)
-   {
-      if (gsl_code[i] == 2)
-      {
-         gsl_elem[i] = 0;
-         for (int d = 0; d < dim; d++) { gsl_ref(i*dim + d) = -1.; }
-      }
-   }
+void FindPointsGSLIB::FindPoints(const Vector &point_pos)
+{
+   const int points_cnt = point_pos.Size() / dim;
+   gsl_code.SetSize(points_cnt);
+   gsl_proc.SetSize(points_cnt);
+   gsl_elem.SetSize(points_cnt);
+   gsl_ref.SetSize(points_cnt * dim);
+   gsl_dist.SetSize(points_cnt);

-   // Map element number for simplices, and ref_pos from [-1,1] to [0,1] for
-   // both simplices and quads.
-   MapRefPosAndElemIndices();
+   FindPoints(point_pos, gsl_code, gsl_proc, gsl_elem, gsl_ref, gsl_dist);
 }

 void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
@@ -191,24 +178,72 @@ void FindPointsGSLIB::FindPoints(Mesh &m, const Vector &point_pos,
   FindPoints(point_pos);
 }

+void FindPointsGSLIB::Interpolate(Array<unsigned int> &codes,
+                                  Array<unsigned int> &proc_ids,
+                                  Array<unsigned int> &elem_ids,
+                                  Vector &ref_pos, const GridFunction &field_in,
+                                  Vector &field_out)
+{
+
+   FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
+   GridFunction field_in_scalar(&ind_fes);
+   Vector node_vals;
+
+   const int ncomp      = field_in.FESpace()->GetVDim(),
+             points_fld = field_in.Size() / ncomp,
+             points_cnt = codes.Size();
+   field_out.SetSize(points_cnt*ncomp);
+
+   for (int i = 0; i < ncomp; i++)
+   {
+      const int dataptrin  = i*points_fld,
+                dataptrout = i*points_cnt;
+      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
+      GetNodeValues(field_in_scalar, node_vals);
+
+      if (dim==2)
+      {
+         findpts_eval_2(field_out.GetData()+dataptrout, sizeof(double),
+                        codes.GetData(),       sizeof(unsigned int),
+                        proc_ids.GetData(),    sizeof(unsigned int),
+                        elem_ids.GetData(),    sizeof(unsigned int),
+                        ref_pos.GetData(),     sizeof(double) * dim,
+                        points_cnt, node_vals.GetData(), fdata2D);
+      }
+      else
+      {
+         findpts_eval_3(field_out.GetData()+dataptrout, sizeof(double),
+                        codes.GetData(),       sizeof(unsigned int),
+                        proc_ids.GetData(),    sizeof(unsigned int),
+                        elem_ids.GetData(),    sizeof(unsigned int),
+                        ref_pos.GetData(),     sizeof(double) * dim,
+                        points_cnt, node_vals.GetData(), fdata3D);
+      }
+   }
+}
+
+void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
+                                  Vector &field_out)
+{
+   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
+}
+
 void FindPointsGSLIB::Interpolate(const Vector &point_pos,
                                  const GridFunction &field_in, Vector &field_out)
 {
   FindPoints(point_pos);
-   Interpolate(field_in, field_out);
+   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
 }

 void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
                                  const GridFunction &field_in, Vector &field_out)
 {
   FindPoints(m, point_pos);
-   Interpolate(field_in, field_out);
+   Interpolate(gsl_code, gsl_proc, gsl_elem, gsl_ref, field_in, field_out);
 }

 void FindPointsGSLIB::FreeData()
 {
-   if (!setupflag) { return; }
-   crystal_free(cr);
   if (dim == 2)
   {
      findpts_free_2(fdata2D);
@@ -217,13 +252,13 @@ void FindPointsGSLIB::FreeData()
   {
      findpts_free_3(fdata3D);
   }
+   setupflag = false;
   gsl_code.DeleteAll();
   gsl_proc.DeleteAll();
   gsl_elem.DeleteAll();
   gsl_mesh.Destroy();
   gsl_ref.Destroy();
   gsl_dist.Destroy();
-   setupflag = false;
 }

 void FindPointsGSLIB::GetNodeValues(const GridFunction &gf_in,
@@ -323,8 +358,9 @@ void FindPointsGSLIB::GetSimplexNodalCoordinates()
   const FiniteElement *fe   = mesh->GetNodalFESpace()->GetFE(0);
   const Geometry::Type gt   = fe->GetGeomType();
   const GridFunction *nodes = mesh->GetNodes();
+   Mesh *meshsplit           = NULL;
   const int NE              = mesh->GetNE();
-   int NEsplit = 0;
+   int NEsplit = -1;

   // Split the reference element into a reference submesh of quads or hexes.
   if (gt == Geometry::TRIANGLE)
@@ -480,361 +516,8 @@ void FindPointsGSLIB::GetSimplexNodalCoordinates()
         pt_id++;
      }
   }
-}

-void FindPointsGSLIB::MapRefPosAndElemIndices()
-{
-   gsl_mfem_ref = gsl_ref;
-   gsl_mfem_elem = gsl_elem;
-   const FiniteElement *fe = mesh->GetNodalFESpace()->GetFE(0);
-   const Geometry::Type gt = fe->GetGeomType();
-   int NEsplit = 0;
-
-   gsl_mfem_ref -= -1.;  // map  [-1, 1] to
-   gsl_mfem_ref *= 0.5;  //      [0,  1]
-   if (gt == Geometry::SQUARE || gt == Geometry::CUBE) { return; }
-
-   H1_FECollection feclin(1, dim);
-   FiniteElementSpace nodal_fes_lin(meshsplit, &feclin, dim);
-   GridFunction gf_lin(&nodal_fes_lin);
-
-   if (gt == Geometry::TRIANGLE)
-   {
-      const double quad_v[7][2] =
-      {
-         {0, 0}, {0.5, 0}, {1, 0}, {0, 0.5},
-         {1./3., 1./3.}, {0.5, 0.5}, {0, 1}
-      };
-      for (int k = 0; k < dim; k++)
-      {
-         for (int j = 0; j < gf_lin.Size()/dim; j++)
-         {
-            gf_lin(j+k*gf_lin.Size()/dim) = quad_v[j][k];
-         }
-      }
-      NEsplit = 3;
-   }
-   else if (gt == Geometry::TETRAHEDRON)
-   {
-      const double hex_v[15][3] =
-      {
-         {0, 0, 0.}, {1, 0., 0.}, {0., 1., 0.}, {0, 0., 1.},
-         {0.5, 0., 0.}, {0.5, 0.5, 0.}, {0., 0.5, 0.},
-         {0., 0., 0.5}, {0.5, 0., 0.5}, {0., 0.5, 0.5},
-         {1./3., 0., 1./3.}, {1./3., 1./3., 1./3.}, {0, 1./3., 1./3.},
-         {1./3., 1./3., 0}, {0.25, 0.25, 0.25}
-      };
-      for (int k = 0; k < dim; k++)
-      {
-         for (int j = 0; j < gf_lin.Size()/dim; j++)
-         {
-            gf_lin(j+k*gf_lin.Size()/dim) = hex_v[j][k];
-         }
-      }
-      NEsplit = 4;
-   }
-   else if (gt == Geometry::PRISM)
-   {
-      const double hex_v[14][3] =
-      {
-         {0, 0, 0}, {0.5, 0, 0}, {1, 0, 0}, {0, 0.5, 0},
-         {1./3., 1./3., 0}, {0.5, 0.5, 0}, {0, 1, 0},
-         {0, 0, 1}, {0.5, 0, 1}, {1, 0, 1}, {0, 0.5, 1},
-         {1./3., 1./3., 1}, {0.5, 0.5, 1}, {0, 1, 1}
-      };
-      for (int k = 0; k < dim; k++)
-      {
-         for (int j = 0; j < gf_lin.Size()/dim; j++)
-         {
-            gf_lin(j+k*gf_lin.Size()/dim) = hex_v[j][k];
-         }
-      }
-      NEsplit = 3;
-   }
-   else
-   {
-      MFEM_ABORT("Element type not currently supported.");
-   }
-
-   // Simplices are split into quads/hexes for GSLIB. For MFEM, we need to find
-   // the original element number and map the rst from micro to macro element.
-   for (int i = 0; i < points_cnt; i++)
-   {
-      if (gsl_code[i] == 2) { continue; }
-      int local_elem   = gsl_elem[i]%NEsplit;
-      gsl_mfem_elem[i] = (gsl_elem[i] - local_elem)/NEsplit; // macro element number
-
-      IntegrationPoint ip;
-      Vector mfem_ref(gsl_mfem_ref.GetData()+i*dim, dim);
-      ip.Set2(mfem_ref.GetData());
-      if (dim == 3) { ip.z = mfem_ref(2); }
-      gf_lin.GetVectorValue(local_elem, ip, mfem_ref); // map to rst of macro element
-   }
-}
-
-void FindPointsGSLIB::Interpolate(const GridFunction &field_in,
-                                  Vector &field_out)
-{
-   const int  gf_order   = field_in.FESpace()->GetFE(0)->GetOrder(),
-              mesh_order = mesh->GetNodalFESpace()->GetFE(0)->GetOrder();
-
-   const FiniteElementCollection *fec_in =  field_in.FESpace()->FEColl();
-   const H1_FECollection *fec_h1 = dynamic_cast<const H1_FECollection *>(fec_in);
-   const L2_FECollection *fec_l2 = dynamic_cast<const L2_FECollection *>(fec_in);
-
-   if (fec_h1 && gf_order == mesh_order &&
-       fec_h1->GetBasisType() == BasisType::GaussLobatto)
-   {
-      InterpolateH1(field_in, field_out);
-      return;
-   }
-   else
-   {
-      InterpolateGeneral(field_in, field_out);
-      if (!fec_l2 || avgtype == AvgType::NONE) { return; }
-   }
-
-   // For points on element borders, project the L2 GridFunction to H1 and
-   // re-interpolate.
-   if (fec_l2)
-   {
-      Array<int> indl2;
-      for (int i = 0; i < points_cnt; i++)
-      {
-         if (gsl_code[i] == 1) { indl2.Append(i); }
-      }
-      if (indl2.Size() == 0) { return; } // no points on element borders
-
-      Vector field_out_l2(field_out.Size());
-      VectorGridFunctionCoefficient field_in_dg(&field_in);
-      int gf_order_h1 = std::max(gf_order, 1); // H1 should be at least order 1
-      H1_FECollection fec(gf_order_h1, dim);
-      const int ncomp = field_in.FESpace()->GetVDim();
-      FiniteElementSpace fes(mesh, &fec, ncomp);
-      GridFunction field_in_h1(&fes);
-
-      if (avgtype == AvgType::ARITHMETIC)
-      {
-         field_in_h1.ProjectDiscCoefficient(field_in_dg, GridFunction::ARITHMETIC);
-      }
-      else if (avgtype == AvgType::HARMONIC)
-      {
-         field_in_h1.ProjectDiscCoefficient(field_in_dg, GridFunction::HARMONIC);
-      }
-      else
-      {
-         MFEM_ABORT("Invalid averaging type.");
-      }
-
-      if (gf_order_h1 == mesh_order) // basis is GaussLobatto by default
-      {
-         InterpolateH1(field_in_h1, field_out_l2);
-      }
-      else
-      {
-         InterpolateGeneral(field_in_h1, field_out_l2);
-      }
-
-      // Copy interpolated values for the points on element border
-      for (int j = 0; j < ncomp; j++)
-      {
-         for (int i = 0; i < indl2.Size(); i++)
-         {
-            int idx = indl2[i] + j*points_cnt;
-            field_out(idx) = field_out_l2(idx);
-         }
-      }
-   }
-}
-
-void FindPointsGSLIB::InterpolateH1(const GridFunction &field_in,
-                                    Vector &field_out)
-{
-   FiniteElementSpace ind_fes(mesh, field_in.FESpace()->FEColl());
-   GridFunction field_in_scalar(&ind_fes);
-   Vector node_vals;
-
-   const int ncomp      = field_in.FESpace()->GetVDim(),
-             points_fld = field_in.Size() / ncomp,
-             points_cnt = gsl_code.Size();
-
-   field_out.SetSize(points_cnt*ncomp);
-   field_out = default_interp_value;
-
-   for (int i = 0; i < ncomp; i++)
-   {
-      const int dataptrin  = i*points_fld,
-                dataptrout = i*points_cnt;
-      field_in_scalar.NewDataAndSize(field_in.GetData()+dataptrin, points_fld);
-      GetNodeValues(field_in_scalar, node_vals);
-
-      if (dim==2)
-      {
-         findpts_eval_2(field_out.GetData()+dataptrout, sizeof(double),
-                        gsl_code.GetData(),       sizeof(unsigned int),
-                        gsl_proc.GetData(),    sizeof(unsigned int),
-                        gsl_elem.GetData(),    sizeof(unsigned int),
-                        gsl_ref.GetData(),     sizeof(double) * dim,
-                        points_cnt, node_vals.GetData(), fdata2D);
-      }
-      else
-      {
-         findpts_eval_3(field_out.GetData()+dataptrout, sizeof(double),
-                        gsl_code.GetData(),       sizeof(unsigned int),
-                        gsl_proc.GetData(),    sizeof(unsigned int),
-                        gsl_elem.GetData(),    sizeof(unsigned int),
-                        gsl_ref.GetData(),     sizeof(double) * dim,
-                        points_cnt, node_vals.GetData(), fdata3D);
-      }
-   }
-}
-
-void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
-                                         Vector &field_out)
-{
-   int ncomp   = field_in.VectorDim(),
-       nptorig = points_cnt,
-       npt     = points_cnt;
-
-   field_out.SetSize(points_cnt*ncomp);
-   field_out = default_interp_value;
-
-   if (gsl_comm->np == 1) // serial
-   {
-      for (int index = 0; index < npt; index++)
-      {
-         if (gsl_code[index] == 2) { continue; }
-         IntegrationPoint ip;
-         ip.Set2(gsl_mfem_ref.GetData()+index*dim);
-         if (dim == 3) { ip.z = gsl_mfem_ref(index*dim + 2); }
-         Vector localval(ncomp);
-         field_in.GetVectorValue(gsl_mfem_elem[index], ip, localval);
-         for (int i = 0; i < ncomp; i++)
-         {
-            field_out(index + i*npt) = localval(i);
-         }
-      }
-   }
-   else // parallel
-   {
-      // Determine number of points to be sent
-      int nptsend = 0;
-      for (int index = 0; index < npt; index++)
-      {
-         if (gsl_code[index] != 2) { nptsend +=1; }
-      }
-
-      // Pack data to send via crystal router
-      struct array *outpt = new array;
-      struct out_pt { double r[3], ival; uint index, el, proc; };
-      struct out_pt *pt;
-      array_init(struct out_pt, outpt, nptsend);
-      outpt->n=nptsend;
-      pt = (struct out_pt *)outpt->ptr;
-      for (int index = 0; index < npt; index++)
-      {
-         if (gsl_code[index] == 2) { continue; }
-         for (int d = 0; d < dim; ++d) { pt->r[d]= gsl_mfem_ref(index*dim + d); }
-         pt->index = index;
-         pt->proc  = gsl_proc[index];
-         pt->el    = gsl_mfem_elem[index];
-         ++pt;
-      }
-
-      // Transfer data to target MPI ranks
-      sarray_transfer(struct out_pt, outpt, proc, 1, cr);
-
-      if (ncomp == 1)
-      {
-         // Interpolate the grid function
-         npt = outpt->n;
-         pt = (struct out_pt *)outpt->ptr;
-         for (int index = 0; index < npt; index++)
-         {
-            IntegrationPoint ip;
-            ip.Set3(&pt->r[0]);
-            pt->ival = field_in.GetValue(pt->el, ip, 1);
-            ++pt;
-         }
-
-         // Transfer data back to source MPI rank
-         sarray_transfer(struct out_pt, outpt, proc, 1, cr);
-         npt = outpt->n;
-         pt = (struct out_pt *)outpt->ptr;
-         for (int index = 0; index < npt; index++)
-         {
-            field_out(pt->index) = pt->ival;
-            ++pt;
-         }
-         array_free(outpt);
-         delete outpt;
-      }
-      else // ncomp > 1
-      {
-         // Interpolate data and store in a Vector
-         npt = outpt->n;
-         pt = (struct out_pt *)outpt->ptr;
-         Vector vec_int_vals(npt*ncomp);
-         for (int index = 0; index < npt; index++)
-         {
-            IntegrationPoint ip;
-            ip.Set3(&pt->r[0]);
-            Vector localval(vec_int_vals.GetData()+index*ncomp, ncomp);
-            field_in.GetVectorValue(pt->el, ip, localval);
-            ++pt;
-         }
-
-         // Save index and proc data in a struct
-         struct array *savpt = new array;
-         struct sav_pt { uint index, proc; };
-         struct sav_pt *spt;
-         array_init(struct sav_pt, savpt, npt);
-         savpt->n=npt;
-         spt = (struct sav_pt *)savpt->ptr;
-         pt  = (struct out_pt *)outpt->ptr;
-         for (int index = 0; index < npt; index++)
-         {
-            spt->index = pt->index;
-            spt->proc  = pt->proc;
-            ++pt; ++spt;
-         }
-
-         array_free(outpt);
-         delete outpt;
-
-         // Copy data from save struct to send struct and send component wise
-         struct array *sendpt = new array;
-         struct send_pt { double ival; uint index, proc; };
-         struct send_pt *sdpt;
-         for (int j = 0; j < ncomp; j++)
-         {
-            array_init(struct send_pt, sendpt, npt);
-            sendpt->n=npt;
-            spt  = (struct sav_pt *)savpt->ptr;
-            sdpt = (struct send_pt *)sendpt->ptr;
-            for (int index = 0; index < npt; index++)
-            {
-               sdpt->index = spt->index;
-               sdpt->proc  = spt->proc;
-               sdpt->ival  = vec_int_vals(j + index*ncomp);
-               ++sdpt; ++spt;
-            }
-
-            sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
-            sdpt = (struct send_pt *)sendpt->ptr;
-            for (int index = 0; index < nptorig; index++)
-            {
-               int idx = sdpt->index + j*nptorig;
-               field_out(idx) = sdpt->ival;
-               ++sdpt;
-            }
-            array_free(sendpt);
-         }
-         array_free(savpt);
-         delete sendpt;
-         delete savpt;
-      } // ncomp > 1
-   } // parallel
+   delete meshsplit;
 }

 } // namespace mfem
@@ -20,66 +20,28 @@
 struct comm;
 struct findpts_data_2;
 struct findpts_data_3;
-struct array;
-struct crystal;

 namespace mfem
 {

-/** \brief FindPointsGSLIB can robustly evaluate a GridFunction on an arbitrary
- *  collection of points. There are three key functions in FindPointsGSLIB:
- *
- *  1. Setup - constructs the internal data structures of gslib.
- *
- *  2. FindPoints - for any given arbitrary set of points in physical space,
- *     gslib finds the element number, MPI rank, and the reference space
- *     coordinates inside the element that each point is located in. gslib also
- *     returns a code that indicates whether the point was found inside an
- *     element, on element border, or not found in the domain.
- *
- *  3. Interpolate - Interpolates any grid function at the points found using 2.
- *
- *  FindPointsGSLIB provides interface to use these functions individually or
- *  using a single call.
- */
 class FindPointsGSLIB
 {
-public:
-   enum AvgType {NONE, ARITHMETIC, HARMONIC}; // Average type for L2 functions
-
 protected:
-   Mesh *mesh, *meshsplit;
-   IntegrationRule *ir_simplex;    // IntegrationRule to split quads/hex -> simplex
-   struct findpts_data_2 *fdata2D; // gslib's internal data
-   struct findpts_data_3 *fdata3D; // gslib's internal data
-   struct crystal *cr;             // gslib's internal data
-   struct comm *gsl_comm;          // gslib's internal data
-   int dim, points_cnt;
-   Array<unsigned int> gsl_code, gsl_proc, gsl_elem, gsl_mfem_elem;
-   Vector gsl_mesh, gsl_ref, gsl_dist, gsl_mfem_ref;
-   bool setupflag;              // flag to indicate whether gslib data has been setup
-   double default_interp_value; // used for points that are not found in the mesh
-   AvgType avgtype;             // average type used for L2 functions
+   Mesh *mesh;
+   IntegrationRule *ir_simplex;
+   struct findpts_data_2 *fdata2D;
+   struct findpts_data_3 *fdata3D;
+   int dim;
+   Array<unsigned int> gsl_code, gsl_proc, gsl_elem;
+   Vector gsl_mesh, gsl_ref, gsl_dist;
+   bool setupflag;
+
+   struct comm *gsl_comm;

-   /// Get GridFunction from MFEM format to GSLIB format
   void GetNodeValues(const GridFunction &gf_in, Vector &node_vals);
-   /// Get nodal coordinates from mesh to the format expected by GSLIB for quads
-   /// and hexes
   void GetQuadHexNodalCoordinates();
-   /// Convert simplices to quad/hexes and then get nodal coordinates for each
-   /// split element into format expected by GSLIB
   void GetSimplexNodalCoordinates();

-   /// Use GSLIB for communication and interpolation
-   void InterpolateH1(const GridFunction &field_in, Vector &field_out);
-   /// Uses GSLIB Crystal Router for communication followed by MFEM's
-   /// interpolation functions
-   void InterpolateGeneral(const GridFunction &field_in, Vector &field_out);
-   /// Map {r,s,t} coordinates from [-1,1] to [0,1] for MFEM. For simplices mesh
-   /// find the original element number (that was split into micro quads/hexes
-   /// by GetSimplexNodalCoordinates())
-   void MapRefPosAndElemIndices();
-
 public:
   FindPointsGSLIB();

@@ -102,37 +64,45 @@ public:
   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
              const int npt_max = 256);

-   /** Searches positions given in physical space by @a point_pos. These positions
-       must by ordered by nodes: (XXX...,YYY...,ZZZ).
-       This function populates the following member variables:
-       #gsl_code        Return codes for each point: inside element (0),
-                        element boundary (1), not found (2).
-       #gsl_proc        MPI proc ids where the points were found.
-       #gsl_elem        Element ids where the points were found.
-                        Defaults to 0 for points that were not found.
-       #gsl_mfem_elem   Element ids corresponding to MFEM-mesh where the points
-                        were found. #gsl_mfem_elem != #gsl_elem for simplices
-                        Defaults to 0 for points that were not found.
-       #gsl_ref         Reference coordinates of the found point.
-                        Ordered by vdim (XYZ,XYZ,XYZ...). Defaults to -1 for
-                        points that were not found. Note: the gslib reference
-                        frame is [-1,1].
-       #gsl_mfem_ref    Reference coordinates #gsl_ref mapped to [0,1].
-                        Defaults to 0 for points that were not found.
-       #gsl_dist        Distance between the sought and the found point
-                        in physical space. */
+   /** Searches positions given in physical space by @a point_pos. All output
+       Arrays and Vectors are expected to have the correct size.
+
+       @param[in]  point_pos  Positions to be found. Must by ordered by nodes
+                              (XXX...,YYY...,ZZZ).
+       @param[out] codes      Return codes for each point: inside element (0),
+                              element boundary (1), not found (2).
+       @param[out] proc_ids   MPI proc ids where the points were found.
+       @param[out] elem_ids   Element ids where the points were found.
+       @param[out] ref_pos    Reference coordinates of the found point. Ordered
+                              by vdim (XYZ,XYZ,XYZ...).
+                              Note: the gslib reference frame is [-1,1].
+       @param[out] dist       Distance between the sought and the found point
+                              in physical space. */
+   void FindPoints(const Vector &point_pos, Array<unsigned int> &codes,
+                   Array<unsigned int> &proc_ids, Array<unsigned int> &elem_ids,
+                   Vector &ref_pos, Vector &dist);
   void FindPoints(const Vector &point_pos);
   /// Setup FindPoints and search positions
   void FindPoints(Mesh &m, const Vector &point_pos, const double bb_t = 0.1,
                   const double newt_tol = 1.0e-12,  const int npt_max = 256);

   /** Interpolation of field values at prescribed reference space positions.
+
+       @param[in] codes       Return codes for each point: inside element (0),
+                              element boundary (1), not found (2).
+       @param[in] proc_ids    MPI proc ids where the points were found.
+       @param[in] elem_ids    Element ids where the points were found.
+       @param[in] ref_pos     Reference coordinates of the found point. Ordered
+                              by vdim (XYZ,XYZ,XYZ...).
+                              Note: the gslib reference frame is [-1,1].
       @param[in] field_in    Function values that will be interpolated on the
                              reference positions. Note: it is assumed that
                              @a field_in is in H1 and in the same space as the
                              mesh that was given to Setup().
-       @param[out] field_out  Interpolated values. For points that are not found
-                              the value is set to #default_interp_value. */
+       @param[out] field_out  Interpolated values. */
+   void Interpolate(Array<unsigned int> &codes, Array<unsigned int> &proc_ids,
+                    Array<unsigned int> &elem_ids, Vector &ref_pos,
+                    const GridFunction &field_in, Vector &field_out);
   void Interpolate(const GridFunction &field_in, Vector &field_out);
   /** Search positions and interpolate */
   void Interpolate(const Vector &point_pos, const GridFunction &field_in,
@@ -141,45 +111,27 @@ public:
   void Interpolate(Mesh &m, const Vector &point_pos,
                    const GridFunction &field_in, Vector &field_out);

-   /// Average type to be used for L2 functions in-case a point is located at
-   /// an element boundary where the function might be multi-valued.
-   void SetL2AvgType(AvgType avgtype_) { avgtype = avgtype_; }
-
-   /// Set the default interpolation value for points that are not found in the
-   /// mesh.
-   void SetDefaultInterpolationValue(double interp_value_)
-   {
-      default_interp_value = interp_value_;
-   }
-
   /** Cleans up memory allocated internally by gslib.
-       Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. */
+       Note that in parallel, this must be called before MPI_Finalize(), as
+       it calls MPI_Comm_free() for internal gslib communicators. */
   void FreeData();

   /// Return code for each point searched by FindPoints: inside element (0), on
   /// element boundary (1), or not found (2).
   const Array<unsigned int> &GetCode() const { return gsl_code; }
   /// Return element number for each point found by FindPoints.
-   const Array<unsigned int> &GetElem() const { return gsl_mfem_elem; }
+   const Array<unsigned int> &GetElem() const { return gsl_elem; }
   /// Return MPI rank on which each point was found by FindPoints.
   const Array<unsigned int> &GetProc() const { return gsl_proc; }
   /// Return reference coordinates for each point found by FindPoints.
-   const Vector &GetReferencePosition() const { return gsl_mfem_ref;  }
+   const Vector &GetReferencePosition() const { return gsl_ref;  }
   /// Return distance Distance between the sought and the found point
   /// in physical space, for each point found by FindPoints.
   const Vector &GetDist()              const { return gsl_dist; }
-
-   /// Return element number for each point found by FindPoints corresponding to
-   /// GSLIB mesh. gsl_mfem_elem != gsl_elem for mesh with simplices.
-   const Array<unsigned int> &GetGSLIBElem() const { return gsl_elem; }
-   /// Return reference coordinates in [-1,1] (internal range in GSLIB) for each
-   /// point found by FindPoints.
-   const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 };

 } // namespace mfem

-#endif // MFEM_USE_GSLIB
+#endif //MFEM_USE_GSLIB

-#endif // MFEM_GSLIB
+#endif //MFEM_GSLIB guard
@@ -68,7 +68,7 @@ void Hybridization::ConstructC()
      {
         const int dim = pmesh->Dimension();
         const NCMesh::NCList &shared = pmesh->pncmesh->GetSharedList(dim-1);
-         num_shared_slave_faces = (HYPRE_Int) shared.slaves.Size();
+         num_shared_slave_faces = (HYPRE_Int)shared.slaves.size();
         MPI_Allreduce(&num_shared_slave_faces, &glob_num_shared_slave_faces, 1,
                       HYPRE_MPI_INT, MPI_SUM, pmesh->GetComm());
         MFEM_ASSERT(glob_num_shared_slave_faces%2 == 0, "");
@@ -11,9 +11,9 @@

 #include "ceed.hpp"

+#ifdef MFEM_USE_CEED
 #include "../../general/device.hpp"
 #include "../../fem/gridfunc.hpp"
-#include "../../linalg/dtensor.hpp"

 #include <sys/types.h>
 #include <sys/stat.h>
@@ -31,454 +31,32 @@ namespace mfem
 namespace internal
 {

-#ifdef MFEM_USE_CEED
 extern Ceed ceed;

 std::string ceed_path;

-extern CeedBasisMap ceed_basis_map;
-extern CeedRestrMap ceed_restr_map;
-#endif
-
 }

-void InitCeedCoeff(Coefficient *Q, Mesh &mesh,
-                   const IntegrationRule &ir, CeedData *ptr)
+void InitCeedCoeff(Coefficient* Q, CeedData* ptr)
 {
-#ifdef MFEM_USE_CEED
-   if ( Q == nullptr )
+   if (ConstantCoefficient* coeff = dynamic_cast<ConstantCoefficient*>(Q))
   {
-      CeedConstCoeff *ceedCoeff = new CeedConstCoeff{1.0};
+      CeedConstCoeff* ceedCoeff = new CeedConstCoeff{coeff->constant};
      ptr->coeff_type = CeedCoeff::Const;
-      ptr->coeff = static_cast<void*>(ceedCoeff);
-   }
-   else if (ConstantCoefficient *coeff = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      CeedConstCoeff *ceedCoeff = new CeedConstCoeff{coeff->constant};
-      ptr->coeff_type = CeedCoeff::Const;
-      ptr->coeff = static_cast<void*>(ceedCoeff);
+      ptr->coeff = (void*)ceedCoeff;
   }
   else if (GridFunctionCoefficient* coeff =
               dynamic_cast<GridFunctionCoefficient*>(Q))
   {
-      CeedGridCoeff *ceedCoeff = new CeedGridCoeff;
+      CeedGridCoeff* ceedCoeff = new CeedGridCoeff;
      ceedCoeff->coeff = coeff->GetGridFunction();
-      InitCeedVector(*ceedCoeff->coeff, ceedCoeff->coeffVector);
      ptr->coeff_type = CeedCoeff::Grid;
-      ptr->coeff = static_cast<void*>(ceedCoeff);
-   }
-   else if (QuadratureFunctionCoefficient *cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      CeedQuadCoeff *ceedCoeff = new CeedQuadCoeff;
-      const int ne = mesh.GetNE();
-      const int nq = ir.GetNPoints();
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      ceedCoeff->coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-      InitCeedVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
-      ptr->coeff_type = CeedCoeff::Quad;
-      ptr->coeff = static_cast<void*>(ceedCoeff);
+      ptr->coeff = (void*)ceedCoeff;
   }
   else
   {
-      CeedQuadCoeff *ceedCoeff = new CeedQuadCoeff;
-      const int ne = mesh.GetNE();
-      const int nq = ir.GetNPoints();
-      ceedCoeff->coeff.SetSize(nq * ne);
-      auto C = Reshape(ceedCoeff->coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation &T = *mesh.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir.IntPoint(q));
-         }
-      }
-      InitCeedVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
-      ptr->coeff_type = CeedCoeff::Quad;
-      ptr->coeff = static_cast<void*>(ceedCoeff);
+      MFEM_ABORT("This type of Coefficient is not supported.");
   }
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void CeedPAAssemble(const CeedPAOperator& op,
-                    CeedData& ceedData)
-{
-#ifdef MFEM_USE_CEED
-   const FiniteElementSpace &fes = op.fes;
-   const mfem::IntegrationRule &irm = op.ir;
-   Ceed ceed(internal::ceed);
-   mfem::Mesh *mesh = fes.GetMesh();
-   CeedInt nqpts, nelem = mesh->GetNE();
-   CeedInt dim = mesh->SpaceDimension(), vdim = fes.GetVDim();
-
-   mesh->EnsureNodes();
-   InitCeedBasisAndRestriction(fes, irm, ceed, &ceedData.basis, &ceedData.restr);
-
-   const mfem::FiniteElementSpace *mesh_fes = mesh->GetNodalFESpace();
-   MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
-   InitCeedBasisAndRestriction(*mesh_fes, irm, ceed, &ceedData.mesh_basis,
-                               &ceedData.mesh_restr);
-
-   CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
-
-   const int qdatasize = op.qdatasize;
-   InitCeedStridedRestriction(*mesh_fes, nelem, nqpts, qdatasize,
-                              CEED_STRIDES_BACKEND,
-                              &ceedData.restr_i);
-
-   InitCeedVector(*mesh->GetNodes(), ceedData.node_coords);
-
-   CeedVectorCreate(ceed, nelem * nqpts * qdatasize, &ceedData.rho);
-
-   // Context data to be passed to the 'f_build_diff' Q-function.
-   ceedData.build_ctx_data.dim = mesh->Dimension();
-   ceedData.build_ctx_data.space_dim = mesh->SpaceDimension();
-   ceedData.build_ctx_data.vdim = fes.GetVDim();
-
-   std::string qf_file = GetCeedPath() + op.header;
-   std::string qf;
-
-   // Create the Q-function that builds the operator (i.e. computes its
-   // quadrature data) and set its context data.
-   switch (ceedData.coeff_type)
-   {
-      case CeedCoeff::Const:
-         qf = qf_file + op.const_func;
-         CeedQFunctionCreateInterior(ceed, 1, op.const_qf,
-                                     qf.c_str(),
-                                     &ceedData.build_qfunc);
-         ceedData.build_ctx_data.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
-         break;
-      case CeedCoeff::Grid:
-         qf = qf_file + op.quad_func;
-         CeedQFunctionCreateInterior(ceed, 1, op.quad_qf,
-                                     qf.c_str(),
-                                     &ceedData.build_qfunc);
-         CeedQFunctionAddInput(ceedData.build_qfunc, "coeff", 1, CEED_EVAL_INTERP);
-         break;
-      case CeedCoeff::Quad:
-         qf = qf_file + op.quad_func;
-         CeedQFunctionCreateInterior(ceed, 1, op.quad_qf,
-                                     qf.c_str(),
-                                     &ceedData.build_qfunc);
-         CeedQFunctionAddInput(ceedData.build_qfunc, "coeff", 1, CEED_EVAL_NONE);
-         break;
-   }
-   CeedQFunctionAddInput(ceedData.build_qfunc, "dx", dim * dim, CEED_EVAL_GRAD);
-   CeedQFunctionAddInput(ceedData.build_qfunc, "weights", 1, CEED_EVAL_WEIGHT);
-   CeedQFunctionAddOutput(ceedData.build_qfunc, "qdata", qdatasize,
-                          CEED_EVAL_NONE);
-
-   CeedQFunctionContextCreate(ceed, &ceedData.build_ctx);
-   CeedQFunctionContextSetData(ceedData.build_ctx, CEED_MEM_HOST, CEED_USE_POINTER,
-                               sizeof(ceedData.build_ctx_data),
-                               &ceedData.build_ctx_data);
-   CeedQFunctionSetContext(ceedData.build_qfunc, ceedData.build_ctx);
-
-   // Create the operator that builds the quadrature data for the operator.
-   CeedOperatorCreate(ceed, ceedData.build_qfunc, NULL, NULL,
-                      &ceedData.build_oper);
-   switch (ceedData.coeff_type)
-   {
-      case CeedCoeff::Const:
-         break;
-      case CeedCoeff::Grid:
-      {
-         CeedGridCoeff* gridCoeff = (CeedGridCoeff*)ceedData.coeff;
-         InitCeedBasisAndRestriction(*gridCoeff->coeff->FESpace(), irm, ceed,
-                                     &gridCoeff->basis,
-                                     &gridCoeff->restr);
-         CeedOperatorSetField(ceedData.build_oper, "coeff", gridCoeff->restr,
-                              gridCoeff->basis, gridCoeff->coeffVector);
-      }
-      break;
-      case CeedCoeff::Quad:
-      {
-         CeedQuadCoeff* quadCoeff = (CeedQuadCoeff*)ceedData.coeff;
-         const int ncomp = 1;
-         CeedInt strides[3] = {1, nqpts, ncomp*nqpts};
-         InitCeedStridedRestriction(*mesh_fes, nelem, nqpts, ncomp, strides,
-                                    &quadCoeff->restr);
-         CeedOperatorSetField(ceedData.build_oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
-      }
-      break;
-   }
-   CeedOperatorSetField(ceedData.build_oper, "dx", ceedData.mesh_restr,
-                        ceedData.mesh_basis, CEED_VECTOR_ACTIVE);
-   CeedOperatorSetField(ceedData.build_oper, "weights", CEED_ELEMRESTRICTION_NONE,
-                        ceedData.mesh_basis, CEED_VECTOR_NONE);
-   CeedOperatorSetField(ceedData.build_oper, "qdata", ceedData.restr_i,
-                        CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
-
-   // Compute the quadrature data for the operator.
-   CeedOperatorApply(ceedData.build_oper, ceedData.node_coords, ceedData.rho,
-                     CEED_REQUEST_IMMEDIATE);
-
-   // Create the Q-function that defines the action of the operator.
-   qf = qf_file + op.apply_func;
-   CeedQFunctionCreateInterior(ceed, 1, op.apply_qf,
-                               qf.c_str(),
-                               &ceedData.apply_qfunc);
-   CeedInt dimU = vdim*(op.trial_op==CEED_EVAL_GRAD ? dim : 1);
-   CeedInt dimV = vdim*(op.test_op==CEED_EVAL_GRAD ? dim : 1);
-   CeedQFunctionAddInput(ceedData.apply_qfunc, "u", dimU, op.trial_op);
-   CeedQFunctionAddInput(ceedData.apply_qfunc, "qdata", qdatasize,
-                         CEED_EVAL_NONE);
-   CeedQFunctionAddOutput(ceedData.apply_qfunc, "v", dimV, op.test_op);
-   CeedQFunctionSetContext(ceedData.apply_qfunc, ceedData.build_ctx);
-
-   // Create the operator.
-   CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);
-   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr, ceedData.basis,
-                        CEED_VECTOR_ACTIVE);
-   CeedOperatorSetField(ceedData.oper, "qdata", ceedData.restr_i,
-                        CEED_BASIS_COLLOCATED, ceedData.rho);
-   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr, ceedData.basis,
-                        CEED_VECTOR_ACTIVE);
-
-   CeedVectorCreate(ceed, vdim*fes.GetNDofs(), &ceedData.u);
-   CeedVectorCreate(ceed, vdim*fes.GetNDofs(), &ceedData.v);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void CeedMFAssemble(const CeedMFOperator& op,
-                    CeedData& ceedData)
-{
-#ifdef MFEM_USE_CEED
-   const FiniteElementSpace &fes = op.fes;
-   const mfem::IntegrationRule &irm = op.ir;
-   Ceed ceed(internal::ceed);
-   mfem::Mesh *mesh = fes.GetMesh();
-   CeedInt nqpts, nelem = mesh->GetNE();
-   CeedInt dim = mesh->SpaceDimension(), vdim = fes.GetVDim();
-
-   mesh->EnsureNodes();
-   InitCeedBasisAndRestriction(fes, irm, ceed, &ceedData.basis, &ceedData.restr);
-
-   const mfem::FiniteElementSpace *mesh_fes = mesh->GetNodalFESpace();
-   MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
-   InitCeedBasisAndRestriction(*mesh_fes, irm, ceed, &ceedData.mesh_basis,
-                               &ceedData.mesh_restr);
-
-   CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
-
-   InitCeedVector(*mesh->GetNodes(), ceedData.node_coords);
-
-   // Context data to be passed to the Q-function.
-   ceedData.build_ctx_data.dim = mesh->Dimension();
-   ceedData.build_ctx_data.space_dim = mesh->SpaceDimension();
-   ceedData.build_ctx_data.vdim = fes.GetVDim();
-
-   std::string qf_file = GetCeedPath() + op.header;
-   std::string qf;
-
-   // Create the Q-function that builds the operator (i.e. computes its
-   // quadrature data) and set its context data.
-   CeedInt dimU = vdim*(op.trial_op==CEED_EVAL_GRAD ? dim : 1);
-   CeedInt dimV = vdim*(op.test_op==CEED_EVAL_GRAD ? dim : 1);
-   switch (ceedData.coeff_type)
-   {
-      case CeedCoeff::Const:
-         qf = qf_file + op.const_func;
-         CeedQFunctionCreateInterior(ceed, 1, op.const_qf,
-                                     qf.c_str(),
-                                     &ceedData.apply_qfunc);
-         ceedData.build_ctx_data.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
-         break;
-      case CeedCoeff::Grid:
-         qf = qf_file + op.quad_func;
-         CeedQFunctionCreateInterior(ceed, 1, op.quad_qf,
-                                     qf.c_str(),
-                                     &ceedData.apply_qfunc);
-         CeedQFunctionAddInput(ceedData.apply_qfunc, "coeff", 1, CEED_EVAL_INTERP);
-         break;
-      case CeedCoeff::Quad:
-         qf = qf_file + op.quad_func;
-         CeedQFunctionCreateInterior(ceed, 1, op.quad_qf,
-                                     qf.c_str(),
-                                     &ceedData.apply_qfunc);
-         CeedQFunctionAddInput(ceedData.apply_qfunc, "coeff", 1, CEED_EVAL_NONE);
-         break;
-   }
-   CeedQFunctionAddInput(ceedData.apply_qfunc, "u", dimU, op.trial_op);
-   CeedQFunctionAddInput(ceedData.apply_qfunc, "dx", dim * dim, CEED_EVAL_GRAD);
-   CeedQFunctionAddInput(ceedData.apply_qfunc, "weights", 1, CEED_EVAL_WEIGHT);
-   CeedQFunctionAddOutput(ceedData.apply_qfunc, "v", dimV, op.test_op);
-
-   CeedQFunctionContextCreate(ceed, &ceedData.build_ctx);
-   CeedQFunctionContextSetData(ceedData.build_ctx, CEED_MEM_HOST, CEED_USE_POINTER,
-                               sizeof(ceedData.build_ctx_data),
-                               &ceedData.build_ctx_data);
-   CeedQFunctionSetContext(ceedData.apply_qfunc, ceedData.build_ctx);
-
-   // Create the operator.
-   CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);
-   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr, ceedData.basis,
-                        CEED_VECTOR_ACTIVE);
-   switch (ceedData.coeff_type)
-   {
-      case CeedCoeff::Const:
-         break;
-      case CeedCoeff::Grid:
-      {
-         CeedGridCoeff* gridCoeff = (CeedGridCoeff*)ceedData.coeff;
-         InitCeedBasisAndRestriction(*gridCoeff->coeff->FESpace(), irm, ceed,
-                                     &gridCoeff->basis,
-                                     &gridCoeff->restr);
-         CeedOperatorSetField(ceedData.oper, "coeff", gridCoeff->restr,
-                              gridCoeff->basis, gridCoeff->coeffVector);
-      }
-      break;
-      case CeedCoeff::Quad:
-      {
-         CeedQuadCoeff* quadCoeff = (CeedQuadCoeff*)ceedData.coeff;
-         const int ncomp = 1;
-         CeedInt strides[3] = {1, nqpts, ncomp*nqpts};
-         InitCeedStridedRestriction(*mesh->GetNodalFESpace(),
-                                    nelem, nqpts, ncomp, strides,
-                                    &quadCoeff->restr);
-         CeedOperatorSetField(ceedData.oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
-      }
-      break;
-   }
-   CeedOperatorSetField(ceedData.oper, "dx", ceedData.mesh_restr,
-                        ceedData.mesh_basis, ceedData.node_coords);
-   CeedOperatorSetField(ceedData.oper, "weights", CEED_ELEMRESTRICTION_NONE,
-                        ceedData.mesh_basis, CEED_VECTOR_NONE);
-   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr, ceedData.basis,
-                        CEED_VECTOR_ACTIVE);
-
-   CeedVectorCreate(ceed, vdim*fes.GetNDofs(), &ceedData.u);
-   CeedVectorCreate(ceed, vdim*fes.GetNDofs(), &ceedData.v);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void CeedAddMult(const CeedData *ceedDataPtr,
-                 const Vector &x,
-                 Vector &y)
-{
-#ifdef MFEM_USE_CEED
-   const CeedScalar *x_ptr;
-   CeedScalar *y_ptr;
-   CeedMemType mem;
-   CeedGetPreferredMemType(internal::ceed, &mem);
-   if ( Device::Allows(Backend::DEVICE_MASK) && mem==CEED_MEM_DEVICE )
-   {
-      x_ptr = x.Read();
-      y_ptr = y.ReadWrite();
-   }
-   else
-   {
-      x_ptr = x.HostRead();
-      y_ptr = y.HostReadWrite();
-      mem = CEED_MEM_HOST;
-   }
-   CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
-                      const_cast<CeedScalar*>(x_ptr));
-   CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
-
-   CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
-                        CEED_REQUEST_IMMEDIATE);
-
-   CeedVectorTakeArray(ceedDataPtr->u, mem, const_cast<CeedScalar**>(&x_ptr));
-   CeedVectorTakeArray(ceedDataPtr->v, mem, &y_ptr);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void CeedAssembleDiagonal(const CeedData *ceedDataPtr,
-                          Vector &diag)
-{
-#ifdef MFEM_USE_CEED
-   CeedScalar *d_ptr;
-   CeedMemType mem;
-   CeedGetPreferredMemType(internal::ceed, &mem);
-   if ( Device::Allows(Backend::DEVICE_MASK) && mem==CEED_MEM_DEVICE )
-   {
-      d_ptr = diag.ReadWrite();
-   }
-   else
-   {
-      d_ptr = diag.HostReadWrite();
-      mem = CEED_MEM_HOST;
-   }
-   CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, d_ptr);
-
-   CeedOperatorLinearAssembleAddDiagonal(ceedDataPtr->oper, ceedDataPtr->v,
-                                         CEED_REQUEST_IMMEDIATE);
-
-   CeedVectorTakeArray(ceedDataPtr->v, mem, &d_ptr);
-#else
-   mfem_error("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-void RemoveCeedBasisAndRestriction(const FiniteElementSpace *fes)
-{
-#ifdef MFEM_USE_CEED
-   auto itb = internal::ceed_basis_map.begin();
-   while (itb != internal::ceed_basis_map.end())
-   {
-      if (std::get<0>(itb->first)==fes)
-      {
-         CeedBasisDestroy(&itb->second);
-         itb = internal::ceed_basis_map.erase(itb);
-      }
-      else
-      {
-         itb++;
-      }
-   }
-   auto itr = internal::ceed_restr_map.begin();
-   while (itr != internal::ceed_restr_map.end())
-   {
-      if (std::get<0>(itr->first)==fes)
-      {
-         CeedElemRestrictionDestroy(&itr->second);
-         itr = internal::ceed_restr_map.erase(itr);
-      }
-      else
-      {
-         itr++;
-      }
-   }
-#endif
-}
-
-#ifdef MFEM_USE_CEED
-void InitCeedVector(const Vector &v, CeedVector &cv)
-{
-   CeedVectorCreate(internal::ceed, v.Size(), &cv);
-   CeedScalar *cv_ptr;
-   CeedMemType mem;
-   CeedGetPreferredMemType(internal::ceed, &mem);
-   if ( Device::Allows(Backend::DEVICE_MASK) && mem==CEED_MEM_DEVICE )
-   {
-      cv_ptr = const_cast<CeedScalar*>(v.Read());
-   }
-   else
-   {
-      cv_ptr = const_cast<CeedScalar*>(v.HostRead());
-      mem = CEED_MEM_HOST;
-   }
-   CeedVectorSetArray(cv, mem, CEED_USE_POINTER, cv_ptr);
 }

 static CeedElemTopology GetCeedTopology(Geometry::Type geom)
@@ -503,206 +81,191 @@ static CeedElemTopology GetCeedTopology(Geometry::Type geom)
   }
 }

-static void InitCeedNonTensorBasis(const FiniteElementSpace &fes,
-                                   const IntegrationRule &ir,
-                                   Ceed ceed, CeedBasis *basis)
-{
-   const DofToQuad &maps = fes.GetFE(0)->GetDofToQuad(ir, DofToQuad::FULL);
-   Mesh *mesh = fes.GetMesh();
-   const int dim = mesh->Dimension();
-   const int ndofs = maps.ndof;
-   const int nqpts = maps.nqpt;
-   DenseMatrix qX(dim,nqpts);
-   Vector qW(nqpts);
-   for (int i = 0; i < nqpts; i++)
-   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      qX(0,i) = ip.x;
-      if (dim>1) { qX(1,i) = ip.y; }
-      if (dim>2) { qX(2,i) = ip.z; }
-      qW(i) = ip.weight;
-   }
-   CeedBasisCreateH1(ceed, GetCeedTopology(fes.GetFE(0)->GetGeomType()),
-                     fes.GetVDim(), ndofs, nqpts,
-                     maps.Bt.GetData(), maps.Gt.GetData(),
-                     qX.GetData(), qW.GetData(), basis);
-}
-
-static void InitCeedNonTensorRestriction(const FiniteElementSpace &fes,
-                                         Ceed ceed, CeedElemRestriction *restr)
+static void InitCeedNonTensorBasisAndRestriction(const FiniteElementSpace &fes,
+                                                 const IntegrationRule &ir,
+                                                 Ceed ceed, CeedBasis *basis,
+                                                 CeedElemRestriction *restr)
 {
   Mesh *mesh = fes.GetMesh();
   const FiniteElement *fe = fes.GetFE(0);
+   const int dim = mesh->Dimension();
   const int P = fe->GetDof();
+   const int Q = ir.GetNPoints();
+   DenseMatrix shape(P, Q);
+   Vector grad(P*dim*Q);
+   DenseMatrix qref(dim, Q);
+   Vector qweight(Q);
+   Vector shape_i(P);
+   DenseMatrix grad_i(P, dim);
+
   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
   const Table &el_dof = fes.GetElementToDofTable();
   Array<int> tp_el_dof(el_dof.Size_of_connections());
   const TensorBasisElement * tfe =
      dynamic_cast<const TensorBasisElement *>(fe);
-   const int stride = compstride == 1 ? fes.GetVDim() : 1;
   if (tfe) // Lexicographic ordering using dof_map
   {
      const Array<int>& dof_map = tfe->GetDofMap();
-      for (int i = 0; i < mesh->GetNE(); i++)
+      for (int i = 0; i < Q; i++)
      {
-         const int el_offset = P * i;
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         qref(0,i) = ip.x;
+         if (dim>1) { qref(1,i) = ip.y; }
+         if (dim>2) { qref(2,i) = ip.z; }
+         qweight(i) = ip.weight;
+         fe->CalcShape(ip, shape_i);
+         fe->CalcDShape(ip, grad_i);
         for (int j = 0; j < P; j++)
         {
-            tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[dof_map[j]+el_offset];
+            shape(j, i) = shape_i(dof_map[j]);
+            for (int d = 0; d < dim; ++d)
+            {
+               grad(j+i*P+d*Q*P) = grad_i(dof_map[j], d);
+            }
+         }
+      }
+
+      for (int i = 0; i < mesh->GetNE(); i++)
+      {
+         const int el_offset = fe->GetDof() * i;
+         for (int j = 0; j < fe->GetDof(); j++)
+         {
+            if (compstride == 1)
+            {
+               tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                          el_dof.GetJ()[dof_map[j] + el_offset];
+            }
+            else
+            {
+               tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
+            }
         }
      }
   }
   else  // Native ordering
   {
+      for (int i = 0; i < Q; i++)
+      {
+         const IntegrationPoint &ip = ir.IntPoint(i);
+         qref(0,i) = ip.x;
+         if (dim>1) { qref(1,i) = ip.y; }
+         if (dim>2) { qref(2,i) = ip.z; }
+         qweight(i) = ip.weight;
+         fe->CalcShape(ip, shape_i);
+         fe->CalcDShape(ip, grad_i);
+         for (int j = 0; j < P; j++)
+         {
+            shape(j, i) = shape_i(j);
+            for (int d = 0; d < dim; ++d)
+            {
+               grad(j+i*P+d*Q*P) = grad_i(j, d);
+            }
+         }
+      }
+
      for (int e = 0; e < mesh->GetNE(); e++)
      {
         for (int i = 0; i < P; i++)
         {
-            tp_el_dof[i + e*P] = stride*el_dof.GetJ()[i + e*P];
+            if (compstride == 1)
+            {
+               tp_el_dof[i + e*P] = fes.GetVDim()*el_dof.GetJ()[i + e*P];
+            }
+            else
+            {
+               tp_el_dof[i + e*P] = el_dof.GetJ()[i + e*P];
+            }
         }
      }
   }
-   CeedElemRestrictionCreate(ceed, mesh->GetNE(), P, fes.GetVDim(),
+   CeedBasisCreateH1(ceed, GetCeedTopology(fe->GetGeomType()), fes.GetVDim(),
+                     fe->GetDof(), ir.GetNPoints(), shape.GetData(),
+                     grad.GetData(), qref.GetData(), qweight.GetData(), basis);
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
                             CEED_MEM_HOST, CEED_COPY_VALUES,
                             tp_el_dof.GetData(), restr);
 }

-static void InitCeedTensorBasis(const FiniteElementSpace &fes,
-                                const IntegrationRule &ir,
-                                Ceed ceed, CeedBasis *basis)
-{
-   const DofToQuad &maps = fes.GetFE(0)->GetDofToQuad(ir, DofToQuad::TENSOR);
-   Mesh *mesh = fes.GetMesh();
-   const int ndofs = maps.ndof;
-   const int nqpts = maps.nqpt;
-   Vector qX(nqpts), qW(nqpts);
-   const IntegrationRule &ir1d = IntRules.Get(Geometry::SEGMENT, ir.GetOrder());
-   for (int i = 0; i < nqpts; i++)
-   {
-      const IntegrationPoint &ip = ir1d.IntPoint(i);
-      qX(i) = ip.x;
-      qW(i) = ip.weight;
-   }
-   CeedBasisCreateTensorH1(ceed, mesh->Dimension(), fes.GetVDim(), ndofs,
-                           nqpts, maps.Bt.GetData(),
-                           maps.Gt.GetData(), qX.GetData(),
-                           qW.GetData(), basis);
-}
-
-static void InitCeedTensorRestriction(const FiniteElementSpace &fes,
-                                      Ceed ceed, CeedElemRestriction *restr)
+static void InitCeedTensorBasisAndRestriction(const FiniteElementSpace &fes,
+                                              const IntegrationRule &ir,
+                                              Ceed ceed, CeedBasis *basis,
+                                              CeedElemRestriction *restr)
 {
   Mesh *mesh = fes.GetMesh();
   const FiniteElement *fe = fes.GetFE(0);
+   const int order = fes.GetOrder(0);
   const TensorBasisElement * tfe =
      dynamic_cast<const TensorBasisElement *>(fe);
   MFEM_VERIFY(tfe, "invalid FE");
   const Array<int>& dof_map = tfe->GetDofMap();
+   const FiniteElement *fe1d =
+      fes.FEColl()->FiniteElementForGeometry(Geometry::SEGMENT);
+   DenseMatrix shape1d(fe1d->GetDof(), ir.GetNPoints());
+   DenseMatrix grad1d(fe1d->GetDof(), ir.GetNPoints());
+   Vector qref1d(ir.GetNPoints()), qweight1d(ir.GetNPoints());
+   Vector shape_i(shape1d.Height());
+   DenseMatrix grad_i(grad1d.Height(), 1);
+   const H1_SegmentElement *h1_fe1d =
+      dynamic_cast<const H1_SegmentElement *>(fe1d);
+   MFEM_VERIFY(h1_fe1d, "invalid FE");
+   const Array<int> &dof_map_1d = h1_fe1d->GetDofMap();
+   for (int i = 0; i < ir.GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir.IntPoint(i);
+      qref1d(i) = ip.x;
+      qweight1d(i) = ip.weight;
+      fe1d->CalcShape(ip, shape_i);
+      fe1d->CalcDShape(ip, grad_i);
+      for (int j = 0; j < shape1d.Height(); j++)
+      {
+         shape1d(j, i) = shape_i(dof_map_1d[j]);
+         grad1d(j, i) = grad_i(dof_map_1d[j], 0);
+      }
+   }
+   CeedBasisCreateTensorH1(ceed, mesh->Dimension(), fes.GetVDim(), order + 1,
+                           ir.GetNPoints(), shape1d.GetData(),
+                           grad1d.GetData(), qref1d.GetData(),
+                           qweight1d.GetData(), basis);

   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
   const Table &el_dof = fes.GetElementToDofTable();
   Array<int> tp_el_dof(el_dof.Size_of_connections());
-   const int dof = fe->GetDof();
-   const int stride = compstride == 1 ? fes.GetVDim() : 1;
-   if (dof_map.Size()>0)
+   for (int i = 0; i < mesh->GetNE(); i++)
   {
-      for (int i = 0; i < mesh->GetNE(); i++)
+      const int el_offset = fe->GetDof() * i;
+      for (int j = 0; j < fe->GetDof(); j++)
      {
-         const int el_offset = dof * i;
-         for (int j = 0; j < dof; j++)
+         if (compstride == 1)
         {
-            tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[dof_map[j]+el_offset];
+            tp_el_dof[j + el_offset] = fes.GetVDim()*
+                                       el_dof.GetJ()[dof_map[j] + el_offset];
+         }
+         else
+         {
+            tp_el_dof[j + el_offset] = el_dof.GetJ()[dof_map[j] + el_offset];
         }
      }
   }
-   else // dof_map.Size == 0, means dof_map[j]==j;
-   {
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         const int el_offset = dof * i;
-         for (int j = 0; j < dof; j++)
-         {
-            tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[j+el_offset];
-         }
-      }
-   }
-   CeedElemRestrictionCreate(ceed, mesh->GetNE(), dof, fes.GetVDim(),
+   CeedElemRestrictionCreate(ceed, mesh->GetNE(), fe->GetDof(), fes.GetVDim(),
                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
                             CEED_MEM_HOST, CEED_COPY_VALUES,
                             tp_el_dof.GetData(), restr);
 }

-void InitCeedStridedRestriction(const FiniteElementSpace &fes,
-                                CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
-                                const CeedInt *strides,
-                                CeedElemRestriction *restr)
-{
-   CeedRestrKey restr_key(&fes, nelem, nqpts, qdatasize);
-   auto restr_itr = internal::ceed_restr_map.find(restr_key);
-   if (restr_itr == internal::ceed_restr_map.end())
-   {
-      CeedElemRestrictionCreateStrided(internal::ceed, nelem, nqpts, qdatasize,
-                                       nelem*nqpts*qdatasize,
-                                       strides,
-                                       restr);
-      internal::ceed_restr_map[restr_key] = *restr;
-   }
-   else
-   {
-      *restr = restr_itr->second;
-   }
-}
-
 void InitCeedBasisAndRestriction(const FiniteElementSpace &fes,
                                 const IntegrationRule &irm,
                                 Ceed ceed, CeedBasis *basis,
                                 CeedElemRestriction *restr)
 {
-   // Check for FES -> basis, restriction in hash tables
-   const Mesh *mesh = fes.GetMesh();
-   const FiniteElement *fe = fes.GetFE(0);
-   const int P = fe->GetDof();
-   const int Q = irm.GetNPoints();
-   const int nelem = mesh->GetNE();
-   const int ncomp = fes.GetVDim();
-   CeedBasisKey basis_key(&fes, &irm, ncomp, P, Q);
-   auto basis_itr = internal::ceed_basis_map.find(basis_key);
-   CeedRestrKey restr_key(&fes, nelem, P, ncomp);
-   auto restr_itr = internal::ceed_restr_map.find(restr_key);
-
-   // Init or retreive key values
-   if (basis_itr == internal::ceed_basis_map.end())
+   if (UsesTensorBasis(fes))
   {
-      if (UsesTensorBasis(fes))
-      {
-         InitCeedTensorBasis(fes, irm, ceed, basis);
-      }
-      else
-      {
-         InitCeedNonTensorBasis(fes, irm, ceed, basis);
-      }
-      internal::ceed_basis_map[basis_key] = *basis;
+      const IntegrationRule &ir = IntRules.Get(Geometry::SEGMENT, irm.GetOrder());
+      InitCeedTensorBasisAndRestriction(fes, ir, ceed, basis, restr);
   }
   else
   {
-      *basis = basis_itr->second;
-   }
-   if (restr_itr == internal::ceed_restr_map.end())
-   {
-      if (UsesTensorBasis(fes))
-      {
-         InitCeedTensorRestriction(fes, ceed, restr);
-      }
-      else
-      {
-         InitCeedNonTensorRestriction(fes, ceed, restr);
-      }
-      internal::ceed_restr_map[restr_key] = *restr;
-   }
-   else
-   {
-      *restr = restr_itr->second;
+      InitCeedNonTensorBasisAndRestriction(fes, irm, ceed, basis, restr);
   }
 }

@@ -732,6 +295,179 @@ const std::string &GetCeedPath()
   return internal::ceed_path;
 }

-#endif // MFEM_USE_CEED
+void CeedPAAssemble(const CeedPAOperator& op,
+                    CeedData& ceedData)
+{
+   const FiniteElementSpace &fes = op.fes;
+   const mfem::IntegrationRule &irm = op.ir;
+   Ceed ceed(internal::ceed);
+   mfem::Mesh *mesh = fes.GetMesh();
+   CeedInt nqpts, nelem = mesh->GetNE();
+   CeedInt dim = mesh->SpaceDimension(), vdim = fes.GetVDim();
+
+   mesh->EnsureNodes();
+   InitCeedBasisAndRestriction(fes, irm, ceed, &ceedData.basis, &ceedData.restr);
+
+   const mfem::FiniteElementSpace *mesh_fes = mesh->GetNodalFESpace();
+   MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
+   InitCeedBasisAndRestriction(*mesh_fes, irm, ceed, &ceedData.mesh_basis,
+                               &ceedData.mesh_restr);
+
+   CeedBasisGetNumQuadraturePoints(ceedData.basis, &nqpts);
+
+   const int qdatasize = op.qdatasize;
+   CeedElemRestrictionCreateStrided(ceed, nelem, nqpts, qdatasize,
+                                    nelem*nqpts*qdatasize, CEED_STRIDES_BACKEND,
+                                    &ceedData.restr_i);
+
+   CeedVectorCreate(ceed, mesh->GetNodes()->Size(), &ceedData.node_coords);
+   CeedVectorSetArray(ceedData.node_coords, CEED_MEM_HOST, CEED_USE_POINTER,
+                      mesh->GetNodes()->GetData());
+
+   CeedVectorCreate(ceed, nelem * nqpts * qdatasize, &ceedData.rho);
+
+   // Context data to be passed to the 'f_build_diff' Q-function.
+   ceedData.build_ctx.dim = mesh->Dimension();
+   ceedData.build_ctx.space_dim = mesh->SpaceDimension();
+
+   std::string qf_file = GetCeedPath() + op.header;
+   std::string qf;
+
+   // Create the Q-function that builds the operator (i.e. computes its
+   // quadrature data) and set its context data.
+   switch (ceedData.coeff_type)
+   {
+      case CeedCoeff::Const:
+         qf = qf_file + op.const_func;
+         CeedQFunctionCreateInterior(ceed, 1, op.const_qf,
+                                     qf.c_str(),
+                                     &ceedData.build_qfunc);
+         ceedData.build_ctx.coeff = ((CeedConstCoeff*)ceedData.coeff)->val;
+         break;
+      case CeedCoeff::Grid:
+         qf = qf_file + op.grid_func;
+         CeedQFunctionCreateInterior(ceed, 1, op.grid_qf,
+                                     qf.c_str(),
+                                     &ceedData.build_qfunc);
+         CeedQFunctionAddInput(ceedData.build_qfunc, "coeff", 1, CEED_EVAL_INTERP);
+         break;
+      default:
+         MFEM_ABORT("This coeff_type is not handled");
+   }
+   CeedQFunctionAddInput(ceedData.build_qfunc, "dx", dim * dim, CEED_EVAL_GRAD);
+   CeedQFunctionAddInput(ceedData.build_qfunc, "weights", 1, CEED_EVAL_WEIGHT);
+   CeedQFunctionAddOutput(ceedData.build_qfunc, "qdata", qdatasize,
+                          CEED_EVAL_NONE);
+   CeedQFunctionSetContext(ceedData.build_qfunc, &ceedData.build_ctx,
+                           sizeof(ceedData.build_ctx));
+
+   // Create the operator that builds the quadrature data for the operator.
+   CeedOperatorCreate(ceed, ceedData.build_qfunc, NULL, NULL,
+                      &ceedData.build_oper);
+   if (ceedData.coeff_type==CeedCoeff::Grid)
+   {
+      CeedGridCoeff* ceedCoeff = (CeedGridCoeff*)ceedData.coeff;
+      InitCeedBasisAndRestriction(*ceedCoeff->coeff->FESpace(), irm, ceed,
+                                  &ceedCoeff->basis,
+                                  &ceedCoeff->restr);
+      CeedVectorCreate(ceed, ceedCoeff->coeff->FESpace()->GetNDofs(),
+                       &ceedCoeff->coeffVector);
+      CeedVectorSetArray(ceedCoeff->coeffVector, CEED_MEM_HOST, CEED_USE_POINTER,
+                         ceedCoeff->coeff->GetData());
+      CeedOperatorSetField(ceedData.build_oper, "coeff", ceedCoeff->restr,
+                           ceedCoeff->basis, ceedCoeff->coeffVector);
+   }
+   CeedOperatorSetField(ceedData.build_oper, "dx", ceedData.mesh_restr,
+                        ceedData.mesh_basis, CEED_VECTOR_ACTIVE);
+   CeedOperatorSetField(ceedData.build_oper, "weights", CEED_ELEMRESTRICTION_NONE,
+                        ceedData.mesh_basis, CEED_VECTOR_NONE);
+   CeedOperatorSetField(ceedData.build_oper, "qdata", ceedData.restr_i,
+                        CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+
+   // Compute the quadrature data for the operator.
+   CeedOperatorApply(ceedData.build_oper, ceedData.node_coords, ceedData.rho,
+                     CEED_REQUEST_IMMEDIATE);
+
+   // Create the Q-function that defines the action of the operator.
+   qf = qf_file + op.apply_func;//":f_apply_diff";
+   CeedQFunctionCreateInterior(ceed, 1, op.apply_qf,
+                               qf.c_str(),
+                               &ceedData.apply_qfunc);
+   CeedInt dimU = vdim*(op.trial_op==CEED_EVAL_GRAD ? dim : 1);
+   CeedInt dimV = vdim*(op.test_op==CEED_EVAL_GRAD ? dim : 1);
+   CeedQFunctionAddInput(ceedData.apply_qfunc, "u", dimU, op.trial_op);
+   CeedQFunctionAddInput(ceedData.apply_qfunc, "qdata", qdatasize,
+                         CEED_EVAL_NONE);
+   CeedQFunctionAddOutput(ceedData.apply_qfunc, "v", dimV, op.test_op);
+   CeedQFunctionSetContext(ceedData.apply_qfunc, &ceedData.build_ctx,
+                           sizeof(ceedData.build_ctx));
+
+   // Create the diff operator.
+   CeedOperatorCreate(ceed, ceedData.apply_qfunc, NULL, NULL, &ceedData.oper);
+   CeedOperatorSetField(ceedData.oper, "u", ceedData.restr, ceedData.basis,
+                        CEED_VECTOR_ACTIVE);
+   CeedOperatorSetField(ceedData.oper, "qdata", ceedData.restr_i,
+                        CEED_BASIS_COLLOCATED, ceedData.rho);
+   CeedOperatorSetField(ceedData.oper, "v", ceedData.restr, ceedData.basis,
+                        CEED_VECTOR_ACTIVE);
+
+   CeedVectorCreate(ceed, fes.GetNDofs(), &ceedData.u);
+   CeedVectorCreate(ceed, fes.GetNDofs(), &ceedData.v);
+}
+
+void CeedAddMultPA(const CeedData *ceedDataPtr,
+                   const Vector &x,
+                   Vector &y)
+{
+   const CeedScalar *x_ptr;
+   CeedScalar *y_ptr;
+   CeedMemType mem;
+   CeedGetPreferredMemType(internal::ceed, &mem);
+   if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
+   {
+      x_ptr = x.Read();
+      y_ptr = y.ReadWrite();
+   }
+   else
+   {
+      x_ptr = x.HostRead();
+      y_ptr = y.HostReadWrite();
+      mem = CEED_MEM_HOST;
+   }
+   CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
+                      const_cast<CeedScalar*>(x_ptr));
+   CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
+
+   CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
+                        CEED_REQUEST_IMMEDIATE);
+
+   CeedVectorTakeArray(ceedDataPtr->u, mem, const_cast<CeedScalar**>(&x_ptr));
+   CeedVectorTakeArray(ceedDataPtr->v, mem, &y_ptr);
+}
+
+void CeedAssembleDiagonalPA(const CeedData *ceedDataPtr,
+                            Vector &diag)
+{
+   CeedScalar *d_ptr;
+   CeedMemType mem;
+   CeedGetPreferredMemType(internal::ceed, &mem);
+   if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
+   {
+      d_ptr = diag.ReadWrite();
+   }
+   else
+   {
+      d_ptr = diag.HostReadWrite();
+      mem = CEED_MEM_HOST;
+   }
+   CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, d_ptr);
+
+   CeedOperatorLinearAssembleAddDiagonal(ceedDataPtr->oper, ceedDataPtr->v,
+                                         CEED_REQUEST_IMMEDIATE);
+
+   CeedVectorTakeArray(ceedDataPtr->v, mem, &d_ptr);
+}

 } // namespace mfem
+
+#endif // MFEM_USE_CEED
--- a/Show More
+++ b/Show More