Updated NVCC build with device linkable option

2019-05-13 14:18:51 -07:00
309 changed files with 9992 additions and 90851 deletions
@@ -26,18 +26,19 @@ install:
 - cd ..

 # Install hypre
- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
- cd hypre-2-10-0b
- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz'
+- 7z x hypre-2.10.0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2.10.0b
+- cmake -Hsrc -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+# - cmake -Hsrc -Bbuild -DCMAKE_BUILD_TYPE=Release -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
 - cd ..

 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2.10.0b\src\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2.10.0b\src\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include

 build_script:
 - cmake --build build_parallel
@@ -9,7 +9,6 @@
 # Object and library files
 *.o
 /libmfem.*
-/miniapps/common/libmfem-common.*

 # CMake generated files
 CMakeCache.txt
@@ -54,7 +53,6 @@ examples/displaced.mesh
 examples/mesh.*
 examples/ex5.mesh
 examples/Example5*
-examples/PVExample*
 examples/Example9*
 examples/Example15*
 examples/Example16*
@@ -62,8 +60,6 @@ examples/sphere_refined.*
 examples/sol.*
 examples/sol_u.*
 examples/sol_p.*
-examples/sol_r.*
-examples/sol_i.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
@@ -86,9 +82,9 @@ examples/ex20.dat
 examples/ex20p_?????.dat
 examples/gnuplot_ex20.inp
 examples/gnuplot_ex20p.inp
-examples/ex21*.mesh
-examples/ex21*.sol
-examples/ex21p_*.*
+examples/ex22*.mesh
+examples/ex22*.sol
+examples/ex22p_*.*

 examples/sundials/ex9
 examples/sundials/ex1[06]
@@ -117,7 +113,6 @@ examples/petsc/sol.*
 examples/petsc/sol_p.*
 examples/petsc/sol_u.*
 examples/petsc/Example5*
-examples/petsc/ex9.mesh
 examples/petsc/ex9-mesh.*
 examples/petsc/ex9-init.*
 examples/petsc/ex9-final.*
@@ -129,11 +124,6 @@ examples/petsc/elastic_energy.*
 examples/pumi/ex1
 examples/pumi/ex[126]p

-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
-
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
@@ -180,46 +170,16 @@ miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
-miniapps/tools/get-values

-miniapps/toys/automata
-miniapps/toys/life
-miniapps/toys/mandel
-miniapps/toys/rubik
-miniapps/toys/snake
-miniapps/toys/lissajous
-miniapps/toys/mondrian
-
-miniapps/toys/snake-init.mesh
-miniapps/toys/snake-user.mesh
-miniapps/toys/snake-joined.mesh
-miniapps/toys/snake-c*.mesh
-miniapps/toys/automata.gf
-miniapps/toys/automata.mesh
-miniapps/toys/rubik-init.mesh
-miniapps/toys/mandel.mesh
-miniapps/toys/life.gf
-miniapps/toys/life.mesh
-miniapps/toys/lissajous.mesh
-miniapps/toys/lissajous.gf
-miniapps/toys/mondrian.mesh
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/ex1
+miniapps/nurbs/ex1p
+miniapps/nurbs/ex11p
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*

-miniapps/gslib/field-diff
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
-
-# VPATH builds
-build-*/*
@@ -1,114 +1,28 @@
-language: cpp
-
 sudo: false

-stages:
-  - checks
-  - tests
-  - optional
+language: cpp

-jobs:
+matrix:
  include:
-
-    # ========================
-    #         Checks
-    # ========================
-    # - code-style
-    # - documentation
-    # - gitignore
-
-    - stage: checks
-      os: linux
-      name: "code-style"
-      addons:
-        apt:
-          packages:
-            - astyle=2.05.1-0ubuntu1
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest code-style
-
-    - stage: checks
-      os: linux
-      name: "documentation"
-      addons:
-        apt:
-          packages:
-            - doxygen
-            - graphviz
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest documentation
-
-    - stage: checks
-      os: linux
-      name: "gitignore"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
-        - make all -j3
-        - make test-noclean
-        - cd tests/scripts
-        - ./runtest gitignore
-
-    # ========================
-    #  Optional Checks/Tests
-    # ========================
-    # - branch-history
-
-    - stage: optional
-      name: "branch-history"
-      # need full git history for the binary/big files check
-      git:
-        depth: false
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        # update master
-        - git fetch origin master:master
-        # checkout a branch (otherwise Travis works in detached head)
-        - git checkout -b travis_tests
-        - cd tests/scripts
-        - ./runtest branch-history
-
-    # ========================
-    #       Linux tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - stage: tests
-      os: linux
+    #
+    # Linux
+    #
+    - os: linux
      compiler: gcc
-      name: "Linux: Serial + Debug"
      env: DEBUG=YES
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-
+    #
    - os: linux
      compiler: gcc
-      name: "Linux: Serial"
      env: DEBUG=NO
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-
+    #
    - os: linux
      compiler: gcc
-      name: "Linux: Parallel + Debug"
      addons:
        apt:
          # sources:
@@ -135,10 +49,9 @@ jobs:
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
-
+    #
    - os: linux
      compiler: gcc
-      name: "Linux: Parallel"
      addons:
        apt:
          # sources:
@@ -165,37 +78,28 @@ jobs:
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
-
-    # ========================
-    #      Mac OS X tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
+    #
+    # Mac OS X
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Serial + Debug"
      env: DEBUG=YES
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Serial"
      env: DEBUG=NO
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Parallel + Debug"
      env: DEBUG=YES
           MPI=YES
           CODECOV=NO
@@ -211,11 +115,10 @@ jobs:
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
-
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Parallel"
      env: DEBUG=NO
           MPI=YES
           CODECOV=YES
@@ -8,196 +8,22 @@
                               http://mfem.org


-Version 4.0.1 (development)
-===========================
+Version 4.0-RC2, Apr 24, 2019
+=============================

-Improved GPU support
--------------------
- Added support for matrix-free diagonal smoothers on GPUs.
-
- Added initial support for AMD GPUs based on HIP: a C++ runtime API and kernel
-  language that can run on both AMD and NVIDIA hardware. With this change and
-  the libCEED addition below, the current list of available backends is:
-  "ceed-cuda", "occa-cuda", "raja-cuda", "cuda", "hip", "occa-omp", "raja-omp",
-  "omp", "ceed-cpu", "occa-cpu", "raja-cpu", and "cpu".
-
- Improved RAJA backend and multi-GPU MPI communications.
-
-libCEED support
---------------
- Added support for libCEED, the portable library for high-order operator
-  evaluation developed by the Center for Efficient Exascale Discretizations in
-  the Exascale Computing Project, https://github.com/CEED/libCEED.
-
- This initial integration includes Mass and Diffusion integrators. libCEED GPU
-  backends can be used without specific MFEM configuration, however it is highly
-  recommended to use the "cuda" build option to minimize memory transfers.
-
- Both CPU and GPU modes are available as MFEM device backends (ceed-cpu and
-  ceed-cuda), using some of the best performing CPU and GPU backends from
-  libCEED, see the sample runs in examples 1 and 6.
-
-Meshing improvements
--------------------
- Added support for non-conforming AMR on prisms and tetrahedra, including
-  coarsening and parallel load balancing. Anisotropic prism refinement is only
-  available in the serial version at the moment.
-
- The TMOP mesh optimization algorithms were extended to support r-adaptivity.
-  Target matrices can now be constructed either via a given analytical function
-  (e.g. spatial dependence of size, aspect ratio, etc., for each element) or via
-  a (Par)GridFunction specified on the original mesh.
-
- New method Mesh::GetHilbertElementOrdering for sorting mesh elements along the
-  Hilbert curve. The ordering can be used to improve caching and parallel
-  partitioning in non-conforming AMR.
-
- Added support for creating refined versions of periodic meshes, making use of
-  the new L2ElementRestriction class. This class also allows for computing
-  geometric factors on periodic meshes using partial assembly.
-
- The TMOP mesh optimization algorithms have been improved to support AMR meshes.
-
- Improved element numbering after uniform mesh refinement.
-
-Discretization improvements
---------------------------
- Added support for GSLIB-FindPoints, a general high-order interpolation utility
-  that can robustly evaluate a GridFunction in an arbitrary collection of points
-  in physical space. See INSTALL for details on building MFEM with GSLIB, and
-  miniapps/gslib for examples of how to use this feature.
-
- Added support for serendipity elements of arbitrary order on affinely-mapped
-  square elements. Basis functions for these elements can be visualized using
-  an option in the display-basis miniapp.
-
- Added support for complex-valued finite element operators and fields using a
-  2x2 block structured linear system to mimic complex arithmetic. New classes
-  include: ComplexGridFunction, SesquilinearForm, ComplexLinearForm, and their
-  parallel counterparts.
-
- Two integrators related to Stokes problems, (Q grad u, v) and (Q div v, u),
-  where u and the components of v are in H1, were added/modified to support full
-  and partial assembly modes. See the new GradientIntegrator and the updated
-  VectorDivergenceIntegrator classes in fem/bilininteg.hpp, as well as the PA
-  kernels in fem/bilininteg_gradient.cpp and fem/bilininteg_divergence.cpp.
-
- Diagonals of partially assembled operators can now be computed efficiently.
-  See the new methods AssembleDiagonal in BilinearForm, AssembleDiagonalPA in
-  BilinearFormIntegrator and the implementations in fem/bilininteg_*.cpp.
-
- Added second order derivatives of NURBS shape functions.
-
- Added initial support for NonlinearForms to support the partial assembly mode.
-
- Added a nonlinear vector valued convection integrator (Q u \cdot grad u, v)
-  where u_i and v_i are in H1. This form occurs e.g. in the Navier-Stokes
-  equations. The integrator supports the partial assembly mode for its
-  action. In full assembly mode we also provide the GetGradient method that
-  computes the linearized version of the integrator.
-
- Added a new method, MixedBilinearForm::FormRectangularLinearSystem, that can
-  be used to impose boundary conditions on the non-square off-diagonal blocks of
-  a block operator (similar to FormLinearSystem in the square case).
-
- Extended the support for partial assembly to vector mass and vector diffusion
-  bilinear integrators.
-
-Linear and nonlinear solvers
+Requirements and Limitations
 ----------------------------
- Added a general interface for specifying and solving nonlinear constrained
-  optimization problems through the new classes OptimizationProblem and
-  OptimizationSolver, see linalg/solver.hpp
+- This is a release candidate for mfem-4.0.
+- Use at your own risk -- not everything will work and the API may change.
+- We are looking for feedback from friendly users.
+- Unlike previous MFEM releases, this version requires a C++11 compiler.

- Added support for HiOp, a lightweight HPC solver for nonlinear optimization
-  problems see class HiOpNLPOptimizer and the example codes in examples/hiop.
-
- Added support for Ginkgo, a high-performance linear algebra library for GPU
-  and manycore nodes, with a focus on sparse solution of linear systems. For
-  more details see linalg/ginkgo.hpp and the example code in examples/gingko.
-
- Added Adams-Bashforth and Adams-Moulton time integrators.
-
- Added a block ILU(0) preconditioner for DG-type discretizations. Example 9
-  (DG advection) now takes advantage of this for implicit time integration.
-
- Added a LinearSolve(A,X) convenience method to solve dense linear systems. In
-  the trivial cases, i.e., square matrices of size 1 or 2, the system is solved
-  directly, otherwise, LU factorization is employed.
-
-New and updated examples and miniapps
-------------------------------------
- Added two new miniapps: Find Points (serial + parallel) and Field Diff in
-  miniapps/gslib that show how GSLIB-FindPoints can be used to interpolate a
-  (Par)GridFunction in an arbitrary number of physical space points in 2D and
-  3D. The GridFunction must be in H1 and in the same space as the mesh that is
-  used to find the points.
-
- Added a new example, Example 22/22p, to demonstrate the use of the new
-  complex-valued finite element operators. The example defines and solves
-  a family of time-harmonic PDEs related to damped harmonic oscillators.
-
- Updated Example 1/1p to use diagonal preconditioning in partial assembly mode.
-
- The mesh-optimizer and pmesh-optimizer miniapps have been updated to
-  demonstrate the new r-adaptivity capabilities of TMOP.
-
- New options to reorder and partition the mesh in the mesh-explorer miniapp.
-
- The mesh-explorer miniapp now supports visualization of boundary attributes of
-  3D meshes (key 'b').
-
- The (p)mesh-optimizer miniapp has been updated to demonstrate mesh
-  optimization for an AMR mesh.
-
- Added a modification of Example 1 in examples/ginkgo that demonstrates the use
-  of the Gingko interface to solve a linear system.
-
- Added a modification of ex9 in examples/hiop that demonstrates the nonlinear
-  constrained optimization interface and the use of the SLBQP and HiOp solvers.
-
- Added a collection of 7 playful miniapps in miniapps/toys that illustrate the
-  meshing and visualization features of the library in more relaxed settings.
-  The toys include simulations of cellular automata, Rubik's cube, Mandelbrot
-  set, a tool to convert any image to mfem mesh, and more.
-
- Example 9 and 9p now support implicit time integration, using the new block
-  ILU(0) solvers as preconditioners for the linear system.
-
-Improved testing
----------------
- Added a new directory, tests/scripts, with several shell scripts that perform
-  simple checks on the code including: code styling, documentation formatting,
-  proper use of .gitignore, and preventing the accidental commit of large files.
-
- It is recommended that developers run the above tests scripts (via the runtest
-  script) before pushing to GitHub. See the README file in tests/scripts.
-
- The Travis CI settings have been updated to include an initial Checks stage
-  which currently runs the code-style, documentation and gitignore test scripts,
-  as well as a final stage for optional checks/tests which currently runs the
-  branch-history script.
-
-Miscellaneous
-------------
- Upgraded the SUNDIALS interface to utilize version 5.0. This necessitated a
-  complete rework of the interface and requires changes at the application
-  level. Example usage of the new interface can be found in examples/sundials.
-
- Added support for output in the ParaView XML format. See Examples 5/5p, 9/9p
-  and the new ParaViewDataCollection class.
-
- Collected object files from the miniapps/common directory into a new library,
-  libmfem-common for the convenience of application developers. The new library
-  is now used in several miniapps in the electromagnetic and tools directories.
-
- Added unit tests for time integrators.
-
-
-Version 4.0, released on May 24, 2019
-=====================================
-
-Unlike previous MFEM releases, this version requires a C++11 compiler.
+- GPU-related limitations:
+  * Hypre preconditioners are not yet available in GPU mode.
+  * Only constant coefficients are currently supported on GPUs.
+  * Full-assembly (on device), element assembly, and matrix-free bilinear forms
+    are not supported yet. Element batching is currently ignored.
+  * Partial assembly kernels are not implemented yet for simplices.

 GPU support
 -----------
@@ -208,7 +34,7 @@ GPU support
  seamlessly with a new lightweight device/host memory manager. The kernels can
  be implemented either in OCCA, or as a simple wrapper around for-loops, which
  can then be dispatched to RAJA and native backends. See the files forall.hpp
-  and mem_manager.hpp in the general/ directory for more details.
+  and mem_manager.hpp in the general/ directory.

 - Several of the MFEM example codes (ex1, ex1p, ex6, and ex6p) can now take
  advantage of GPU acceleration with the backend selectable at runtime. Many of
@@ -216,43 +42,26 @@ GPU support
  bilinear forms) have been extended to take advantage of kernel acceleration by
  simply replacing loops with the MFEM_FORALL() macro.

- In addition to native CUDA kernels, the library currently supports OCCA, RAJA
-  and OpenMP kernels, which could be mixed and matched in different parts of the
-  same application. We plan on adding support for more programming models and
-  devices in the future, without the need for significant modifications in user
-  code. The list of current backends is: "occa-cuda", "raja-cuda", "cuda",
-  "occa-omp", "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".
-
- GPU-related limitations:
-  * Hypre preconditioners are not yet available in GPU mode, and in particular
-    hypre must be built in CPU mode.
-  * Only constant coefficients are currently supported on GPUs.
-  * Optimized element assembly, and matrix-free bilinear forms are not
-    implemented yet. Element batching is currently ignored.
-  * In device mode, full assembly is performed on the host (but the matvec
-    action is performed on the device).
-  * Partial assembly kernels are not implemented yet for simplices.
+- In addition to pure CUDA, the library currently supports OCCA, RAJA and OpenMP
+  kernels, which could be mixed and matched in different parts of the same
+  application. We plan on adding support for more programming models and devices
+  in the future, without the need for significant modifications in user code.
+  The list of current backends is: "occa-cuda", "raja-cuda", "cuda", "occa-omp",
+  "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".

 Discretization improvements
 ---------------------------
- Partial assembled finite element operators are now available in the core
-  library, based on the new classes PABilinearFormExtension, ElementRestriction,
-  DofToQuad and GeometricFactors (associated with the classes BilinearForm,
-  FiniteElementSpace, FiniteElement and Mesh, respectively). The kernels for
-  partial assembled Setup/Assembly and Action/Mult are implemented in the
-  BilinearFormIntegrator methods AssemblePA and AddMultPA.
-
 - Added support for a general "low-order refined"-to-"high-order" transfer of
  GridFunction data from a "low-order refined" (LOR) space defined on a refined
  mesh to a "high-order" (HO) finite element space defined on a coarse mesh. See
  the new classes InterpolationGridTransfer and L2ProjectionGridTransfer and the
  new LOR Transfer miniapp: miniapps/tools/lor-transfer.cpp.

+- Added support for derefinement of vector (RT + ND) spaces.
+
 - Added element flux, and flux energy computation in class ElasticityIntegrator,
  allowing for the use of Zienkiewicz-Zhu type error estimators with the
-  integrator. For an illustration of this addition, see the new Example 21.
-
- Added support for derefinement of vector (RT + ND) spaces.
+  integrator. For an illustration of this addition, see the new Example 22.

 - Added a variety of coefficients which are sums or products of existing
  coefficients as well as grid function coefficients which return the
@@ -264,13 +73,13 @@ Support for wedge elements and meshes with mixed element types
  type PRISM) which have two triangular faces and three quadrilateral faces.
  Several examples of such meshes can be found in the data/ directory.

+- Added H1 and L2 finite elements of arbitrary order for Wedge elements.
+
 - Added support for mixed meshes containing triangles and quadrilaterals in 2D
  or tetrahedra, wedges, and hexahedra in 3D. This includes support for uniform
  refinement of such meshes. Several examples of such meshes can be found in the
  data/ directory.

- Added H1 and L2 finite elements of arbitrary order for Wedge elements.
-
 - Added support for reading and writing linear and quadratic meshes containing
  wedge elements in VTK mesh format. Several examples of such meshes can be
  found in the data/ directory.
@@ -291,10 +100,6 @@ Other meshing improvements
  This guarantees that the shape regularity of the elements will be preserved
  under refinement.

- The TMOP mesh optimization algorithms were extended to support user-defined
-  space-dependent limiting terms. Improved the TMOP objective functions by more
-  accurate normalization of the different terms.
-
 - Added support for parallel communication groups on non-conforming meshes.

 - Improved parallel partitioning of non-conforming meshes. If the coarse mesh
@@ -308,6 +113,10 @@ Other meshing improvements
 - Added support for reading linear and quadratic 2D quadrilateral and triangular
  Cubit meshes.

+- The TMOP mesh optimization algorithms were extended to support user-defined
+  space-dependent limiting terms. Improved the TMOP objective functions by more
+  accurate normalization of the different terms.
+
 New and updated examples and miniapps
 -------------------------------------
 - Added a new meshing miniapp, Toroid, which can produce a variety of torus
@@ -323,7 +132,7 @@ New and updated examples and miniapps
  from a Hamiltonian. The example demonstrates the use of the variable order,
  symplectic integration algorithm implemented in class SIAVSolver.

- Added a new example, Example 21/21p, that illustrates the use of AMR to solve
+- Added a new example, Example 22/22p, that illustrates the use of AMR to solve
  a linear elasticity problem. This is an extension of Example 2/2p.

 New and improved solvers and preconditioners
@@ -335,24 +144,21 @@ New and improved solvers and preconditioners

 Miscellaneous
 -------------
- Added unit tests based on the Catch++ library in the test/ directory.
+- In SparseMatrix added the option to perform MultTranspose() by matvec with
+  computed and stored transpose matrix. This is required for deterministic
+  results when using devices such as CUDA and OpenMP.
+
+- Added unit tests based on the Catch++ library.

 - Renamed the option MFEM_USE_OPENMP to MFEM_USE_LEGACY_OPENMP. This legacy
  option is deprecated and planned for removal in a future release. The original
  option name, MFEM_USE_OPENMP, is now used to enable the new OpenMP backends in
  the new kernels.

- In SparseMatrix added the option to perform MultTranspose() by matvec with
-  computed and stored transpose matrix. This is required for deterministic
-  results when using devices such as CUDA and OpenMP.
-
 - Altered the way FGMRES counts its iterations so that it matches GMRES.

 - Various other simplifications, extensions, and bugfixes in the code.

- Construct abstract parallel rectangular truedof-to-truedof operators via
-  Operator::FormDiscreteOperator().
-
 API changes
 -----------
 - In multiple places, use Geometry::Type instead of int, where appropriate.
@@ -50,7 +50,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.0.1)
+set(${PROJECT_NAME}_VERSION 3.4.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -164,15 +164,6 @@ if (MFEM_USE_METIS)
  find_package(METIS REQUIRED)
 endif()

-if (MFEM_USE_GINKGO)
-  find_package(Ginkgo REQUIRED)
-  if (Ginkgo_FOUND)
-    get_target_property(Ginkgo_INCLUDE_DIRS
-      Ginkgo::ginkgo INTERFACE_INCLUDE_DIRECTORIES)
-    set(Ginkgo_LIBRARIES Ginkgo::ginkgo)
-  endif()
-endif()
-
 # GZSTREAM -> zlib
 if (MFEM_USE_GZSTREAM)
  find_package(ZLIB REQUIRED)
@@ -257,10 +248,6 @@ if (MFEM_USE_MPFR)
  find_package(MPFR REQUIRED)
 endif()

-if (MFEM_USE_CEED)
-  find_package(libCEED REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -286,12 +273,6 @@ if (MFEM_USE_PUMI)
  endif()
 endif()

-# HiOp optimizer
-if (MFEM_USE_HIOP)
-  find_package(HIOP REQUIRED)
-  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
-endif()
-
 # CUDA
 if (MFEM_USE_CUDA)
   set(CMAKE_CUDA_STANDARD 11)
@@ -305,6 +286,7 @@ if (MFEM_USE_CUDA)
      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
+   set(MFEM_USE_MM YES CACHE BOOL "Enable MFEM's memory manager" FORCE)
 endif()

 # OCCA
@@ -341,8 +323,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO Ginkgo GNUTLS NETCDF MPFR
-    PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
+    POSIXCLOCKS MFEMBacktrace ZLIB OCCA RAJA)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -414,11 +396,11 @@ endif()
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")

-# If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
-# inside the build directory.
+# If building out-of-source, define MFEM_BUILD_DIR to point to the build
+# directory.
 if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  target_compile_definitions(mfem PRIVATE
-    "MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
+    "MFEM_BUILD_DIR=${PROJECT_BINARY_DIR}")
 endif()

 # Generate configuration file in the build directory: config/_config.hpp.
@@ -434,7 +416,7 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
      "Writing substitute header --> \"${Header}\"")
    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
 "// Auto-generated file.
-#define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
+#define MFEM_BUILD_DIR ${PROJECT_BINARY_DIR}
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
    # This version will be installed in the top include directory:
@@ -554,13 +536,6 @@ if (MFEM_USE_OCCA)
    FILES_MATCHING PATTERN "*.okl")
 endif()

-# Install the libCEED files
-if (MFEM_USE_CEED)
-  install(DIRECTORY ${MFEM_SOURCE_DIRS}
-    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/libceed/*.h")
-endif()
-
 # Install ${HEADERS}
 # ---
 # foreach (HDR ${HEADERS})
@@ -83,8 +83,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  │   └── web
  │       └── examples
  ├── examples
-  │   ├── ginkgo
-  │   ├── hiop
  │   ├── petsc
  │   ├── pumi
  │   └── sundials
@@ -95,7 +93,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  ├── miniapps
  │   ├── common
  │   ├── electromagnetics
-  │   ├── gslib
  │   ├── meshing
  │   ├── nurbs
  │   ├── performance
@@ -353,7 +350,6 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add the example code to the `ALL_EXE_SRCS` variable.
      - [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
   - [ ] List the new example in `doc/CodeDocumentation.dox`.
-   - [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -368,7 +364,6 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
      - [ ] Consider adding a new test for the new miniapp.
   - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
     - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -28,16 +28,13 @@ The METIS dependency can be disabled but that is not generally recommended, see
 the option MFEM_USE_METIS.

 MFEM also includes support for devices such as GPUs, and programming models such
-as CUDA, HIP, OCCA, OpenMP and RAJA.
+as CUDA, OCCA, OpenMP and RAJA.

 - Starting with version 4.0, MFEM requires a C++11 compiler

 - CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
  https://developer.nvidia.com/cuda-toolkit

- HIP support requires an AMD GPU and an installation of the ROCm software stack
-  https://rocm.github.io/ROCmInstall.html#installing-from-amd-rocm-repositories
-
 - OCCA support requires the OCCA library
  https://libocca.org

@@ -78,10 +75,6 @@ CUDA build:
   make cuda -j 4
   (build for a specific compute capability: 'make cuda -j 4 CUDA_ARCH=sm_30')

-HIP build:
-   make hip -j 4
-   (build for a specific AMD GPU chip: 'make hip -j 4 HIP_ARCH=gfx900')
-
 Example codes (serial/parallel, depending on the build):
   cd examples
   make -j 4
@@ -168,18 +161,14 @@ Note that re-configuration is only needed to change the currently configured
 options. Several shortcut targets combining (re-)configuration and compilation
 are also defined:

-   make serial    -> Builds serial optimized version of the library
-   make parallel  -> Builds parallel optimized version of the library
-   make debug     -> Builds serial debug version of the library
-   make pdebug    -> Builds parallel debug version of the library
-   make cuda      -> Builds serial cuda optimized version of the library
-   make pcuda     -> Builds parallel cuda optimized version of the library
-   make cudebug   -> Builds serial cuda debug version of the library
-   make pcudebug  -> Builds parallel cuda debug version of the library
-   make hip       -> Builds serial hip optimized version of the library
-   make phip      -> Builds parallel hip optimized version of the library
-   make hipdebug  -> Builds serial hip debug version of the library
-   make phipdebug -> Builds parallel hip debug version of the library
+   make serial   -> Builds serial optimized version of the library
+   make parallel -> Builds parallel optimized version of the library
+   make debug    -> Builds serial debug version of the library
+   make pdebug   -> Builds parallel debug version of the library
+   make cuda     -> Builds serial cuda optimized version of the library
+   make pcuda    -> Builds parallel cuda optimized version of the library
+   make cudebug  -> Builds serial cuda debug version of the library
+   make pcudebug -> Builds parallel cuda debug version of the library

 Note that any of the above shortcuts accept configuration options, either at the
 command line or through a user configuration file.
@@ -349,12 +338,6 @@ MFEM_USE_STRUMPACK = YES/NO
   classes. When enabled, this option uses the STRUMPACK_* library options, see
   below.

-MFEM_USE_GINKGO = YES/NO
-   Enable MFEM functionality based on the Ginkgo library, which provides
-   iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
-   https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
-   Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.
-
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -389,11 +372,11 @@ MFEM_USE_MPFR = YES/NO
   see below.

 MFEM_USE_SIDRE = YES/NO
-   Sidre is a component of LLNL's axom project, https://github.com/LLNL/axom,
-   that provides an HDF5-based file format for visualization or restart
-   capability following the Conduit (https://github.com/LLNL/conduit) mesh
-   blueprint specification. When enabled, this option requires installation of
-   HDF5 (see also MFEM_USE_NETCDF), Conduit and LLNL's axom project.
+   Sidre is a component of LLNL's axom project, http://goo.gl/cZyJdn, that
+   provides an HDF5-based file format for visualization or restart capability
+   following the Conduit (https://github.com/LLNL/conduit) mesh blueprint
+   specification. When enabled, this option requires installation of HDF5 (see
+   also MFEM_USE_NETCDF), Conduit and LLNL's axom project.

 MFEM_USE_CONDUIT = YES/NO
   Enables support for converting MFEM Mesh and Grid Function objects to and
@@ -421,23 +404,18 @@ MFEM_USE_PUMI = YES/NO
   models and effectively supports automated adaptive analysis. PUMI enables
   support for parallel unstructured mesh modifications in MFEM.

-MFEM_USE_HIOP = YES/NO
-  Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
-  HPC solver for nonlinear optimization problems.
+MFEM_USE_MM = YES/NO
+   Enables support for the MFEM's memory manager (MM), which is required to
+   support devices with different memory spaces. This option is required when
+   CUDA support is enabled, i.e. when MFEM_USE_CUDA=YES.

 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
   platform and programming model for general computing on graphical processing
-   units (GPUs). The variable CUDA_ARCH is used to specify the CUDA compute
-   capability used during compilation (by default, CUDA_ARCH=sm_60). When
-   enabled, this option uses the CUDA_* build options, see below.
-
-MFEM_USE_HIP = YES/NO
-   Enables support for AMD devices in MFEM. HIP is a heterogeneous-compute
-   interface for portability developed by AMD that can target both AMD and
-   NVIDIA GPUs. The variable HIP_ARCH is used to specify the AMD GPU processor
-   used during compilation (by default, HIP_ARCH=gfx900).  When enabled, this
-   option uses the HIP_* build options, see below.
+   units (GPUs). This option requires MFEM_USE_MM. The variable CUDA_ARCH is
+   used to specify the CUDA compute capability used during compilation (by
+   default, CUDA_ARCH=sm_60). When enabled, this option uses the CUDA_* build
+   options, see below.

 MFEM_USE_RAJA = YES/NO
   Enable support for the RAJA performance portability layer in MFEM. RAJA
@@ -452,17 +430,6 @@ MFEM_USE_OCCA = YES/NO
   backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
   in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.

-MFEM_USE_GSLIB = YES/NO
-   Enables MFEM functionality based on the GSLIB library, and specifically its
-   FindPoints component, which provides a robust algorithms to evaluate finite
-   element functions in a collection of points in physical space. When enabled,
-   the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.
-
-MFEM_USE_CEED = YES/NO
-   Enables support for the libCEED library in MFEM. libCEED is a portable
-   library for performant high-order operator evaluation developed by the Center
-   for Efficient Exascale Discretizations in the Exascale Computing Project.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -509,7 +476,6 @@ The specific libraries and their options are:
 - SUNDIALS (optional), used when MFEM_USE_SUNDIALS = YES.
  Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
  Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
-  Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
  If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
  both MPI and hypre.
  URL: http://computation.llnl.gov/projects/sundials/sundials-software
@@ -539,12 +505,6 @@ The specific libraries and their options are:
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.

- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-11 standard. For additional requirements
-  and dependencies of specific modules see the Ginkgo webpage below.
-  URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT (Not used), GINKGO_LIB.
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -570,55 +530,30 @@ The specific libraries and their options are:
  Options: PETSC_OPT, PETSC_LIB.

 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
-  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
-  URL: https://github.com/LLNL/axom
+  URL: http://goo.gl/cZyJdn (axom, to be released)
       https://github.com/LLNL/conduit (Conduit)
       https://support.hdfgroup.org/HDF5 (HDF5)
  Options: SIDRE_OPT, SIDRE_LIB.

- Conduit (optional), used when MFEM_USE_CONDUIT = YES. Conduit Mesh Blueprint
+- Conduit, used when MFEM_USE_CONDUIT = YES. Direct Conduit Mesh Blueprint
  support requires Conduit >= v0.3.1 and VisIt >= v2.13.1 to read the output.
  URL: https://github.com/LLNL/conduit (Conduit)
       https://support.hdfgroup.org/HDF5 (HDF5)
  Options: CONDUIT_OPT, CONDUIT_LIB.

- PUMI (optional), used when MFEM_USE_PUMI = YES.
+- PUMI, used when MFEM_USE_PUMI = YES.
  URL: https://scorec.rpi.edu/pumi
  Options: PUMI_OPT, PUMI_LIB.

- HiOp (optional), used when MFEM_USE_HIOP = YES.
-  URL: https://github.com/LLNL/hiop
-  Options: HIOP_OPT, HIOP_LIB.
-
- GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.5 gslib".
-  Build gslib in parallel or in serial based on the desired MFEM build: "make
-  clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
-  MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
-  Options: GSLIB_OPT, GSLIB_LIB.
-
- CUDA (optional), used when MFEM_USE_CUDA = YES.
+- CUDA, used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.

- HIP (optional), used when MFEM_USE_HIP = YES.
-  URL: https://rocm.github.io/ROCmInstall.html
-  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
-
- OCCA (optional), used when MFEM_USE_OCCA = YES.
+- OCCA, used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.

- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED's master
-  branch, specifically, git-hash c00ee0d or later.
-  URL: https://github.com/CEED/libCEED
-       https://ceed.exascaleproject.org/libceed
-  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-
- RAJA (optional), used when MFEM_USE_RAJA = YES.
-  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
+- RAJA, used when MFEM_USE_RAJA = YES.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.

@@ -637,6 +572,7 @@ The specific libraries and their options are:
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -752,18 +688,15 @@ MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
-MFEM_USE_GINKGO
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
 MFEM_USE_GZSTREAM
 MFEM_USE_PUMI
-MFEM_USE_HIOP
 MFEM_USE_CUDA
 MFEM_USE_OCCA
-MFEM_USE_CEED
 MFEM_USE_RAJA
-MFEM_USE_SIDRE
+MFEM_USE_MM

 The following options are CMake specific:

@@ -804,17 +737,14 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - SuiteSparse
 - SuperLUDist, STRUMPACK
 - ParMETIS
- - Ginkgo
 - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
 - NETCDF
 - MPFR
 - LIBUNWIND
 - POSIXCLOCKS
 - PUMI
- - HIOP
 - OCCA
 - RAJA
- - AXOM - Used when MFEM_USE_SIDRE is enabled

 The following built-in CMake packages are also used:

@@ -232,10 +232,6 @@ IF (DEFINED TPL_ENABLE_GECKO)
  SET(MFEM_USE_GECKO ${TPL_ENABLE_GECKO} CACHE BOOL "Enable GECKO usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_GINKGO)
-  SET(MFEM_USE_GINKGO ${TPL_ENABLE_GINKGO} CACHE BOOL "Enable GINKGO usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_GNUTLS)
  SET(MFEM_USE_GNUTLS ${TPL_ENABLE_GNUTLS} CACHE BOOL "Enable GNUTLS usage" FORCE)
 ENDIF()
@@ -34,7 +34,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GECKO @MFEM_USE_GECKO@)
-set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
@@ -42,10 +41,10 @@ set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
 set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
+set(MFEM_USE_MM @MFEM_USE_MM@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
 set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
 set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
-set(MFEM_USE_CEED @MFEM_USE_CEED@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -92,9 +92,6 @@
 // Enable functionality based on the Gecko library
 #cmakedefine MFEM_USE_GECKO

-// Enable functionality based on the Ginkgo library
-#cmakedefine MFEM_USE_GINKGO
-
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

@@ -113,9 +110,6 @@
 // Enable MFEM functionality based on the PUMI library
 #cmakedefine MFEM_USE_PUMI

-// Enable MFEM functionality based on the HiOp library
-#cmakedefine MFEM_USE_HIOP
-
 // Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
@@ -126,8 +120,8 @@
 // Enable MFEM functionality based on the OCCA library
 #cmakedefine MFEM_USE_OCCA

-// Enable MFEM functionality based on the libCEED library
-#cmakedefine MFEM_USE_CEED
+// Enable MFEM's internal Memory Manager (needed e.g. for MFEM_USE_CUDA)
+#cmakedefine MFEM_USE_MM

 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
@@ -1,36 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Sets the following variables:
-#   - HIOP_FOUND
-#   - HIOP_INCLUDE_DIRS
-#   - HIOP_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(HIOP HIOP HIOP_DIR
-  "include" "hiopInterface.hpp"
-  "lib" "hiop"
-  "Paths to headers required by HIOP."
-  "Libraries required by HIOP.")
-
-# this test fails with parallel MFEM since mpi.h is not available (cxx compiler is used for some reason)
-#  CHECK_BUILD HIOP_VERSION_OK TRUE
-#"
-##include <hiopInterface.hpp>
-#using namespace hiop;
-#int main(int argc, char *argv[])
-#{
-#   MPI_Init(&argc, &argv);
-#   MPI_Comm comm = MPI_COMM_WORLD;
-#
-#   return 0;
-#}
-#")
@@ -1,19 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Defines the following variables:
-#   - CEED_FOUND
-#   - CEED_LIBRARIES
-#   - CEED_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(libCEED CEED CEED_DIR "include" ceed.h "lib" ceed
-  "Paths to headers required by libCEED." "Libraries required by libCEED.")
@@ -720,7 +720,8 @@ function(mfem_export_mk_files)
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GECKO MFEM_USE_GNUTLS
      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA)
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_MM MFEM_USE_CUDA MFEM_USE_OCCA
+      MFEM_USE_RAJA)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -814,7 +815,7 @@ function(mfem_export_mk_files)
  foreach(lib ${TPL_LIBRARIES})
    get_filename_component(suffix ${lib} EXT)
    # handle interfaces (e.g., SCOREC::apf)
-    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
+    if ("${lib}" MATCHES "SCOREC::.*")
    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
      message(FATAL_ERROR "***** interface lib found ... exiting *****")
      # handle static and shared libs
@@ -10,15 +10,18 @@
 // Software Foundation) version 2.1 dated February 1999.


-// Support out-of-source builds: if MFEM_CONFIG_FILE is defined, include it.
+// Support out-of-source builds: if MFEM_BUILD_DIR is defined, load the config
+// file MFEM_BUILD_DIR/config/_config.hpp.
 //
 // Otherwise, use the local file: _config.hpp.

 #ifndef MFEM_CONFIG_HPP
 #define MFEM_CONFIG_HPP

-#ifdef MFEM_CONFIG_FILE
-#include MFEM_CONFIG_FILE
+#ifdef MFEM_BUILD_DIR
+#define MFEM_QUOTE(a) #a
+#define MFEM_MAKE_PATH(x,y) MFEM_QUOTE(x/y)
+#include MFEM_MAKE_PATH(MFEM_BUILD_DIR,config/_config.hpp)
 #else
 #include "_config.hpp"
 #endif
@@ -53,4 +56,9 @@
 #endif
 #endif // MFEM_USE_MPI not defined

+// CUDA requires the memory manager
+#if defined(MFEM_USE_CUDA) && !defined(MFEM_USE_MM)
+#error Building with CUDA (MFEM_USE_CUDA=YES) requires MFEM_USE_MM=YES
+#endif
+
 #endif // MFEM_CONFIG_HPP
@@ -100,9 +100,6 @@
 // Enable functionality based on the Gecko library
 // #define MFEM_USE_GECKO

-// Enable MFEM features based on the Ginkgo library
-// #define MFEM_USE_GINKGO
-
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

@@ -124,28 +121,18 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI

-// Enable MFEM functionality based on the HIOP library.
-// #define MFEM_USE_HIOP
-
-// Enable MFEM functionality based on the GSLIB library
-// #define MFEM_USE_GSLIB
-
-// Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
+// Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA

-// Build the AMD GPU/HIP-enabled version of the MFEM library.
-// Requires a HIP compiler (hipcc).
-// #define MFEM_USE_HIP
-
 // Enable functionality based on the RAJA library.
 // #define MFEM_USE_RAJA

 // Enable functionality based on the OCCA library.
 // #define MFEM_USE_OCCA

-// Enable functionality based on the libCEED library.
-// #define MFEM_USE_CEED
+// Enable MFEM's internal Memory Manager (needed e.g. for MFEM_USE_CUDA)
+// #define MFEM_USE_MM

 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@
@@ -34,7 +34,6 @@ MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GECKO         = @MFEM_USE_GECKO@
-MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -42,13 +41,10 @@ MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
-MFEM_USE_HIOP          = @MFEM_USE_HIOP@
-MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
-MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
-MFEM_USE_CEED          = @MFEM_USE_CEED@
+MFEM_USE_MM            = @MFEM_USE_MM@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -35,7 +35,6 @@ option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GECKO "Enable GECKO usage" OFF)
-option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
@@ -43,11 +42,10 @@ option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
-option(MFEM_USE_HIOP "Enable HiOp" OFF)
+option(MFEM_USE_MM "Enable MFEM's memory manager" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
-option(MFEM_USE_CEED "Enable CEED" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -84,7 +82,7 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
+set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-3.0.0" CACHE PATH
    "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
@@ -139,8 +137,6 @@ set(ScaLAPACK_TARGET_NAMES scalapack)

 set(GECKO_DIR "${MFEM_DIR}/../gecko" CACHE PATH "Path to the Gecko library.")

-set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
@@ -159,20 +155,14 @@ set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH

 set(AXOM_DIR "${MFEM_DIR}/../axom" CACHE PATH "Path to the Axom library.")
 # May need to add "Boost" as requirement.
-set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
+set(Axom_REQUIRED_PACKAGES "Conduit/relay" CACHE STRING
    "Additional packages required by Axom.")

 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
    "Directory where PUMI is installed")

-set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
-    "Directory where HiOp is installed")
-set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Packages that HiOp depends on.")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
@@ -46,14 +46,6 @@ CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
 CUDA_XCOMPILER = -Xcompiler=
 CUDA_XLINKER   = -Xlinker=

-# HIP configuration options
-HIP_CXX = hipcc
-# The HIP_ARCH option specifies the AMD GPU processor, similar to CUDA_ARCH. For
-# example: gfx600 (tahiti), gfx700 (kaveri), gfx701 (hawaii), gfx801 (carrizo),
-# gfx900, gfx1010, etc.
-HIP_ARCH = gfx900
-HIP_FLAGS = --amdgpu-target=$(HIP_ARCH)
-
 ifneq ($(NOTMAC),)
   AR      = ar
   ARFLAGS = cruv
@@ -122,7 +114,6 @@ MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GECKO         = NO
-MFEM_USE_GINKGO        = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -130,13 +121,10 @@ MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
-MFEM_USE_HIOP          = NO
-MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
-MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
-MFEM_USE_CEED          = NO
+MFEM_USE_MM            = NO

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -186,9 +174,9 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-3.0.0
 SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
+SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib -L$(SUNDIALS_DIR)/lib\
 -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
@@ -213,7 +201,7 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
 # SuperLU library configuration
 SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
 SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
-SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0
+SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/SRC -L$(SUPERLU_DIR)/SRC -lsuperlu_dist

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -248,11 +236,6 @@ GECKO_DIR = @MFEM_DIR@/../gecko
 GECKO_OPT = -I$(GECKO_DIR)/inc
 GECKO_LIB = -L$(GECKO_DIR)/lib -lgecko

-# Ginkgo library configuration (currently not needed)
-GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference
-
 # GnuTLS library configuration
 GNUTLS_OPT =
 GNUTLS_LIB = -lgnutls
@@ -308,7 +291,7 @@ SIDRE_LIB = \
   -Wl,-rpath,$(SIDRE_DIR)/lib -L$(SIDRE_DIR)/lib \
   -Wl,-rpath,$(CONDUIT_DIR)/lib -L$(CONDUIT_DIR)/lib \
   -Wl,-rpath,$(HDF5_DIR)/lib -L$(HDF5_DIR)/lib \
-   -laxom -lconduit -lconduit_relay -lconduit_blueprint -lhdf5 $(ZLIB_LIB) -ldl
+   -laxom -lconduit -lconduit_relay -lhdf5 $(ZLIB_LIB) -ldl

 # PUMI
 # Note that PUMI_DIR is needed -- it is used to check for gmi_sim.h
@@ -317,34 +300,15 @@ PUMI_OPT = -I$(PUMI_DIR)/include
 PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
   -llion -lmth -lapf_zoltan -lspr

-# HIOP
-HIOP_DIR = @MFEM_DIR@/../hiop/install
-HIOP_OPT = -I$(HIOP_DIR)/include
-HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)
-
-# GSLIB library
-GSLIB_DIR = @MFEM_DIR@/../gslib/build
-GSLIB_OPT = -I$(GSLIB_DIR)/include
-GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
-
 # CUDA library configuration (currently not needed)
 CUDA_OPT =
 CUDA_LIB =

-# HIP library configuration (currently not needed)
-HIP_OPT =
-HIP_LIB =
-
 # OCCA library configuration
 OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca

-# libCEED library configuration
-CEED_DIR ?= @MFEM_DIR@/../libCEED
-CEED_OPT = -I$(CEED_DIR)/include
-CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed
-
 # RAJA library configuration
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
@@ -36,7 +36,6 @@ CONFIG_MK = config.mk
 all: header config-mk

 MPI = $(MFEM_USE_MPI:NO=)
-GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
 GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
@@ -45,7 +44,7 @@ SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))

 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
-	$(GHV_CXX) ${GHV_FLAGS} $(SRC)$(GHV).cpp -o $(GHV)
+	$(MFEM_CXX) ${GHV_FLAGS} $(SRC)$(GHV).cpp -o $(GHV)
 $(GHV).out: $(GHV)
 	./$(GHV) > $(GHV).out
 .INTERMEDIATE: $(GHV) $(GHV).out
@@ -152,15 +152,6 @@ function extract_sample_runs()
   fi
   if [ "$device_runs" == "yes" ]; then
      runs=`printf "%s" "$runs" | grep ".* -d .*"`
-      if [ "$have_occa" == "no" ]; then
-         runs=`printf "%s" "$runs" | grep -v ".* -d occa-.*"`
-      fi
-      if [ "$have_raja" == "no" ]; then
-         runs=`printf "%s" "$runs" | grep -v ".* -d raja-.*"`
-      fi
-      if [ "$have_ceed" == "no" ]; then
-         runs=`printf "%s" "$runs" | grep -v ".* -d ceed-.*"`
-      fi
   else
      runs=`printf "%s" "$runs" | grep -v ".* -d .*"`
   fi
@@ -285,8 +276,8 @@ case "$1" in
      ;;
   -dev)
       device_runs="yes"
-       mfem_config+=" MFEM_USE_CUDA=YES MFEM_USE_OPENMP=YES"
-       # OCCA, RAJA, libCEED are enabled below, if available
+       mfem_config+=" MFEM_USE_CUDA=YES MFEM_USE_MM=YES \
+MFEM_USE_OCCA=YES MFEM_USE_RAJA=YES MFEM_USE_OPENMP=YES"
      ;;
   -v)
      valgrind="yes"
@@ -467,30 +458,6 @@ fi

 TIMEFORMAT="${base_timeformat}"

-# Setup optional libraries when not using externally built MFEM:
-if [ "${built}" == "no" ]; then
-   have_occa="no"
-   have_raja="no"
-   have_ceed="no"
-   if [ "${device_runs}" == "yes" ]; then
-      if [ -n "${CUDA_ARCH}" ]; then
-         mfem_config+=" CUDA_ARCH=${CUDA_ARCH}"
-      fi
-      if [ -d "${mfem_dir}/../occa" ]; then
-         mfem_config+=" MFEM_USE_OCCA=YES"
-         have_occa="yes"
-      fi
-      if [ -d "${mfem_dir}/../raja" ]; then
-         mfem_config+=" MFEM_USE_RAJA=YES"
-         have_raja="yes"
-      fi
-      if [ -d "${mfem_dir}/../libCEED" ]; then
-         mfem_config+=" MFEM_USE_CEED=YES"
-         have_ceed="yes"
-      fi
-   fi
-fi
-
 function set_echo_log()
 {
   local dirname=`dirname "$1"`
@@ -43,14 +43,15 @@
 #define MFEM_ALIGN_SIZE(size,type) \
   MFEM_ROUNDUP(size,(MFEM_SIMD_SIZE)/sizeof(type))

-#ifdef MFEM_COUNT_FLOPS
 namespace mfem
 {
 namespace internal
 {
-extern long long flop_count;
+long long flop_count;
 }
 }
+
+#ifdef MFEM_COUNT_FLOPS
 #define MFEM_FLOPS_RESET() (mfem::internal::flop_count = 0)
 #define MFEM_FLOPS_ADD(cnt) (mfem::internal::flop_count += (cnt))
 #define MFEM_FLOPS_GET() (mfem::internal::flop_count)
@@ -82,7 +82,6 @@ test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
-test-noclean: all test-par-$(MFEM_USE_MPI)
 test-clean: ; @rm -f *.stderr
 test-print: mfem-test=printf "   $(3) [$(2) ./$(1) -no-vis $(if $(4),$(4) )]\n"
 test-print: mfem-test-file=printf "   $(3) [$(2) ./$(1) -no-vis ]\n"
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.0.1
+PROJECT_NUMBER         = v3.4.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -768,16 +768,13 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
                         @MFEM_SOURCE_DIR@/miniapps/common \
-                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
-                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
-                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
+                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
+                         @MFEM_SOURCE_DIR@/miniapps/performance

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -37,9 +37,7 @@ namespace mfem {
 *
 * <H3>Main GPU classes</H3>
 * - Device
- * - Memory
 * - MemoryManager
- * - MFEM_FORALL macro in forall.hpp
 *
 * <H3>Example codes</H3>
 * - <a class="el" href="examples_2ex1_8cpp_source.html">Example 1</a>: nodal H1 FEM for the Laplace problem
@@ -79,10 +77,8 @@ namespace mfem {
 * - <a class="el" href="ex19p_8cpp_source.html">Example 19p</a>: parallel incompressible nonlinear elasticity
 * - <a class="el" href="ex20_8cpp_source.html">Example 20</a>: symplectic ODE integration
 * - <a class="el" href="ex20p_8cpp_source.html">Example 20p</a>: parallel symplectic ODE integration
- * - <a class="el" href="ex21_8cpp_source.html">Example 21</a>: adaptive mesh refinement for linear elasticity
- * - <a class="el" href="ex21p_8cpp_source.html">Example 21p</a>: parallel adaptive mesh refinement for linear elasticity
- * - <a class="el" href="ex22_8cpp_source.html">Example 22</a>: complex-valued linear systems for damped harmonic oscillators
- * - <a class="el" href="ex22p_8cpp_source.html">Example 22p</a>: parallel complex-valued linear systems for damped harmonic oscillators
+ * - <a class="el" href="ex22_8cpp_source.html">Example 22</a>: adaptive mesh refinement for linear elasticity
+ * - <a class="el" href="ex22p_8cpp_source.html">Example 22p</a>: parallel adaptive mesh refinement for linear elasticity
 *
 * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
@@ -117,12 +113,6 @@ namespace mfem {
 *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
 *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
 *
- * <H4>HiOp Examples</H4>
- * - Variants of Examples
- *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
- *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
- *   demonstrating the use of MFEM's \link hiop.hpp HiOp classes\endlink
- *
 * <H3>Miniapps</H3>
 * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
@@ -136,12 +126,9 @@ namespace mfem {
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
 * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
- * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
 * - <a class="el" href="load-dc_8cpp_source.html">Load DC</a>: visualize fields saved via DataCollection classes
- * - <a class="el" href="convert-dc_8cpp_source.html">Convert DC</a>: convert between different DataCollection formats
+ * - <a class="el" href="convert-dc_8cpp_source.html">Convert DC</a>: convert between diffirent DataCollection formats
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
- * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -27,7 +27,6 @@ list(APPEND ALL_EXE_SRCS
  ex18.cpp
  ex19.cpp
  ex20.cpp
-  ex21.cpp
  ex22.cpp
  )

@@ -53,7 +52,6 @@ if (MFEM_USE_MPI)
    ex18p.cpp
    ex19p.cpp
    ex20p.cpp
-    ex21p.cpp
    ex22p.cpp
    )
 endif()
@@ -102,11 +100,6 @@ if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
 endif()

-# Include the examples/sundials directory if SUNDIALS is enabled.
-if (MFEM_USE_GINKGO)
-  add_subdirectory(ginkgo)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -116,8 +109,3 @@ endif()
 if (MFEM_USE_PUMI)
  add_subdirectory(pumi)
 endif()
-
-if (MFEM_USE_HIOP)
-  add_subdirectory(hiop)
-endif()
-
@@ -31,8 +31,6 @@
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -64,7 +62,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -77,7 +75,7 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -90,18 +88,13 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 50,000
   //    elements.
@@ -114,7 +107,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
   //    Lagrange finite elements of the specified order. If order < 1, we
   //    instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
@@ -135,7 +128,7 @@ int main(int argc, char *argv[])
   cout << "Number of finite element unknowns: "
        << fespace->GetTrueVSize() << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
@@ -147,7 +140,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
   LinearForm *b = new LinearForm(fespace);
@@ -155,6 +148,12 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(one));
   b->Assemble();

+   // 7. Set device config parameters from the command line options and switch
+   //    to working on the device.
+   Device::Configure(device);
+   Device::Print();
+   Device::Enable();
+
   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero,
   //    which satisfies the boundary conditions.
@@ -196,16 +195,18 @@ int main(int argc, char *argv[])
      umf_solver.Mult(B, X);
 #endif
   }
-   else // Jacobi preconditioning in partial assembly mode
+   else // No preconditioning for now in partial assembly mode.
   {
-      OperatorJacobiSmoother M(*a, ess_tdof_list);
-      PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
   }

   // 12. Recover the solution as a finite element grid function.
   a->RecoverFEMSolution(X, *b, x);

-   // 13. Save the refined mesh and the solution. This output can be viewed later
+   // 13. Switch back to the host.
+   Device::Disable();
+
+   // 14. Save the refined mesh and the solution. This output can be viewed later
   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
@@ -214,7 +215,7 @@ int main(int argc, char *argv[])
   sol_ofs.precision(8);
   x.Save(sol_ofs);

-   // 14. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -224,7 +225,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *mesh << x << flush;
   }

-   // 15. Free the used memory.
+   // 16. Free the used memory.
   delete a;
   delete b;
   delete fespace;
@@ -179,9 +179,7 @@ int main(int argc, char *argv[])
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
                  "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4."
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            13 - RK3 SSP, 14 - RK4.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -193,9 +193,7 @@ int main(int argc, char *argv[])
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
                  "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4."
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            13 - RK3 SSP, 14 - RK4.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -5,11 +5,11 @@
 // Sample runs:
 //    mpirun -np 4 ex12p -m ../data/beam-tri.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh
-//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
-//    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
-//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
-//    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3877 -o 2 -sys
-//    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 79 -n 10 -o 2 -elast
+//    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3876
+//    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 79
+//    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3876 -o 2 -sys
+//    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4526 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-hex-nurbs.mesh
 //
@@ -57,7 +57,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/beam-tri.mesh";
   int order = 1;
   int nev = 5;
-   int seed = 66;
+   int seed = 75;
   bool visualization = 1;
   bool amg_elast = 0;

@@ -3,7 +3,7 @@
 // Compile with: make ex13p
 //
 // Sample runs:  mpirun -np 4 ex13p -m ../data/star.mesh
-//               mpirun -np 4 ex13p -m ../data/square-disc.mesh -o 2 -n 4
+//               mpirun -np 4 ex13p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex13p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex13p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex13p -m ../data/escher.mesh
@@ -16,6 +16,9 @@
 //               ex15 -m ../data/ball-nurbs.mesh -tf 0.3
 //               ex15 -m ../data/mobius-strip.mesh
 //               ex15 -m ../data/amr-quad.mesh
+//
+//               Conforming meshes (no derefinement):
+//
 //               ex15 -m ../data/square-disc.mesh
 //               ex15 -m ../data/escher.mesh -r 2 -tf 0.3
 //
@@ -127,7 +130,7 @@ int main(int argc, char *argv[])
      if (ref_levels > 0) { ref_levels--; }
      mesh.SetCurvature(2);
   }
-   mesh.EnsureNCMesh(true);
+   mesh.EnsureNCMesh();
   for (int l = 0; l < ref_levels; l++)
   {
      mesh.UniformRefinement();
@@ -16,6 +16,9 @@
 //               mpirun -np 4 ex15p -m ../data/ball-nurbs.mesh -tf 0.5
 //               mpirun -np 4 ex15p -m ../data/mobius-strip.mesh
 //               mpirun -np 4 ex15p -m ../data/amr-quad.mesh
+//
+//               Conforming meshes (no load balancing and derefinement):
+//
 //               mpirun -np 4 ex15p -m ../data/square-disc.mesh
 //               mpirun -np 4 ex15p -m ../data/escher.mesh -r 2 -tf 0.3
 //
@@ -143,7 +146,7 @@ int main(int argc, char *argv[])
      if (ref_levels > 0) { ref_levels--; }
      mesh->SetCurvature(2);
   }
-   mesh->EnsureNCMesh(true);
+   mesh->EnsureNCMesh();
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
@@ -252,7 +252,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      GMRES(A, M, B, X, 3, 5000, 100, rtol*rtol, 0.0);
+      GMRES(A, M, B, X, 3, 5000, 50, rtol*rtol, 0.0);
   }
 #else
   // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
@@ -144,7 +144,7 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options
-   const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/beam-hex.mesh";
   int ref_levels = 0;
   int order = 2;
   bool visualization = true;
@@ -150,7 +150,7 @@ int main(int argc, char *argv[])
   MPI_Comm_rank(MPI_COMM_WORLD, &myid);

   // 2. Parse command-line options
-   const char *mesh_file = "../data/beam-tet.mesh";
+   const char *mesh_file = "../data/beam-hex.mesh";
   int ser_ref_levels = 0;
   int par_ref_levels = 0;
   int order = 2;
@@ -29,8 +29,6 @@
 //               mpirun -np 4 ex1p -pa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -67,7 +65,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -80,7 +78,7 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -100,18 +98,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 10,000 elements.
@@ -124,7 +117,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -137,7 +130,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use continuous Lagrange finite elements of the specified order. If
   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
@@ -164,7 +157,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
@@ -176,7 +169,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (1,phi_i) where phi_i are the basis functions in fespace.
   ParLinearForm *b = new ParLinearForm(fespace);
@@ -184,6 +177,12 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new DomainLFIntegrator(one));
   b->Assemble();

+   // 9. Set device config parameters from the command line options and switch
+   //    to working on the device.
+   Device::Configure(device);
+   if (myid == 0) { Device::Print(); }
+   Device::Enable();
+
   // 10. Define the solution vector x as a parallel finite element grid function
   //     corresponding to fespace. Initialize x with initial guess of zero,
   //     which satisfies the boundary conditions.
@@ -210,16 +209,9 @@ int main(int argc, char *argv[])

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
+   //     * With partial assembly, use no preconditioner, for now.
   Solver *prec = NULL;
-   if (pa)
-   {
-      prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
+   if (!pa) { prec = new HypreBoomerAMG; }
   CGSolver cg(MPI_COMM_WORLD);
   cg.SetRelTol(1e-12);
   cg.SetMaxIter(2000);
@@ -233,7 +225,10 @@ int main(int argc, char *argv[])
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 15. Save the refined mesh and the solution in parallel. This output can
+   // 15. Switch back to the host.
+   Device::Disable();
+
+   // 16. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -249,7 +244,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -260,7 +255,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 17. Free the used memory.
+   // 18. Free the used memory.
   delete a;
   delete b;
   delete fespace;
@@ -1,310 +0,0 @@
-//                                MFEM Example 21
-//
-// Compile with: make ex21
-//
-// Sample runs:  ex21
-//               ex21 -o 3
-//               ex21 -m ../data/beam-quad.mesh
-//               ex21 -m ../data/beam-quad.mesh -o 3
-//               ex21 -m ../data/beam-quad.mesh -o 3 -f 1
-//               ex21 -m ../data/beam-tet.mesh
-//               ex21 -m ../data/beam-tet.mesh -o 2
-//               ex21 -m ../data/beam-hex.mesh
-//               ex21 -m ../data/beam-hex.mesh -o 2
-//
-// Description:  This is a version of Example 2 with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the linear
-//               elasticity describing a multi-material cantilever beam.
-//               The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
-//
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear and curved meshes. Interpolation of functions from
-//               coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
-//
-//               We recommend viewing Examples 2 and 6 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-tri.mesh";
-   int order = 1;
-   bool static_cond = false;
-   int flux_averaging = 0;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&flux_averaging, "-f", "--flux-averaging",
-                  "Flux averaging: 0 - global, 1 - by mesh attribute.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
-
-   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
-   {
-      cerr << "\nInput mesh should have at least two materials and "
-           << "two boundary attributes! (See schematic in ex2.cpp)\n"
-           << endl;
-      return 3;
-   }
-
-   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
-   //    convert it to a piecewise-polynomial curved mesh. First we refine the
-   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext)
-   {
-      for (int i = 0; i < 2; i++)
-      {
-         mesh.UniformRefinement();
-      }
-      mesh.SetCurvature(2);
-   }
-
-   // 4. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   FiniteElementSpace fespace(&mesh, &fec, dim);
-
-   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
-   //    the right-hand side of the FEM linear system. In this case, b_i equals
-   //    the boundary integral of f*phi_i where f represents a "pull down"
-   //    force on the Neumann part of the boundary and phi_i are the basis
-   //    functions in the finite element fespace. The force is defined by the
-   //    VectorArrayCoefficient object f, which is a vector of Coefficient
-   //    objects. The fact that f is non-zero on boundary attribute 2 is
-   //    indicated by the use of piece-wise constants coefficient for its last
-   //    component. We don't assemble the discrete problem yet, this will be
-   //    done in the main loop.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-   {
-      Vector pull_force(mesh.bdr_attributes.Max());
-      pull_force = 0.0;
-      pull_force(1) = -1.0e-2;
-      f.Set(dim-1, new PWConstCoefficient(pull_force));
-   }
-
-   LinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
-
-   // 6. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the linear elasticity integrator with piece-wise
-   //    constants coefficient lambda and mu.
-   Vector lambda(mesh.attributes.Max());
-   lambda = 1.0;
-   lambda(0) = lambda(1)*50;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(mesh.attributes.Max());
-   mu = 1.0;
-   mu(0) = mu(1)*50;
-   PWConstCoefficient mu_func(mu);
-
-   BilinearForm a(&fespace);
-   BilinearFormIntegrator *integ =
-      new ElasticityIntegrator(lambda_func,mu_func);
-   a.AddDomainIntegrator(integ);
-   if (static_cond) { a.EnableStaticCondensation(); }
-
-   // 7. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   Vector zero_vec(dim);
-   zero_vec = 0.0;
-   VectorConstantCoefficient zero_vec_coeff(zero_vec);
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking only
-   //    boundary attribute 1 from the mesh as essential and converting it to a
-   //    list of true dofs.  The conversion to true dofs will be done in the
-   //    main loop.
-   Array<int> ess_bdr(mesh.bdr_attributes.Max());
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
-
-   // 9. Connect to GLVis.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-   if (visualization)
-   {
-      sol_sock.open(vishost, visport);
-      sol_sock.precision(8);
-   }
-
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
-   //     recover a smoothed flux (stress) that is subtracted from the element
-   //     flux to get an error indicator. We need to supply the space for the
-   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
-   //     Here, tdim represents the number of components for a symmetric (dim x
-   //     dim) tensor.
-   const int tdim = dim*(dim+1)/2;
-   FiniteElementSpace flux_fespace(&mesh, &fec, tdim);
-   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
-   estimator.SetFluxAveraging(flux_averaging);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   const int max_dofs = 50000;
-   const int max_amr_itr = 20;
-   for (int it = 0; it <= max_amr_itr; it++)
-   {
-      int cdofs = fespace.GetTrueVSize();
-      cout << "\nAMR iteration " << it << endl;
-      cout << "Number of unknowns: " << cdofs << endl;
-
-      // 13. Assemble the stiffness matrix and the right-hand side.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Set Dirichlet boundary values in the GridFunction x.
-      //     Determine the list of Dirichlet true DOFs in the linear system.
-      Array<int> ess_tdof_list;
-      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-      // 15. Create the linear system: eliminate boundary conditions, constrain
-      //     hanging nodes and possibly apply other transformations. The system
-      //     will be solved for true (unconstrained) DOFs only.
-      SparseMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-#ifndef MFEM_USE_SUITESPARSE
-      // 16. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-      //     solve the linear system with PCG.
-      GSSmoother M(A);
-      PCG(A, M, B, X, 3, 2000, 1e-12, 0.0);
-#else
-      // 16. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
-      //     the linear system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(A);
-      umf_solver.Mult(B, X);
-#endif
-
-      // 17. After solving the linear system, reconstruct the solution as a
-      //     finite element GridFunction. Constrained nodes are interpolated
-      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
-      a.RecoverFEMSolution(X, b, x);
-
-      // 18. Send solution by socket to the GLVis server.
-      if (visualization && sol_sock.good())
-      {
-         GridFunction nodes(&fespace), *nodes_p = &nodes;
-         mesh.GetNodes(nodes);
-         nodes += x;
-         int own_nodes = 0;
-         mesh.SwapNodes(nodes_p, own_nodes);
-         x.Neg(); // visualize the backward displacement
-         sol_sock << "solution\n" << mesh << x << flush;
-         x.Neg();
-         mesh.SwapNodes(nodes_p, own_nodes);
-         if (it == 0)
-         {
-            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
-         }
-         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
-                  << "pause" << endl;
-         cout << "Visualization paused. "
-              "Press <space> in the GLVis window to continue." << endl;
-      }
-
-      if (cdofs > max_dofs)
-      {
-         cout << "Reached the maximum number of dofs. Stop." << endl;
-         break;
-      }
-
-      // 19. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(mesh);
-      if (refiner.Stop())
-      {
-         cout << "Stopping criterion satisfied. Stop." << endl;
-         break;
-      }
-
-      // 20. Update the space to reflect the new state of the mesh. Also,
-      //     interpolate the solution x so that it lies in the new space but
-      //     represents the same function. This saves solver iterations later
-      //     since we'll have a good initial guess of x in the next step.
-      //     Internally, FiniteElementSpace::Update() calculates an
-      //     interpolation matrix which is then used by GridFunction::Update().
-      fespace.Update();
-      x.Update();
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
-   }
-
-   {
-      ofstream mesh_ref_out("ex21_reference.mesh");
-      mesh_ref_out.precision(16);
-      mesh.Print(mesh_ref_out);
-
-      ofstream mesh_out("ex21_deformed.mesh");
-      mesh_out.precision(16);
-      GridFunction nodes(&fespace), *nodes_p = &nodes;
-      mesh.GetNodes(nodes);
-      nodes += x;
-      int own_nodes = 0;
-      mesh.SwapNodes(nodes_p, own_nodes);
-      mesh.Print(mesh_out);
-      mesh.SwapNodes(nodes_p, own_nodes);
-
-      ofstream x_out("ex21_displacement.sol");
-      x_out.precision(16);
-      x.Save(x_out);
-   }
-
-   return 0;
-}
@@ -1,366 +0,0 @@
-//                                MFEM Example 21
-//
-// Compile with: make ex21p
-//
-// Sample runs:  mpirun -np 4 ex21p
-//               mpirun -np 4 ex21p -o 3
-//               mpirun -np 4 ex21p -m ../data/beam-quad.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-quad.mesh -o 3
-//               mpirun -np 4 ex21p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-tet.mesh -o 2
-//               mpirun -np 4 ex21p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex21p -m ../data/beam-hex.mesh -o 2
-//
-// Description:  This is a version of Example 2p with a simple adaptive mesh
-//               refinement loop. The problem being solved is again the linear
-//               elasticity describing a multi-material cantilever beam.
-//               The problem is solved on a sequence of meshes which
-//               are locally refined in a conforming (triangles, tetrahedrons)
-//               or non-conforming (quadrilaterals, hexahedra) manner according
-//               to a simple ZZ error estimator.
-//
-//               The example demonstrates MFEM's capability to work with both
-//               conforming and nonconforming refinements, in 2D and 3D, on
-//               linear and curved meshes. Interpolation of functions from
-//               coarse to fine meshes, as well as persistent GLVis
-//               visualization are also illustrated.
-//
-//               We recommend viewing Examples 2p and 6p before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 0. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-tri.mesh";
-   int serial_ref_levels = 0;
-   int order = 1;
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&serial_ref_levels, "-rs", "--refine-serial",
-                  "Number of uniform serial refinements (before parallel"
-                  " partitioning)");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");
-
-   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
-   {
-      cerr << "\nInput mesh should have at least two materials and "
-           << "two boundary attributes! (See schematic in ex2.cpp)\n"
-           << endl;
-      MPI_Finalize();
-      return 3;
-   }
-
-   // 3. Refine the mesh before parallel partitioning. Since a NURBS mesh can
-   //    currently only be refined uniformly, we need to convert it to a
-   //    piecewise-polynomial curved mesh. First we refine the NURBS mesh a bit
-   //    more and then project the curvature to quadratic Nodes.
-   if (mesh.NURBSext && serial_ref_levels == 0)
-   {
-      serial_ref_levels = 2;
-   }
-   for (int i = 0; i < serial_ref_levels; i++)
-   {
-      mesh.UniformRefinement();
-   }
-   if (mesh.NURBSext)
-   {
-      mesh.SetCurvature(2);
-   }
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // 4. Define a finite element space on the mesh. The polynomial order is
-   //    one (linear) by default, but this can be changed on the command line.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec, dim);
-
-   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
-   //    the right-hand side of the FEM linear system. In this case, b_i equals
-   //    the boundary integral of f*phi_i where f represents a "pull down"
-   //    force on the Neumann part of the boundary and phi_i are the basis
-   //    functions in the finite element fespace. The force is defined by the
-   //    VectorArrayCoefficient object f, which is a vector of Coefficient
-   //    objects. The fact that f is non-zero on boundary attribute 2 is
-   //    indicated by the use of piece-wise constants coefficient for its last
-   //    component. We don't assemble the discrete problem yet, this will be
-   //    done in the main loop.
-   VectorArrayCoefficient f(dim);
-   for (int i = 0; i < dim-1; i++)
-   {
-      f.Set(i, new ConstantCoefficient(0.0));
-   }
-   {
-      Vector pull_force(pmesh.bdr_attributes.Max());
-      pull_force = 0.0;
-      pull_force(1) = -1.0e-2;
-      f.Set(dim-1, new PWConstCoefficient(pull_force));
-   }
-
-   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));
-
-   // 6. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the linear elasticity integrator with piece-wise
-   //    constants coefficient lambda and mu.
-   Vector lambda(pmesh.attributes.Max());
-   lambda = 1.0;
-   lambda(0) = lambda(1)*50;
-   PWConstCoefficient lambda_func(lambda);
-   Vector mu(pmesh.attributes.Max());
-   mu = 1.0;
-   mu(0) = mu(1)*50;
-   PWConstCoefficient mu_func(mu);
-
-   ParBilinearForm a(&fespace);
-   BilinearFormIntegrator *integ =
-      new ElasticityIntegrator(lambda_func,mu_func);
-   a.AddDomainIntegrator(integ);
-   if (static_cond) { a.EnableStaticCondensation(); }
-
-   // 7. The solution vector x and the associated finite element grid function
-   //    will be maintained over the AMR iterations. We initialize it to zero.
-   Vector zero_vec(dim);
-   zero_vec = 0.0;
-   VectorConstantCoefficient zero_vec_coeff(zero_vec);
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking only
-   //    boundary attribute 1 from the mesh as essential and converting it to a
-   //    list of true dofs.  The conversion to true dofs will be done in the
-   //    main loop.
-   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
-
-   // 9. GLVis visualization.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sol_sock;
-
-   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
-   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
-   //     recover a smoothed flux (stress) that is subtracted from the element
-   //     flux to get an error indicator. We need to supply the space for the
-   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
-   //     Here, tdim represents the number of components for a symmetric (dim x
-   //     dim) tensor.
-   const int tdim = dim*(dim+1)/2;
-   L2_FECollection flux_fec(order, dim);
-   ParFiniteElementSpace flux_fespace(&pmesh, &flux_fec, tdim);
-   ParFiniteElementSpace smooth_flux_fespace(&pmesh, &fec, tdim);
-   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace,
-                                       smooth_flux_fespace);
-
-   // 11. A refiner selects and refines elements based on a refinement strategy.
-   //     The strategy here is to refine elements with errors larger than a
-   //     fraction of the maximum element error. Other strategies are possible.
-   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
-   refiner.SetTotalErrorFraction(0.7);
-
-   // 12. The main AMR loop. In each iteration we solve the problem on the
-   //     current mesh, visualize the solution, and refine the mesh.
-   const int max_dofs = 50000;
-   const int max_amr_itr = 20;
-   for (int it = 0; it <= max_amr_itr; it++)
-   {
-      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
-      if (myid == 0)
-      {
-         cout << "\nAMR iteration " << it << endl;
-         cout << "Number of unknowns: " << global_dofs << endl;
-      }
-
-      // 13. Assemble the stiffness matrix and the right-hand side.
-      a.Assemble();
-      b.Assemble();
-
-      // 14. Set Dirichlet boundary values in the GridFunction x.
-      //     Determine the list of Dirichlet true DOFs in the linear system.
-      Array<int> ess_tdof_list;
-      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-      // 15. Create the linear system: eliminate boundary conditions, constrain
-      //     hanging nodes and possibly apply other transformations. The system
-      //     will be solved for true (unconstrained) DOFs only.
-
-      HypreParMatrix A;
-      Vector B, X;
-      const int copy_interior = 1;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
-
-      // 16. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-      //     preconditioner from hypre.
-      HypreBoomerAMG amg;
-      amg.SetPrintLevel(0);
-      // amg.SetSystemsOptions(dim); // optional
-      CGSolver pcg(A.GetComm());
-      pcg.SetPreconditioner(amg);
-      pcg.SetOperator(A);
-      pcg.SetRelTol(1e-6);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(3); // print the first and the last iterations only
-      pcg.Mult(B, X);
-
-      // 17. After solving the linear system, reconstruct the solution as a
-      //     finite element GridFunction. Constrained nodes are interpolated
-      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
-      a.RecoverFEMSolution(X, b, x);
-
-      // 18. Send solution by socket to the GLVis server.
-      if (visualization && it == 0)
-      {
-         sol_sock.open(vishost, visport);
-         sol_sock.precision(8);
-      }
-      if (visualization && sol_sock.good())
-      {
-         GridFunction nodes(&fespace), *nodes_p = &nodes;
-         pmesh.GetNodes(nodes);
-         nodes += x;
-         int own_nodes = 0;
-         pmesh.SwapNodes(nodes_p, own_nodes);
-         x.Neg(); // visualize the backward displacement
-         sol_sock << "parallel " << num_procs << ' ' << myid << '\n';
-         sol_sock << "solution\n" << pmesh << x << flush;
-         x.Neg();
-         pmesh.SwapNodes(nodes_p, own_nodes);
-         if (it == 0)
-         {
-            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
-         }
-         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
-                  << "pause" << endl;
-         if (myid == 0)
-         {
-            cout << "Visualization paused. "
-                 "Press <space> in the GLVis window to continue." << endl;
-         }
-      }
-
-      if (global_dofs > max_dofs)
-      {
-         if (myid == 0)
-         {
-            cout << "Reached the maximum number of dofs. Stop." << endl;
-         }
-         break;
-      }
-
-      // 19. Call the refiner to modify the mesh. The refiner calls the error
-      //     estimator to obtain element errors, then it selects elements to be
-      //     refined and finally it modifies the mesh. The Stop() method can be
-      //     used to determine if a stopping criterion was met.
-      refiner.Apply(pmesh);
-      if (refiner.Stop())
-      {
-         if (myid == 0)
-         {
-            cout << "Stopping criterion satisfied. Stop." << endl;
-         }
-         break;
-      }
-
-      // 20. Update the space to reflect the new state of the mesh. Also,
-      //     interpolate the solution x so that it lies in the new space but
-      //     represents the same function. This saves solver iterations later
-      //     since we'll have a good initial guess of x in the next step.
-      //     Internally, FiniteElementSpace::Update() calculates an
-      //     interpolation matrix which is then used by GridFunction::Update().
-      fespace.Update();
-      x.Update();
-
-      // 21. Load balance the mesh, and update the space and solution. Currently
-      //     available only for nonconforming meshes.
-      if (pmesh.Nonconforming())
-      {
-         pmesh.Rebalance();
-
-         // Update the space and the GridFunction. This time the update matrix
-         // redistributes the GridFunction among the processors.
-         fespace.Update();
-         x.Update();
-      }
-
-      // 21. Inform also the bilinear and linear forms that the space has
-      //     changed.
-      a.Update();
-      b.Update();
-   }
-
-   {
-      ostringstream mref_name, mesh_name, sol_name;
-      mref_name << "ex21p_reference_mesh." << setfill('0') << setw(6) << myid;
-      mesh_name << "ex21p_deformed_mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex21p_displacement." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ref_out(mref_name.str().c_str());
-      mesh_ref_out.precision(16);
-      pmesh.Print(mesh_ref_out);
-
-      ofstream mesh_out(mesh_name.str().c_str());
-      mesh_out.precision(16);
-      GridFunction nodes(&fespace), *nodes_p = &nodes;
-      pmesh.GetNodes(nodes);
-      nodes += x;
-      int own_nodes = 0;
-      pmesh.SwapNodes(nodes_p, own_nodes);
-      pmesh.Print(mesh_out);
-      pmesh.SwapNodes(nodes_p, own_nodes);
-
-      ofstream x_out(sol_name.str().c_str());
-      x_out.precision(16);
-      x.Save(x_out);
-   }
-
-   MPI_Finalize();
-   return 0;
-}
@@ -1,44 +1,32 @@
-//                               MFEM Example 22
+//                                MFEM Example 22
 //
 // Compile with: make ex22
 //
-// Sample runs:  ex22 -m ../data/inline-segment.mesh -o 3
-//               ex22 -m ../data/inline-tri.mesh -o 3
-//               ex22 -m ../data/inline-quad.mesh -o 3
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
-//               ex22 -m ../data/inline-tet.mesh -o 2
-//               ex22 -m ../data/inline-hex.mesh -o 2
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
+// Sample runs:  ex22
+//               ex22 -o 3
+//               ex22 -m ../data/beam-quad.mesh
+//               ex22 -m ../data/beam-quad.mesh -o 3
+//               ex22 -m ../data/beam-quad.mesh -o 3 -f 1
+//               ex22 -m ../data/beam-tet.mesh
+//               ex22 -m ../data/beam-tet.mesh -o 2
+//               ex22 -m ../data/beam-hex.mesh
+//               ex22 -m ../data/beam-hex.mesh -o 2
 //
-// Description:  This example code demonstrates the use of MFEM to define and
-//               solve simple complex-valued linear systems. It implements three
-//               variants of a damped harmonic oscillator:
+// Description:  This is a version of Example 2 with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the linear
+//               elasticity describing a multi-material cantilever beam.
+//               The problem is solved on a sequence of meshes which
+//               are locally refined in a conforming (triangles, tetrahedrons)
+//               or non-conforming (quadrilaterals, hexahedra) manner according
+//               to a simple ZZ error estimator.
 //
-//               1) A scalar H1 field
-//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
+//               The example demonstrates MFEM's capability to work with both
+//               conforming and nonconforming refinements, in 2D and 3D, on
+//               linear and curved meshes. Interpolation of functions from
+//               coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
 //
-//               2) A vector H(Curl) field
-//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
-//
-//               3) A vector H(Div) field
-//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
-//
-//               In each case the field is driven by a forced oscillation, with
-//               angular frequency omega, imposed at the boundary or a portion
-//               of the boundary.
-//
-//               In electromagnetics, the coefficients are typically named the
-//               permeability, mu = 1/a, permittivity, epsilon = b, and
-//               conductivity, sigma = c. The user can specify these constants
-//               using either set of names.
-//
-//               The example also demonstrates how to display a time-varying
-//               solution as a sequence of fields sent to a single GLVis socket.
-//
-//               We recommend viewing examples 1, 3 and 4 before viewing this
+//               We recommend viewing Examples 2 and 6 before viewing this
 //               example.

 #include "mfem.hpp"
@@ -48,61 +36,24 @@
 using namespace std;
 using namespace mfem;

-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 20.0;
-static double omega_ = 10.0;
-
-double u0_real_exact(const Vector &);
-double u0_imag_exact(const Vector &);
-
-void u1_real_exact(const Vector &, Vector &);
-void u1_imag_exact(const Vector &, Vector &);
-
-void u2_real_exact(const Vector &, Vector &);
-void u2_imag_exact(const Vector &, Vector &);
-
-bool check_for_inline_mesh(const char * mesh_file);
-
 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
-   const char *mesh_file = "../data/inline-quad.mesh";
-   int ref_levels = 0;
+   const char *mesh_file = "../data/beam-tri.mesh";
   int order = 1;
-   int prob = 0;
-   double freq = -1.0;
-   double a_coef = 0.0;
+   bool static_cond = false;
+   int flux_averaging = 0;
   bool visualization = 1;
-   bool herm_conv = true;
-   bool exact_sol = true;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H_1, 1: H(Curl), or 2: H(Div) "
-                  "damped harmonic oscillator.");
-   args.AddOption(&a_coef, "-a", "--stiffness-coef",
-                  "Stiffness coefficient (spring constant or 1/mu).");
-   args.AddOption(&epsilon_, "-b", "--mass-coef",
-                  "Mass coefficient (or epsilon).");
-   args.AddOption(&sigma_, "-c", "--damping-coef",
-                  "Damping coefficient (or sigma).");
-   args.AddOption(&mu_, "-mu", "--permeability",
-                  "Permeability of free space (or 1/(spring constant)).");
-   args.AddOption(&epsilon_, "-eps", "--permittivity",
-                  "Permittivity of free space (or mass constant).");
-   args.AddOption(&sigma_, "-sigma", "--conductivity",
-                  "Conductivity (or damping constant).");
-   args.AddOption(&freq, "-f", "--frequency",
-                  "Frequency (in Hz).");
-   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
-                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&flux_averaging, "-f", "--flux-averaging",
+                  "Flux averaging: 0 - global, 1 - by mesh attribute.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -114,448 +65,246 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   MFEM_VERIFY(prob >= 0 && prob <=2,
-               "Unrecognized problem type: " << prob);
-
-   if ( a_coef != 0.0 )
-   {
-      mu_ = 1.0 / a_coef;
-   }
-   if ( freq > 0.0 )
-   {
-      omega_ = 2.0 * M_PI * freq;
-   }
-
-   exact_sol = check_for_inline_mesh(mesh_file);
-   if (exact_sol)
-   {
-      cout << "Identified a mesh with known exact solution" << endl;
-   }
-
-   ComplexOperator::Convention conv =
-      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
-
   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
-   //    with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");

-   // 3. Refine the mesh to increase resolution. In this example we do
-   //    'ref_levels' of uniform refinement where the user specifies
-   //    the number of levels with the '-r' option.
-   for (int l = 0; l < ref_levels; l++)
+   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
   {
-      mesh->UniformRefinement();
+      cerr << "\nInput mesh should have at least two materials and "
+           << "two boundary attributes! (See schematic in ex2.cpp)\n"
+           << endl;
+      return 3;
   }

-   // 4. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
-   //    order.
-   if (dim == 1 && prob != 0 )
+   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
+   //    convert it to a piecewise-polynomial curved mesh. First we refine the
+   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
+   if (mesh.NURBSext)
   {
-      cout << "Switching to problem type 0, H1 basis functions, "
-           << "for 1 dimensional mesh." << endl;
-      prob = 0;
-   }
-
-   FiniteElementCollection *fec = NULL;
-   switch (prob)
-   {
-      case 0:  fec = new H1_FECollection(order, dim);      break;
-      case 1:  fec = new ND_FECollection(order, dim);      break;
-      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
-      default: break; // This should be unreachable
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
-        << endl;
-
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined based on the type
-   //    of mesh and the problem type.
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      if (exact_sol)
+      for (int i = 0; i < 2; i++)
      {
-         switch (prob)
-         {
-            case 0:   ess_bdr = 0; ess_bdr[0] = 1;  break;
-            default:  ess_bdr = 1; ess_bdr[2] = 0;  break;
-         }
+         mesh.UniformRefinement();
      }
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      mesh.SetCurvature(2);
   }

-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
-   ComplexLinearForm b(fespace, conv);
-   b.Vector::operator=(0.0);
+   // 4. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec, dim);

-   // 7. Define the solution vector u as a complex finite element grid function
-   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
-   //    the exact solution if it is known.
-   ComplexGridFunction u(fespace);
-   ComplexGridFunction * u_exact = NULL;
-   if (exact_sol) { u_exact = new ComplexGridFunction(fespace); }
-
-   FunctionCoefficient u0_r(u0_real_exact);
-   FunctionCoefficient u0_i(u0_imag_exact);
-   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
-   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
-   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
-   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
-
-   ConstantCoefficient zeroCoef(0.0);
-   ConstantCoefficient oneCoef(1.0);
-
-   Vector zeroVec(dim); zeroVec = 0.0;
-   Vector  oneVec(dim);  oneVec = 0.0; oneVec[(prob==2)?(dim-1):0] = 1.0;
-   VectorConstantCoefficient zeroVecCoef(zeroVec);
-   VectorConstantCoefficient oneVecCoef(oneVec);
-
-   u = 0.0;
-   switch (prob)
+   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
+   //    the right-hand side of the FEM linear system. In this case, b_i equals
+   //    the boundary integral of f*phi_i where f represents a "pull down"
+   //    force on the Neumann part of the boundary and phi_i are the basis
+   //    functions in the finite element fespace. The force is defined by the
+   //    VectorArrayCoefficient object f, which is a vector of Coefficient
+   //    objects. The fact that f is non-zero on boundary attribute 2 is
+   //    indicated by the use of piece-wise constants coefficient for its last
+   //    component. We don't assemble the discrete problem yet, this will be
+   //    done in the main loop.
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
   {
-      case 0:
-         u.ProjectBdrCoefficient(oneCoef, zeroCoef, ess_bdr);
-         if (exact_sol) { u_exact->ProjectCoefficient(u0_r, u0_i); }
-         break;
-      case 1:
-         u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
-         if (exact_sol) { u_exact->ProjectCoefficient(u1_r, u1_i); }
-         break;
-      case 2:
-         u.ProjectBdrCoefficientNormal(oneVecCoef, zeroVecCoef, ess_bdr);
-         if (exact_sol) { u_exact->ProjectCoefficient(u2_r, u2_i); }
-         break;
-      default: break; // This should be unreachable
+      f.Set(i, new ConstantCoefficient(0.0));
   }
-
-   if (visualization && exact_sol)
   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *mesh << u_exact->real()
-                 << "window_title 'Exact: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *mesh << u_exact->imag()
-                 << "window_title 'Exact: Imaginary Part'" << flush;
+      Vector pull_force(mesh.bdr_attributes.Max());
+      pull_force = 0.0;
+      pull_force(1) = -1.0e-2;
+      f.Set(dim-1, new PWConstCoefficient(pull_force));
   }

-   // 8. Set up the sesquilinear form a(.,.) on the finite element space
-   //    corresponding to the damped harmonic oscillator operator of the
-   //    appropriate type:
-   //
-   //    0) A scalar H1 field
-   //       -Div(a Grad) - omega^2 b + i omega c
-   //
-   //    1) A vector H(Curl) field
-   //       Curl(a Curl) - omega^2 b + i omega c
-   //
-   //    2) A vector H(Div) field
-   //       -Grad(a Div) - omega^2 b + i omega c
-   //
-   ConstantCoefficient stiffnessCoef(1.0/mu_);
-   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
-   ConstantCoefficient lossCoef(omega_ * sigma_);
-   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
+   LinearForm b(&fespace);
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));

-   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
-   switch (prob)
-   {
-      case 0:
-         a->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new MassIntegrator(massCoef),
-                                new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         a->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
+   // 6. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the linear elasticity integrator with piece-wise
+   //    constants coefficient lambda and mu.
+   Vector lambda(mesh.attributes.Max());
+   lambda = 1.0;
+   lambda(0) = lambda(1)*50;
+   PWConstCoefficient lambda_func(lambda);
+   Vector mu(mesh.attributes.Max());
+   mu = 1.0;
+   mu(0) = mu(1)*50;
+   PWConstCoefficient mu_func(mu);

-   // 8a. Set up the bilinear form for the preconditioner corresponding to the
-   //     appropriate operator
-   //
-   //      0) A scalar H1 field
-   //         -Div(a Grad) - omega^2 b + omega c
-   //
-   //      1) A vector H(Curl) field
-   //         Curl(a Curl) + omega^2 b + omega c
-   //
-   //      2) A vector H(Div) field
-   //         -Grad(a Div) - omega^2 b + omega c
-   //
-   BilinearForm *pcOp = new BilinearForm(fespace);
-   switch (prob)
-   {
-      case 0:
-         pcOp->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         pcOp->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         pcOp->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
+   BilinearForm a(&fespace);
+   BilinearFormIntegrator *integ =
+      new ElasticityIntegrator(lambda_func,mu_func);
+   a.AddDomainIntegrator(integ);
+   if (static_cond) { a.EnableStaticCondensation(); }

-   // 9. Assemble the form and the corresponding linear system, applying any
-   //    necessary transformations such as: assembly, eliminating boundary
-   //    conditions, conforming constraints for non-conforming AMR, etc.
-   a->Assemble();
-   pcOp->Assemble();
+   // 7. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
+   Vector zero_vec(dim);
+   zero_vec = 0.0;
+   VectorConstantCoefficient zero_vec_coeff(zero_vec);
+   GridFunction x(&fespace);
+   x = 0.0;

-   OperatorHandle A;
-   Vector B, U;
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking only
+   //    boundary attribute 1 from the mesh as essential and converting it to a
+   //    list of true dofs.  The conversion to true dofs will be done in the
+   //    main loop.
+   Array<int> ess_bdr(mesh.bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;

-   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
-   u = 0.0;
-   U = 0.0;
-
-   OperatorHandle PCOp;
-   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-
-   {
-      ComplexSparseMatrix * Asp =
-         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
-
-      cout << "Size of linear system: "
-           << 2 * Asp->real().Width() << endl << endl;
-   }
-
-   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
-   //     preconditioner based on the appropriate sparse smoother.
-   {
-      Array<int> blockOffsets;
-      blockOffsets.SetSize(3);
-      blockOffsets[0] = 0;
-      blockOffsets[1] = PCOp.Ptr()->Height();
-      blockOffsets[2] = PCOp.Ptr()->Height();
-      blockOffsets.PartialSum();
-
-      BlockDiagonalPreconditioner BDP(blockOffsets);
-
-      Operator * pc_r = NULL;
-      Operator * pc_i = NULL;
-
-      switch (prob)
-      {
-         case 0: // fallthrough to case 2
-         case 2:
-            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-            break;
-         case 1:
-            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-            break;
-         default: break; // This should be unreachable
-      }
-      pc_i = new ScaledOperator(pc_r,
-                                (conv == ComplexOperator::HERMITIAN) ?
-                                1.0:-1.0);
-
-      BDP.SetDiagonalBlock(0, pc_r);
-      BDP.SetDiagonalBlock(1, pc_i);
-      BDP.owns_blocks = 1;
-
-      GMRESSolver gmres;
-      gmres.SetPreconditioner(BDP);
-      gmres.SetOperator(*A.Ptr());
-      gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(1000);
-      gmres.SetPrintLevel(1);
-      gmres.Mult(B, U);
-   }
-
-   // 11. Recover the solution as a finite element grid function and compute the
-   //     errors if the exact solution is known.
-   a->RecoverFEMSolution(U, b, u);
-
-   if (exact_sol)
-   {
-      double err_r = -1.0;
-      double err_i = -1.0;
-
-      switch (prob)
-      {
-         case 0:
-            err_r = u.real().ComputeL2Error(u0_r);
-            err_i = u.imag().ComputeL2Error(u0_i);
-            break;
-         case 1:
-            err_r = u.real().ComputeL2Error(u1_r);
-            err_i = u.imag().ComputeL2Error(u1_i);
-            break;
-         case 2:
-            err_r = u.real().ComputeL2Error(u2_r);
-            err_i = u.imag().ComputeL2Error(u2_i);
-            break;
-         default: break; // This should be unreachable
-      }
-
-      cout << endl;
-      cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
-      cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
-      cout << endl;
-   }
-
-   // 12. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m mesh -g sol".
-   {
-      ofstream mesh_ofs("refined.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-
-      ofstream sol_r_ofs("sol_r.gf");
-      ofstream sol_i_ofs("sol_i.gf");
-      sol_r_ofs.precision(8);
-      sol_i_ofs.precision(8);
-      u.real().Save(sol_r_ofs);
-      u.imag().Save(sol_i_ofs);
-   }
-
-   // 13. Send the solution by socket to a GLVis server.
+   // 9. Connect to GLVis.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
   if (visualization)
   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *mesh << u.real()
-                 << "window_title 'Solution: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *mesh << u.imag()
-                 << "window_title 'Solution: Imaginary Part'" << flush;
-   }
-   if (visualization && exact_sol)
-   {
-      *u_exact -= u;
-
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *mesh << u_exact->real()
-                 << "window_title 'Error: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *mesh << u_exact->imag()
-                 << "window_title 'Error: Imaginary Part'" << flush;
-   }
-   if (visualization)
-   {
-      GridFunction u_t(fespace);
-      u_t = u.real();
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
+      sol_sock.open(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << u_t
-               << "window_title 'Harmonic Solution (t = 0.0 T)'"
-               << "pause\n" << flush;
-
-      cout << "GLVis visualization paused."
-           << " Press space (in the GLVis window) to resume it.\n";
-      int num_frames = 32;
-      int i = 0;
-      while (sol_sock)
-      {
-         double t = (double)(i % num_frames) / num_frames;
-         ostringstream oss;
-         oss << "Harmonic Solution (t = " << t << " T)";
-
-         add(cos( 2.0 * M_PI * t), u.real(),
-             sin(-2.0 * M_PI * t), u.imag(), u_t);
-         sol_sock << "solution\n" << *mesh << u_t
-                  << "window_title '" << oss.str() << "'" << flush;
-         i++;
-      }
   }

-   // 14. Free the used memory.
-   delete a;
-   delete u_exact;
-   delete pcOp;
-   delete fespace;
-   delete fec;
-   delete mesh;
+   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
+   //     recover a smoothed flux (stress) that is subtracted from the element
+   //     flux to get an error indicator. We need to supply the space for the
+   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
+   //     Here, tdim represents the number of components for a symmetric (dim x
+   //     dim) tensor.
+   const int tdim = dim*(dim+1)/2;
+   FiniteElementSpace flux_fespace(&mesh, &fec, tdim);
+   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
+   estimator.SetFluxAveraging(flux_averaging);
+
+   // 11. A refiner selects and refines elements based on a refinement strategy.
+   //     The strategy here is to refine elements with errors larger than a
+   //     fraction of the maximum element error. Other strategies are possible.
+   //     The refiner will call the given error estimator.
+   ThresholdRefiner refiner(estimator);
+   refiner.SetTotalErrorFraction(0.7);
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 50000;
+   const int max_amr_itr = 20;
+   for (int it = 0; it <= max_amr_itr; it++)
+   {
+      int cdofs = fespace.GetTrueVSize();
+      cout << "\nAMR iteration " << it << endl;
+      cout << "Number of unknowns: " << cdofs << endl;
+
+      // 13. Assemble the stiffness matrix and the right-hand side.
+      a.Assemble();
+      b.Assemble();
+
+      // 14. Set Dirichlet boundary values in the GridFunction x.
+      //     Determine the list of Dirichlet true DOFs in the linear system.
+      Array<int> ess_tdof_list;
+      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      // 15. Create the linear system: eliminate boundary conditions, constrain
+      //     hanging nodes and possibly apply other transformations. The system
+      //     will be solved for true (unconstrained) DOFs only.
+      SparseMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+
+#ifndef MFEM_USE_SUITESPARSE
+      // 16. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+      //     solve the linear system with PCG.
+      GSSmoother M(A);
+      PCG(A, M, B, X, 3, 2000, 1e-12, 0.0);
+#else
+      // 16. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
+      //     the linear system.
+      UMFPackSolver umf_solver;
+      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      umf_solver.SetOperator(A);
+      umf_solver.Mult(B, X);
+#endif
+
+      // 17. After solving the linear system, reconstruct the solution as a
+      //     finite element GridFunction. Constrained nodes are interpolated
+      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
+      a.RecoverFEMSolution(X, b, x);
+
+      // 18. Send solution by socket to the GLVis server.
+      if (visualization && sol_sock.good())
+      {
+         GridFunction nodes(&fespace), *nodes_p = &nodes;
+         mesh.GetNodes(nodes);
+         nodes += x;
+         int own_nodes = 0;
+         mesh.SwapNodes(nodes_p, own_nodes);
+         x.Neg(); // visualize the backward displacement
+         sol_sock << "solution\n" << mesh << x << flush;
+         x.Neg();
+         mesh.SwapNodes(nodes_p, own_nodes);
+         if (it == 0)
+         {
+            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
+         }
+         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
+                  << "pause" << endl;
+         cout << "Visualization paused. "
+              "Press <space> in the GLVis window to continue." << endl;
+      }
+
+      if (cdofs > max_dofs)
+      {
+         cout << "Reached the maximum number of dofs. Stop." << endl;
+         break;
+      }
+
+      // 19. Call the refiner to modify the mesh. The refiner calls the error
+      //     estimator to obtain element errors, then it selects elements to be
+      //     refined and finally it modifies the mesh. The Stop() method can be
+      //     used to determine if a stopping criterion was met.
+      refiner.Apply(mesh);
+      if (refiner.Stop())
+      {
+         cout << "Stopping criterion satisfied. Stop." << endl;
+         break;
+      }
+
+      // 20. Update the space to reflect the new state of the mesh. Also,
+      //     interpolate the solution x so that it lies in the new space but
+      //     represents the same function. This saves solver iterations later
+      //     since we'll have a good initial guess of x in the next step.
+      //     Internally, FiniteElementSpace::Update() calculates an
+      //     interpolation matrix which is then used by GridFunction::Update().
+      fespace.Update();
+      x.Update();
+
+      // 21. Inform also the bilinear and linear forms that the space has
+      //     changed.
+      a.Update();
+      b.Update();
+   }
+
+   {
+      ofstream mesh_ref_out("ex22_reference.mesh");
+      mesh_ref_out.precision(16);
+      mesh.Print(mesh_ref_out);
+
+      ofstream mesh_out("ex22_deformed.mesh");
+      mesh_out.precision(16);
+      GridFunction nodes(&fespace), *nodes_p = &nodes;
+      mesh.GetNodes(nodes);
+      nodes += x;
+      int own_nodes = 0;
+      mesh.SwapNodes(nodes_p, own_nodes);
+      mesh.Print(mesh_out);
+      mesh.SwapNodes(nodes_p, own_nodes);
+
+      ofstream x_out("ex22_displacement.sol");
+      x_out.precision(16);
+      x.Save(x_out);
+   }

   return 0;
 }
-
-bool check_for_inline_mesh(const char * mesh_file)
-{
-   string file(mesh_file);
-   size_t p0 = file.find_last_of("/");
-   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
-   return s0 == "inline-";
-}
-
-complex<double> u0_exact(const Vector &x)
-{
-   int dim = x.Size();
-   complex<double> i(0.0, 1.0);
-   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
-   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
-   return std::exp(-i * kappa * x[dim - 1]);
-}
-
-double u0_real_exact(const Vector &x)
-{
-   return u0_exact(x).real();
-}
-
-double u0_imag_exact(const Vector &x)
-{
-   return u0_exact(x).imag();
-}
-
-void u1_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
-}
-
-void u1_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
-}
-
-void u2_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
-}
-
-void u2_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
-}
@@ -1,47 +1,33 @@
-//                       MFEM Example 22 - Parallel Version
+//                                MFEM Example 22
 //
 // Compile with: make ex22p
 //
-// Sample runs:  mpirun -np 4 ex22p -m ../data/inline-segment.mesh -o 3
-//               mpirun -np 4 ex22p -m ../data/inline-tri.mesh -o 3
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
+// Sample runs:  mpirun -np 4 ex22p
+//               mpirun -np 4 ex22p -o 3
+//               mpirun -np 4 ex22p -m ../data/beam-quad.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-quad.mesh -o 3
+//               mpirun -np 4 ex22p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-tet.mesh -o 2
+//               mpirun -np 4 ex22p -m ../data/beam-hex.mesh
+//               mpirun -np 4 ex22p -m ../data/beam-hex.mesh -o 2
 //
-// Description:  This example code demonstrates the use of MFEM to define and
-//               solve simple complex-valued linear systems. It implements three
-//               variants of a damped harmonic oscillator:
+// Description:  This is a version of Example 2p with a simple adaptive mesh
+//               refinement loop. The problem being solved is again the linear
+//               elasticity describing a multi-material cantilever beam.
+//               The problem is solved on a sequence of meshes which
+//               are locally refined in a conforming (triangles, tetrahedrons)
+//               or non-conforming (quadrilaterals, hexahedra) manner according
+//               to a simple ZZ error estimator.
 //
-//               1) A scalar H1 field
-//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
+//               The example demonstrates MFEM's capability to work with both
+//               conforming and nonconforming refinements, in 2D and 3D, on
+//               linear and curved meshes. Interpolation of functions from
+//               coarse to fine meshes, as well as persistent GLVis
+//               visualization are also illustrated.
 //
-//               2) A vector H(Curl) field
-//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
-//
-//               3) A vector H(Div) field
-//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
-//
-//               In each case the field is driven by a forced oscillation, with
-//               angular frequency omega, imposed at the boundary or a portion
-//               of the boundary.
-//
-//               In electromagnetics the coefficients are typically named the
-//               permeability, mu = 1/a, permittivity, epsilon = b, and
-//               conductivity, sigma = c. The user can specify these constants
-//               using either set of names.
-//
-//               The example also demonstrates how to display a time-varying
-//               solution as a sequence of fields sent to a single GLVis socket.
-//
-//               We recommend viewing examples 1, 3 and 4 before viewing this
+//               We recommend viewing Examples 2p and 6p before viewing this
 //               example.

-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -49,70 +35,31 @@
 using namespace std;
 using namespace mfem;

-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 20.0;
-static double omega_ = 10.0;
-
-double u0_real_exact(const Vector &);
-double u0_imag_exact(const Vector &);
-
-void u1_real_exact(const Vector &, Vector &);
-void u1_imag_exact(const Vector &, Vector &);
-
-void u2_real_exact(const Vector &, Vector &);
-void u2_imag_exact(const Vector &, Vector &);
-
-bool check_for_inline_mesh(const char * mesh_file);
-
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI.
+   // 0. Initialize MPI.
   int num_procs, myid;
   MPI_Init(&argc, &argv);
   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
   MPI_Comm_rank(MPI_COMM_WORLD, &myid);

-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/inline-quad.mesh";
-   int ser_ref_levels = 1;
-   int par_ref_levels = 1;
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/beam-tri.mesh";
+   int serial_ref_levels = 0;
   int order = 1;
-   int prob = 0;
-   double freq = -1.0;
-   double a_coef = 0.0;
+   bool static_cond = false;
   bool visualization = 1;
-   bool herm_conv = true;
-   bool exact_sol = true;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&serial_ref_levels, "-rs", "--refine-serial",
+                  "Number of uniform serial refinements (before parallel"
+                  " partitioning)");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H_1, 1: H(Curl), or 2: H(Div) "
-                  "damped harmonic oscillator.");
-   args.AddOption(&a_coef, "-a", "--stiffness-coef",
-                  "Stiffness coefficient (spring constant or 1/mu).");
-   args.AddOption(&epsilon_, "-b", "--mass-coef",
-                  "Mass coefficient (or epsilon).");
-   args.AddOption(&sigma_, "-c", "--damping-coef",
-                  "Damping coefficient (or sigma).");
-   args.AddOption(&mu_, "-mu", "--permeability",
-                  "Permeability of free space (or 1/(spring constant)).");
-   args.AddOption(&epsilon_, "-eps", "--permittivity",
-                  "Permittivity of free space (or mass constant).");
-   args.AddOption(&sigma_, "-sigma", "--conductivity",
-                  "Conductivity (or damping constant).");
-   args.AddOption(&freq, "-f", "--frequency",
-                  "Frequency (in Hz).");
-   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
-                  "--no-hermitian", "Use convention for Hermitian operators.");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -131,492 +78,289 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   MFEM_VERIFY(prob >= 0 && prob <=2,
-               "Unrecognized problem type: " << prob);
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, and hexahedral meshes with the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+   MFEM_VERIFY(mesh.SpaceDimension() == dim, "invalid mesh");

-   if ( a_coef != 0.0 )
+   if (mesh.attributes.Max() < 2 || mesh.bdr_attributes.Max() < 2)
   {
-      mu_ = 1.0 / a_coef;
-   }
-   if ( freq > 0.0 )
-   {
-      omega_ = 2.0 * M_PI * freq;
+      cerr << "\nInput mesh should have at least two materials and "
+           << "two boundary attributes! (See schematic in ex2.cpp)\n"
+           << endl;
+      MPI_Finalize();
+      return 3;
   }

-   exact_sol = check_for_inline_mesh(mesh_file);
-   if (myid == 0 && exact_sol)
+   // 3. Refine the mesh before parallel partitioning. Since a NURBS mesh can
+   //    currently only be refined uniformly, we need to convert it to a
+   //    piecewise-polynomial curved mesh. First we refine the NURBS mesh a bit
+   //    more and then project the curvature to quadratic Nodes.
+   if (mesh.NURBSext && serial_ref_levels == 0)
   {
-      cout << "Identified a mesh with known exact solution" << endl;
+      serial_ref_levels = 2;
+   }
+   for (int i = 0; i < serial_ref_levels; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   if (mesh.NURBSext)
+   {
+      mesh.SetCurvature(2);
+   }
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define a finite element space on the mesh. The polynomial order is
+   //    one (linear) by default, but this can be changed on the command line.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec, dim);
+
+   // 5. As in Example 2, we set up the linear form b(.) which corresponds to
+   //    the right-hand side of the FEM linear system. In this case, b_i equals
+   //    the boundary integral of f*phi_i where f represents a "pull down"
+   //    force on the Neumann part of the boundary and phi_i are the basis
+   //    functions in the finite element fespace. The force is defined by the
+   //    VectorArrayCoefficient object f, which is a vector of Coefficient
+   //    objects. The fact that f is non-zero on boundary attribute 2 is
+   //    indicated by the use of piece-wise constants coefficient for its last
+   //    component. We don't assemble the discrete problem yet, this will be
+   //    done in the main loop.
+   VectorArrayCoefficient f(dim);
+   for (int i = 0; i < dim-1; i++)
+   {
+      f.Set(i, new ConstantCoefficient(0.0));
+   }
+   {
+      Vector pull_force(pmesh.bdr_attributes.Max());
+      pull_force = 0.0;
+      pull_force(1) = -1.0e-2;
+      f.Set(dim-1, new PWConstCoefficient(pull_force));
   }

-   ComplexOperator::Convention conv =
-      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
+   ParLinearForm b(&fespace);
+   b.AddDomainIntegrator(new VectorBoundaryLFIntegrator(f));

-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
+   // 6. Set up the bilinear form a(.,.) on the finite element space
+   //    corresponding to the linear elasticity integrator with piece-wise
+   //    constants coefficient lambda and mu.
+   Vector lambda(pmesh.attributes.Max());
+   lambda = 1.0;
+   lambda(0) = lambda(1)*50;
+   PWConstCoefficient lambda_func(lambda);
+   Vector mu(pmesh.attributes.Max());
+   mu = 1.0;
+   mu(0) = mu(1)*50;
+   PWConstCoefficient mu_func(mu);

-   // 4. Refine the serial mesh on all processors to increase the resolution.
-   for (int l = 0; l < ser_ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int l = 0; l < par_ref_levels; l++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
-   //    the specified order.
-   if (dim == 1 && prob != 0 )
+   ParBilinearForm a(&fespace);
+   BilinearFormIntegrator *integ =
+      new ElasticityIntegrator(lambda_func,mu_func);
+   a.AddDomainIntegrator(integ);
+   if (static_cond) { a.EnableStaticCondensation(); }
+
+   // 7. The solution vector x and the associated finite element grid function
+   //    will be maintained over the AMR iterations. We initialize it to zero.
+   Vector zero_vec(dim);
+   zero_vec = 0.0;
+   VectorConstantCoefficient zero_vec_coeff(zero_vec);
+   ParGridFunction x(&fespace);
+   x = 0.0;
+
+   // 8. Determine the list of true (i.e. conforming) essential boundary dofs.
+   //    In this example, the boundary conditions are defined by marking only
+   //    boundary attribute 1 from the mesh as essential and converting it to a
+   //    list of true dofs.  The conversion to true dofs will be done in the
+   //    main loop.
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+
+   // 9. GLVis visualization.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sol_sock;
+
+   // 10. Set up an error estimator. Here we use the Zienkiewicz-Zhu estimator
+   //     that uses the ComputeElementFlux method of the ElasticityIntegrator to
+   //     recover a smoothed flux (stress) that is subtracted from the element
+   //     flux to get an error indicator. We need to supply the space for the
+   //     smoothed flux: an (H1)^tdim (i.e., vector-valued) space is used here.
+   //     Here, tdim represents the number of components for a symmetric (dim x
+   //     dim) tensor.
+   const int tdim = dim*(dim+1)/2;
+   L2_FECollection flux_fec(order, dim);
+   ParFiniteElementSpace flux_fespace(&pmesh, &flux_fec, tdim);
+   ParFiniteElementSpace smooth_flux_fespace(&pmesh, &fec, tdim);
+   L2ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace,
+                                       smooth_flux_fespace);
+
+   // 11. A refiner selects and refines elements based on a refinement strategy.
+   //     The strategy here is to refine elements with errors larger than a
+   //     fraction of the maximum element error. Other strategies are possible.
+   //     The refiner will call the given error estimator.
+   ThresholdRefiner refiner(estimator);
+   refiner.SetTotalErrorFraction(0.7);
+
+   // 12. The main AMR loop. In each iteration we solve the problem on the
+   //     current mesh, visualize the solution, and refine the mesh.
+   const int max_dofs = 50000;
+   const int max_amr_itr = 20;
+   for (int it = 0; it <= max_amr_itr; it++)
   {
+      HYPRE_Int global_dofs = fespace.GlobalTrueVSize();
      if (myid == 0)
      {
-         cout << "Switching to problem type 0, H1 basis functions, "
-              << "for 1 dimensional mesh." << endl;
+         cout << "\nAMR iteration " << it << endl;
+         cout << "Number of unknowns: " << global_dofs << endl;
      }
-      prob = 0;
-   }

-   FiniteElementCollection *fec = NULL;
-   switch (prob)
-   {
-      case 0:  fec = new H1_FECollection(order, dim);      break;
-      case 1:  fec = new ND_FECollection(order, dim);      break;
-      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
-      default: break; // This should be unreachable
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
+      // 13. Assemble the stiffness matrix and the right-hand side.
+      a.Assemble();
+      b.Assemble();

-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    based on the type of mesh and the problem type.
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      if (exact_sol)
+      // 14. Set Dirichlet boundary values in the GridFunction x.
+      //     Determine the list of Dirichlet true DOFs in the linear system.
+      Array<int> ess_tdof_list;
+      x.ProjectBdrCoefficient(zero_vec_coeff, ess_bdr);
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+
+      // 15. Create the linear system: eliminate boundary conditions, constrain
+      //     hanging nodes and possibly apply other transformations. The system
+      //     will be solved for true (unconstrained) DOFs only.
+
+      HypreParMatrix A;
+      Vector B, X;
+      const int copy_interior = 1;
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
+
+      // 16. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
+      //     preconditioner from hypre.
+      HypreBoomerAMG amg;
+      amg.SetPrintLevel(0);
+      // amg.SetSystemsOptions(dim); // optional
+      CGSolver pcg(A.GetComm());
+      pcg.SetPreconditioner(amg);
+      pcg.SetOperator(A);
+      pcg.SetRelTol(1e-6);
+      pcg.SetMaxIter(500);
+      pcg.SetPrintLevel(3); // print the first and the last iterations only
+      pcg.Mult(B, X);
+
+      // 17. After solving the linear system, reconstruct the solution as a
+      //     finite element GridFunction. Constrained nodes are interpolated
+      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
+      a.RecoverFEMSolution(X, b, x);
+
+      // 18. Send solution by socket to the GLVis server.
+      if (visualization && it == 0)
      {
-         switch (prob)
+         sol_sock.open(vishost, visport);
+         sol_sock.precision(8);
+      }
+      if (visualization && sol_sock.good())
+      {
+         GridFunction nodes(&fespace), *nodes_p = &nodes;
+         pmesh.GetNodes(nodes);
+         nodes += x;
+         int own_nodes = 0;
+         pmesh.SwapNodes(nodes_p, own_nodes);
+         x.Neg(); // visualize the backward displacement
+         sol_sock << "parallel " << num_procs << ' ' << myid << '\n';
+         sol_sock << "solution\n" << pmesh << x << flush;
+         x.Neg();
+         pmesh.SwapNodes(nodes_p, own_nodes);
+         if (it == 0)
         {
-            case 0:   ess_bdr = 0; ess_bdr[0] = 1;  break;
-            default:  ess_bdr = 1; ess_bdr[2] = 0;  break;
+            sol_sock << "keys '" << ((dim == 2) ? "Rjl" : "") << "m'" << endl;
+         }
+         sol_sock << "window_title 'AMR iteration: " << it << "'\n"
+                  << "pause" << endl;
+         if (myid == 0)
+         {
+            cout << "Visualization paused. "
+                 "Press <space> in the GLVis window to continue." << endl;
         }
      }
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }

-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system.
-   ParComplexLinearForm b(fespace, conv);
-   b.Vector::operator=(0.0);
-
-   // 9. Define the solution vector u as a parallel complex finite element grid
-   //    function corresponding to fespace. Initialize u with initial guess of
-   //    1+0i or the exact solution if it is known.
-   ParComplexGridFunction u(fespace);
-   ParComplexGridFunction * u_exact = NULL;
-   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
-
-   FunctionCoefficient u0_r(u0_real_exact);
-   FunctionCoefficient u0_i(u0_imag_exact);
-   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
-   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
-   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
-   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
-
-   ConstantCoefficient zeroCoef(0.0);
-   ConstantCoefficient oneCoef(1.0);
-
-   Vector zeroVec(dim); zeroVec = 0.0;
-   Vector  oneVec(dim);  oneVec = 0.0; oneVec[(prob==2)?(dim-1):0] = 1.0;
-   VectorConstantCoefficient zeroVecCoef(zeroVec);
-   VectorConstantCoefficient oneVecCoef(oneVec);
-
-   u = 0.0;
-   switch (prob)
-   {
-      case 0:
-         u.ProjectBdrCoefficient(oneCoef, zeroCoef, ess_bdr);
-         if (exact_sol) { u_exact->ProjectCoefficient(u0_r, u0_i); }
-         break;
-      case 1:
-         u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
-         if (exact_sol) { u_exact->ProjectCoefficient(u1_r, u1_i); }
-         break;
-      case 2:
-         u.ProjectBdrCoefficientNormal(oneVecCoef, zeroVecCoef, ess_bdr);
-         if (exact_sol) { u_exact->ProjectCoefficient(u2_r, u2_i); }
-         break;
-      default: break; // This should be unreachable
-   }
-
-   if (visualization && exact_sol)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
-                 << "window_title 'Exact: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
-                 << "window_title 'Exact: Imaginary Part'" << flush;
-   }
-
-   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
-   //     space corresponding to the damped harmonic oscillator operator of the
-   //     appropriate type:
-   //
-   //     0) A scalar H1 field
-   //        -Div(a Grad) - omega^2 b + i omega c
-   //
-   //     1) A vector H(Curl) field
-   //        Curl(a Curl) - omega^2 b + i omega c
-   //
-   //     2) A vector H(Div) field
-   //        -Grad(a Div) - omega^2 b + i omega c
-   //
-   ConstantCoefficient stiffnessCoef(1.0/mu_);
-   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
-   ConstantCoefficient lossCoef(omega_ * sigma_);
-   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
-
-   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
-   switch (prob)
-   {
-      case 0:
-         a->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new MassIntegrator(massCoef),
-                                new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         a->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
-
-   // 10a. Set up the parallel bilinear form for the preconditioner
-   //      corresponding to the appropriate operator
-   //
-   //      0) A scalar H1 field
-   //         -Div(a Grad) - omega^2 b + omega c
-   //
-   //      1) A vector H(Curl) field
-   //         Curl(a Curl) + omega^2 b + omega c
-   //
-   //      2) A vector H(Div) field
-   //         -Grad(a Div) - omega^2 b + omega c
-   //
-   ParBilinearForm *pcOp = new ParBilinearForm(fespace);
-   switch (prob)
-   {
-      case 0:
-         pcOp->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         pcOp->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         pcOp->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, etc.
-   a->Assemble();
-   pcOp->Assemble();
-
-   OperatorHandle A;
-   Vector B, U;
-
-   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
-   u = 0.0;
-   U = 0.0;
-
-   OperatorHandle PCOp;
-   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-
-   if (myid == 0)
-   {
-      ComplexHypreParMatrix * Ahyp =
-         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
-
-      cout << "Size of linear system: "
-           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
-   }
-
-   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
-   //     diagonal preconditioner based on the appropriate multigrid
-   //     preconditioner from hypre.
-   {
-      Array<HYPRE_Int> blockTrueOffsets;
-      blockTrueOffsets.SetSize(3);
-      blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = PCOp.Ptr()->Height();
-      blockTrueOffsets[2] = PCOp.Ptr()->Height();
-      blockTrueOffsets.PartialSum();
-
-      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
-
-      Operator * pc_r = NULL;
-      Operator * pc_i = NULL;
-
-      switch (prob)
+      if (global_dofs > max_dofs)
      {
-         case 0:
-            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-            break;
-         case 1:
-            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-            break;
-         case 2:
-            if (dim == 2 )
-            {
-               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-            }
-            else
-            {
-               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-            }
-            break;
-         default: break; // This should be unreachable
-      }
-      pc_i = new ScaledOperator(pc_r,
-                                (conv == ComplexOperator::HERMITIAN) ?
-                                1.0:-1.0);
-
-      BDP.SetDiagonalBlock(0, pc_r);
-      BDP.SetDiagonalBlock(1, pc_i);
-      BDP.owns_blocks = 1;
-
-      FGMRESSolver fgmres(MPI_COMM_WORLD);
-      fgmres.SetPreconditioner(BDP);
-      fgmres.SetOperator(*A.Ptr());
-      fgmres.SetRelTol(1e-12);
-      fgmres.SetMaxIter(1000);
-      fgmres.SetPrintLevel(1);
-      fgmres.Mult(B, U);
-   }
-
-   // 13. Recover the parallel grid function corresponding to U. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(U, b, u);
-
-   if (exact_sol)
-   {
-      double err_r = -1.0;
-      double err_i = -1.0;
-
-      switch (prob)
-      {
-         case 0:
-            err_r = u.real().ComputeL2Error(u0_r);
-            err_i = u.imag().ComputeL2Error(u0_i);
-            break;
-         case 1:
-            err_r = u.real().ComputeL2Error(u1_r);
-            err_i = u.imag().ComputeL2Error(u1_i);
-            break;
-         case 2:
-            err_r = u.real().ComputeL2Error(u2_r);
-            err_i = u.imag().ComputeL2Error(u2_i);
-            break;
-         default: break; // This should be unreachable
+         if (myid == 0)
+         {
+            cout << "Reached the maximum number of dofs. Stop." << endl;
+         }
+         break;
      }

-      if ( myid == 0 )
+      // 19. Call the refiner to modify the mesh. The refiner calls the error
+      //     estimator to obtain element errors, then it selects elements to be
+      //     refined and finally it modifies the mesh. The Stop() method can be
+      //     used to determine if a stopping criterion was met.
+      refiner.Apply(pmesh);
+      if (refiner.Stop())
      {
-         cout << endl;
-         cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
-         cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
-         cout << endl;
+         if (myid == 0)
+         {
+            cout << "Stopping criterion satisfied. Stop." << endl;
+         }
+         break;
      }
-   }

-   // 14. Save the refined mesh and the solution in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_r_name, sol_i_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
-      sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
+      // 20. Update the space to reflect the new state of the mesh. Also,
+      //     interpolate the solution x so that it lies in the new space but
+      //     represents the same function. This saves solver iterations later
+      //     since we'll have a good initial guess of x in the next step.
+      //     Internally, FiniteElementSpace::Update() calculates an
+      //     interpolation matrix which is then used by GridFunction::Update().
+      fespace.Update();
+      x.Update();

-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_r_ofs(sol_r_name.str().c_str());
-      ofstream sol_i_ofs(sol_i_name.str().c_str());
-      sol_r_ofs.precision(8);
-      sol_i_ofs.precision(8);
-      u.real().Save(sol_r_ofs);
-      u.imag().Save(sol_i_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *pmesh << u.real()
-                 << "window_title 'Solution: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *pmesh << u.imag()
-                 << "window_title 'Solution: Imaginary Part'" << flush;
-   }
-   if (visualization && exact_sol)
-   {
-      *u_exact -= u;
-
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
-                 << "window_title 'Error: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
-                 << "window_title 'Error: Imaginary Part'" << flush;
-   }
-   if (visualization)
-   {
-      ParGridFunction u_t(fespace);
-      u_t = u.real();
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << u_t
-               << "window_title 'Harmonic Solution (t = 0.0 T)'"
-               << "pause\n" << flush;
-      if (myid == 0)
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      int num_frames = 32;
-      int i = 0;
-      while (sol_sock)
+      // 21. Load balance the mesh, and update the space and solution. Currently
+      //     available only for nonconforming meshes.
+      if (pmesh.Nonconforming())
      {
-         double t = (double)(i % num_frames) / num_frames;
-         ostringstream oss;
-         oss << "Harmonic Solution (t = " << t << " T)";
+         pmesh.Rebalance();

-         add(cos( 2.0 * M_PI * t), u.real(),
-             sin(-2.0 * M_PI * t), u.imag(), u_t);
-         sol_sock << "parallel " << num_procs << " " << myid << "\n";
-         sol_sock << "solution\n" << *pmesh << u_t
-                  << "window_title '" << oss.str() << "'" << flush;
-         i++;
+         // Update the space and the GridFunction. This time the update matrix
+         // redistributes the GridFunction among the processors.
+         fespace.Update();
+         x.Update();
      }
+
+      // 22. Inform also the bilinear and linear forms that the space has
+      //     changed.
+      a.Update();
+      b.Update();
   }

-   // 16. Free the used memory.
-   delete a;
-   delete u_exact;
-   delete pcOp;
-   delete fespace;
-   delete fec;
-   delete pmesh;
+   {
+      ostringstream mref_name, mesh_name, sol_name;
+      mref_name << "ex22p_reference_mesh." << setfill('0') << setw(6) << myid;
+      mesh_name << "ex22p_deformed_mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "ex22p_displacement." << setfill('0') << setw(6) << myid;
+
+      ofstream mesh_ref_out(mref_name.str().c_str());
+      mesh_ref_out.precision(16);
+      pmesh.Print(mesh_ref_out);
+
+      ofstream mesh_out(mesh_name.str().c_str());
+      mesh_out.precision(16);
+      GridFunction nodes(&fespace), *nodes_p = &nodes;
+      pmesh.GetNodes(nodes);
+      nodes += x;
+      int own_nodes = 0;
+      pmesh.SwapNodes(nodes_p, own_nodes);
+      pmesh.Print(mesh_out);
+      pmesh.SwapNodes(nodes_p, own_nodes);
+
+      ofstream x_out(sol_name.str().c_str());
+      x_out.precision(16);
+      x.Save(x_out);
+   }

   MPI_Finalize();
-
   return 0;
 }
-
-bool check_for_inline_mesh(const char * mesh_file)
-{
-   string file(mesh_file);
-   size_t p0 = file.find_last_of("/");
-   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
-   return s0 == "inline-";
-}
-
-complex<double> u0_exact(const Vector &x)
-{
-   int dim = x.Size();
-   complex<double> i(0.0, 1.0);
-   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
-   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
-   return std::exp(-i * kappa * x[dim - 1]);
-}
-
-double u0_real_exact(const Vector &x)
-{
-   return u0_exact(x).real();
-}
-
-double u0_imag_exact(const Vector &x)
-{
-   return u0_exact(x).imag();
-}
-
-void u1_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
-}
-
-void u1_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
-}
-
-void u2_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
-}
-
-void u2_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
-}
@@ -102,7 +102,8 @@ int main(int argc, char *argv[])
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
   {
-      int ref_levels = (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
@@ -21,7 +21,7 @@
 //
 //               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in a
-//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
+//               VisIt (visit.llnl.gov) visualization format.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

@@ -264,16 +264,7 @@ int main(int argc, char *argv[])
   visit_dc.RegisterField("pressure", &p);
   visit_dc.Save();

-   // 14. Save data in the ParaView format
-   ParaViewDataCollection paraview_dc("PVExample5S", mesh);
-   paraview_dc.SetLevelsOfDetail(2);
-   paraview_dc.SetCycle(1);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("velocity",&u);
-   paraview_dc.RegisterField("pressure",&p);
-   paraview_dc.Save();
-
-   // 15. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -286,7 +277,7 @@ int main(int argc, char *argv[])
      p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
   }

-   // 16. Free the used memory.
+   // 15. Free the used memory.
   delete fform;
   delete gform;
   delete invM;
@@ -21,7 +21,7 @@
 //
 //               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in a
-//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
+//               VisIt (visit.llnl.gov) visualization format.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

@@ -239,6 +239,7 @@ int main(int argc, char *argv[])

   // 12. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
+
   int maxIter(500);
   double rtol(1.e-6);
   double atol(1.e-10);
@@ -325,16 +326,7 @@ int main(int argc, char *argv[])
                      DataCollection::PARALLEL_FORMAT);
   visit_dc.Save();

-   // 16. Save data in the ParaView format
-   ParaViewDataCollection paraview_dc("PVExample5P", pmesh);
-   paraview_dc.SetLevelsOfDetail(1);
-   paraview_dc.SetCycle(1);
-   paraview_dc.SetTime(0.0);
-   paraview_dc.RegisterField("velocity",u);
-   paraview_dc.RegisterField("pressure",p);
-   paraview_dc.Save();
-
-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -354,7 +346,7 @@ int main(int argc, char *argv[])
             << endl;
   }

-   // 18. Free the used memory.
+   // 17. Free the used memory.
   delete fform;
   delete gform;
   delete u;
@@ -19,8 +19,6 @@
 //               ex6 -pa -d cuda
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
-//               ex6 -pa -d ceed-cpu
-//               ex6 -pa -d ceed-cuda
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
@@ -51,7 +49,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -61,7 +59,7 @@ int main(int argc, char *argv[])
                  "Finite element order (polynomial degree).");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -74,19 +72,14 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh mesh(mesh_file, 1, 1);
   int dim = mesh.Dimension();
   int sdim = mesh.SpaceDimension();

-   // 4. Since a NURBS mesh can currently only be refined uniformly, we need to
+   // 3. Since a NURBS mesh can currently only be refined uniformly, we need to
   //    convert it to a piecewise-polynomial curved mesh. First we refine the
   //    NURBS mesh a bit more and then project the curvature to quadratic Nodes.
   if (mesh.NURBSext)
@@ -98,11 +91,15 @@ int main(int argc, char *argv[])
      mesh.SetCurvature(2);
   }

-   // 5. Define a finite element space on the mesh. The polynomial order is
+   // 4. Define a finite element space on the mesh. The polynomial order is
   //    one (linear) by default, but this can be changed on the command line.
   H1_FECollection fec(order, dim);
   FiniteElementSpace fespace(&mesh, &fec);

+   // 5. Set device config parameters from the command line options.
+   Device::Configure(device);
+   Device::Print();
+
   // 6. As in Example 1, we set up bilinear and linear forms corresponding to
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
@@ -171,7 +168,8 @@ int main(int argc, char *argv[])
      x.ProjectBdrCoefficient(zero, ess_bdr);
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);

-      // 15. Assemble the stiffness matrix.
+      // 15. Switch to the device and assemble the stiffness matrix.
+      Device::Enable();
      a.Assemble();

      // 16. Create the linear system: eliminate boundary conditions, constrain
@@ -206,6 +204,7 @@ int main(int argc, char *argv[])
      // 18. After solving the linear system, reconstruct the solution as a
      //     finite element GridFunction. Constrained nodes are interpolated
      //     from true DOFs (it may therefore happen that x.Size() >= X.Size()).
+      Device::Disable();
      a.RecoverFEMSolution(X, b, x);

      // 19. Send solution by socket to the GLVis server.
@@ -19,8 +19,6 @@
 //               mpirun -np 4 ex6p -pa -d cuda
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
-//               mpirun -np 4 ex6p -pa -d ceed-cpu
-//               mpirun -np 4 ex6p -pa -d ceed-cuda
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
@@ -57,7 +55,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   bool pa = false;
-   const char *device_config = "cpu";
+   const char *device = "cpu";
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -67,7 +65,7 @@ int main(int argc, char *argv[])
                  "Finite element order (polynomial degree).");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
+   args.AddOption(&device, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -87,19 +85,14 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
   //    Also project a NURBS mesh to a piecewise-quadratic curved mesh. Make
   //    sure that the mesh is non-conforming.
   if (mesh->NURBSext)
@@ -109,7 +102,7 @@ int main(int argc, char *argv[])
   }
   mesh->EnsureNCMesh();

-   // 6. Define a parallel mesh by partitioning the serial mesh.
+   // 5. Define a parallel mesh by partitioning the serial mesh.
   //    Once the parallel mesh is defined, the serial mesh can be deleted.
   ParMesh pmesh(MPI_COMM_WORLD, *mesh);
   delete mesh;
@@ -119,11 +112,15 @@ int main(int argc, char *argv[])
   Array<int> ess_bdr(pmesh.bdr_attributes.Max());
   ess_bdr = 1;

-   // 7. Define a finite element space on the mesh. The polynomial order is
+   // 6. Define a finite element space on the mesh. The polynomial order is
   //    one (linear) by default, but this can be changed on the command line.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(&pmesh, &fec);

+   // 7. Set device config parameters from the command line options.
+   Device::Configure(device);
+   if (myid == 0) { Device::Print(); }
+
   // 8. As in Example 1p, we set up bilinear and linear forms corresponding to
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
@@ -203,10 +200,11 @@ int main(int argc, char *argv[])
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
      b.Assemble();

-      // 15. Assemble the stiffness matrix. Note that MFEM doesn't care at this
-      //     point that the mesh is nonconforming and parallel.  The FE space is
-      //     considered 'cut' along hanging edges/faces, and also across
-      //     processor boundaries.
+      // 15. Switch to the device and assemble the stiffness matrix. Note that
+      //     MFEM doesn't care at this point that the mesh is nonconforming and
+      //     parallel.  The FE space is considered 'cut' along hanging
+      //     edges/faces, and also across processor boundaries.
+      Device::Enable();
      a.Assemble();

      // 16. Create the parallel linear system: eliminate boundary conditions.
@@ -234,6 +232,7 @@ int main(int argc, char *argv[])
      // 18. Switch back to the host and extract the parallel grid function
      //     corresponding to the finite element approximation X. This is the
      //     local solution on each processor.
+      Device::Disable();
      a.RecoverFEMSolution(X, b, x);

      // 19. Send the solution by socket to a GLVis server.
@@ -9,7 +9,6 @@
 //    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 13 -tf 9
 //    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
@@ -22,13 +21,12 @@
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -54,54 +52,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &K, A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         A = K;
-         A *= -dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -115,16 +65,13 @@ private:
   const Vector &b;
   DSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
-                const FiniteElementSpace &fes);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

   virtual ~FE_Evolution() { }
 };
@@ -142,7 +89,6 @@ int main(int argc, char *argv[])
   double dt = 0.01;
   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;

@@ -160,11 +106,7 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
-                  "            11 - Backward Euler,\n\t"
-                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -175,9 +117,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -201,21 +140,11 @@ int main(int argc, char *argv[])
   ODESolver *ode_solver = NULL;
   switch (ode_solver_type)
   {
-      // Explicit methods
      case 1: ode_solver = new ForwardEulerSolver; break;
      case 2: ode_solver = new RK2Solver(1.0); break;
      case 3: ode_solver = new RK3SSPSolver; break;
      case 4: ode_solver = new RK4Solver; break;
      case 6: ode_solver = new RK6Solver; break;
-      // Implicit (L-stable) methods
-      case 11: ode_solver = new BackwardEulerSolver; break;
-      case 12: ode_solver = new SDIRK23Solver(2); break;
-      case 13: ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-
      default:
         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
         return 3;
@@ -308,16 +237,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("PVExample9S", &mesh);
-      pd->RegisterField("solution", &u);
-      pd->SetLevelsOfDetail(2);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-   }
-
   socketstream sout;
   if (visualization)
   {
@@ -345,7 +264,7 @@ int main(int argc, char *argv[])
   // 8. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and perform time-integration (looping over the time
   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m.SpMat(), k.SpMat(), b, fes);
+   FE_Evolution adv(m.SpMat(), k.SpMat(), b);

   double t = 0.0;
   adv.SetTime(t);
@@ -375,13 +294,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
      }
   }

@@ -395,7 +307,6 @@ int main(int argc, char *argv[])

   // 10. Free the used memory.
   delete ode_solver;
-   delete pd;
   delete dc;

   return 0;
@@ -403,10 +314,8 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
-                           const FiniteElementSpace &fes)
-   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), dg_solver(M, K, fes),
-     z(_M.Size())
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
 {
   M_solver.SetPreconditioner(M_prec);
   M_solver.SetOperator(M);
@@ -426,13 +335,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
   M_solver.Mult(z, y);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   K.Mult(x, z);
-   z += b;
-   dg_solver.SetTimeStep(dt);
-   dg_solver.Mult(z, k);
-}

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -9,7 +9,6 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
-//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 13 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
@@ -22,13 +21,12 @@
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -53,66 +51,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &K;
-   SparseMatrix M_diag;
-   HypreParMatrix *A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        A(NULL),
-        linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-
-      M.GetDiag(M_diag);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         delete A;
-         A = Add(-dt, K, 0.0, K);
-         SparseMatrix A_diag;
-         A->GetDiag(A_diag);
-         A_diag.Add(1.0, M_diag);
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   ~DG_Solver()
-   {
-      delete A;
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -126,16 +64,13 @@ private:
   const Vector &b;
   HypreSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b,
-                const FiniteElementSpace &fes);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

   virtual ~FE_Evolution() { }
 };
@@ -160,7 +95,6 @@ int main(int argc, char *argv[])
   double dt = 0.01;
   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;

@@ -180,11 +114,7 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
-                  "            11 - Backward Euler,\n\t"
-                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -195,9 +125,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -228,20 +155,11 @@ int main(int argc, char *argv[])
   ODESolver *ode_solver = NULL;
   switch (ode_solver_type)
   {
-      // Explicit methods
      case 1: ode_solver = new ForwardEulerSolver; break;
      case 2: ode_solver = new RK2Solver(1.0); break;
      case 3: ode_solver = new RK3SSPSolver; break;
      case 4: ode_solver = new RK4Solver; break;
      case 6: ode_solver = new RK6Solver; break;
-      // Implicit (L-stable) methods
-      case 11: ode_solver = new BackwardEulerSolver; break;
-      case 12: ode_solver = new SDIRK23Solver(2); break;
-      case 13: ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
      default:
         if (myid == 0)
         {
@@ -363,17 +281,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("PVExample9P", pmesh);
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(2);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
   socketstream sout;
   if (visualization)
   {
@@ -407,7 +314,7 @@ int main(int argc, char *argv[])
   // 10. Define the time-dependent evolution operator describing the ODE
   //     right-hand side, and perform time-integration (looping over the time
   //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*M, *K, *B, *fes);
+   FE_Evolution adv(*M, *K, *B);

   double t = 0.0;
   adv.SetTime(t);
@@ -445,13 +352,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
      }
   }

@@ -478,7 +378,6 @@ int main(int argc, char *argv[])
   delete fes;
   delete pmesh;
   delete ode_solver;
-   delete pd;
   delete dc;

   MPI_Finalize();
@@ -488,10 +387,9 @@ int main(int argc, char *argv[])

 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                           const Vector &_b, const FiniteElementSpace &fes)
+                           const Vector &_b)
   : TimeDependentOperator(_M.Height()),
-     M(_M), K(_K), b(_b), M_solver(M.GetComm()),
-     dg_solver(M, K, fes), z(_M.Height())
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
 {
   M_prec.SetType(HypreSmoother::Jacobi);
   M_solver.SetPreconditioner(M_prec);
@@ -504,14 +402,6 @@ FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
   M_solver.SetPrintLevel(0);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   K.Mult(x, z);
-   z += b;
-   dg_solver.SetTimeStep(dt);
-   dg_solver.Mult(z, k);
-}
-
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
   // y = M^{-1} (K x + b)
@@ -1,59 +0,0 @@
-# Copyright (c) 2019, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-set(GINKGO_EXAMPLES_SRCS)
-list(APPEND GINKGO_EXAMPLES_SRCS
-  ex1.cpp
-)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_ginkgo" target, see below.
-add_custom_target(test_ginkgo
-  ${CMAKE_CTEST_COMMAND} -R ginkgo USES_TERMINAL)
-
-# Add one executable per cpp file, adding "ginkgo_" as prefix. Sets
-# "test_ginkgo" as a target that depends on the given examples.
-set(PFX ginkgo_)
-add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)
-
-# Testing.
-# The GINKGO tests can be run separately using the target "test_ginkgo"
-# which builds the examples and runs:
-#   ctest -R ginkgo
-
-# Command line options for the tests.
-set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
-set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,259 +0,0 @@
-//                                MFEM Example 1
-//                             GINKGO Modification
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_GINKGO
-#error This example requires that MFEM is built with MFEM_USE_GINKGO=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool use_ginkgo_solver= true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&use_ginkgo_solver, "-gko", "--use_gko_solver", "-no-gko",
-                  "--no-gko-solver",
-                  "Solve using ginkgo.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
-      if (use_ginkgo_solver)
-      {
-#ifdef MFEM_USE_GINKGO
-         // Solve the linear system with CG + ILU from Ginkgo.
-         std::string executor = "reference";
-         auto exec = gko::ReferenceExecutor::create();
-         auto ilu_precond =
-            gko::preconditioner::Ilu<gko::solver::LowerTrs<>,
-            gko::solver::UpperTrs<>, false>::build()
-            .on(exec);
-         GinkgoWrappers::CGSolver ginkgo_solver(executor, 1, 2000, 1e-12, 0.0,
-                                                ilu_precond.release() );
-         ginkgo_solver.solve(&((SparseMatrix&)(*A)), X, B);
-#endif
-      }
-      else
-      {
-#ifndef MFEM_USE_SUITESPARSE
-         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-         GSSmoother M((SparseMatrix&)(*A));
-         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-         UMFPackSolver umf_solver;
-         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-         umf_solver.SetOperator(*A);
-         umf_solver.Mult(B, X);
-#endif
-      }
-   }
-   else // No preconditioning for now in partial assembly mode.
-   {
-      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
-
-   return 0;
-}
@@ -1,81 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ginkgo/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-# Currently there are only serial Ginkgo examples
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES =
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_GINKGO),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with GINKO)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI_NP = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
-RUN_MPI = $(RUN_MPI_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial GINKGO example
-PARALLEL_NAME := Parallel GINKGO example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: Example-specific execution options:
-ex1-test-par: ex1
-	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f refined.mesh sol.gf mesh.* sol.*
@@ -1,63 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-set(HIOP_EXAMPLES_SRCS)
-list(APPEND HIOP_EXAMPLES_SRCS ex9.cpp)
-
-if (MFEM_USE_MPI)
-  list(APPEND HIOP_EXAMPLES_SRCS ex9p.cpp)
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_hiop" target, see below.
-add_custom_target(test_hiop
-  ${CMAKE_CTEST_COMMAND} -R hiop USES_TERMINAL)
-
-# Add one executable per cpp file, adding "hiop_" as prefix. Sets
-# "test_hiop" as a target that depends on the given examples.
-set(PFX hiop_)
-add_mfem_examples(HIOP_EXAMPLES_SRCS ${PFX} "" test_hiop)
-
-# Testing.
-# The HIOP tests can be run separately using the target "test_hiop"
-# which builds the examples and runs:
-#   ctest -R hiop
-
-# Command line options for the tests.
-# Example 9:
-set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
-set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,684 +0,0 @@
-//            MFEM Example 9 with Nonlinear Constrained Optimization
-//
-// Compile with: make ex9
-//
-// Sample runs:
-//
-//    ex9 -m ../../data/periodic-segment.mesh -r 3 -p 0 -o 2 -dt 0.002 -opt 1
-//    ex9 -m ../../data/periodic-segment.mesh -r 3 -p 0 -o 2 -dt 0.002 -opt 2
-//
-//    ex9 -m ../../data/periodic-square.mesh -p 0 -r 2 -dt 0.01 -tf 10 -opt 1
-//    ex9 -m ../../data/periodic-square.mesh -p 0 -r 2 -dt 0.01 -tf 10 -opt 2
-//
-//    ex9 -m ../../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 1
-//    ex9 -m ../../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/amr-quad.mesh -p 1 -r 1 -dt 0.002 -tf 9 -opt 1
-//    ex9 -m ../../data/amr-quad.mesh -p 1 -r 1 -dt 0.002 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/disc-nurbs.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 1
-//    ex9 -m ../../data/disc-nurbs.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/disc-nurbs.mesh -p 2 -r 2 -dt 0.01 -tf 9 -opt 1
-//    ex9 -m ../../data/disc-nurbs.mesh -p 2 -r 2 -dt 0.01 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/periodic-square.mesh -p 3 -r 3 -dt 0.0025 -tf 9 -opt 1
-//    ex9 -m ../../data/periodic-square.mesh -p 3 -r 3 -dt 0.0025 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8 -opt 1
-//    ex9 -m ../../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8 -opt 2
-
-// Description:  This example modifies the standard MFEM ex9 by adding nonlinear
-//               constrained optimization capabilities through the SLBQP and
-//               HIOP solvers. It demonstrates how a user can define a custom
-//               class OptimizationProblem that includes linear/nonlinear
-//               equality/inequality constraints. This optimization is applied
-//               as post-processing to the solution of the transport equation.
-//
-//               Description of ex9:
-//               This example code solves the time-dependent advection equation
-//               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem;
-
-// Nonlinear optimizer.
-int optimizer_type;
-
-// Velocity coefficient
-bool invert_velocity = false;
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-/// Computes C(x) = sum w_i x_i, where w is a given Vector.
-class LinearScaleOperator : public Operator
-{
-private:
-   const Vector &w;
-   mutable DenseMatrix grad;
-
-public:
-   LinearScaleOperator(const Vector &weight)
-      : Operator(1, weight.Size()), w(weight), grad(1, width)
-   {
-      for (int i = 0; i < width; i++) { grad(0, i) = w(i); }
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      y(0) = w * x;
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      return grad;
-   }
-};
-
-/// Nonlinear monotone bounded operator to test nonlinear ineq constraints.
-/// Computes D(x) = tanh(sum(x_i)).
-class TanhSumOperator : public Operator
-{
-private:
-   mutable DenseMatrix grad;
-
-public:
-   TanhSumOperator(int size) : Operator(1, size), grad(1, width) { }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      y(0) = std::tanh(x.Sum());
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      const double ts = std::tanh(x.Sum());
-      const double dtanh = 1.0 - ts * ts;
-      for (int i = 0; i < width; i++) { grad(0, i) = dtanh; }
-      return grad;
-   }
-};
-
-/** Monotone and conservative a-posteriori correction for transport solutions:
- *  Find x that minimizes 0.5 || x - x_HO ||^2, subject to
- *  sum w_i x_i = mass,
- *  tanh(sum(x_i_min)) <= tanh(sum(x_i)) <= tanh(sum(x_i_max)),
- *  x_i_min <= x_i <= x_i_max,
- */
-class OptimizedTransportProblem : public OptimizationProblem
-{
-private:
-   const Vector &x_HO;
-   Vector massvec, d_lo, d_hi;
-   const LinearScaleOperator LSoper;
-   const TanhSumOperator TSoper;
-
-public:
-   OptimizedTransportProblem(const Vector &xho, const Vector &w, double mass,
-                             const Vector &xmin, const Vector &xmax)
-      : OptimizationProblem(xho.Size(), NULL, NULL),
-        x_HO(xho), massvec(1), d_lo(1), d_hi(1),
-        LSoper(w), TSoper(w.Size())
-   {
-      C = &LSoper;
-      massvec(0) = mass;
-      SetEqualityConstraint(massvec);
-
-      D = &TSoper;
-      d_lo(0) = std::tanh(xmin.Sum());
-      d_hi(0) = std::tanh(xmax.Sum());
-      MFEM_ASSERT(d_lo(0) < d_hi(0),
-                  "The bounds produce an infeasible optimization problem");
-      SetInequalityConstraint(d_lo, d_hi);
-
-      SetSolutionBounds(xmin, xmax);
-   }
-
-   virtual double CalcObjective(const Vector &x) const
-   {
-      double res = 0.0;
-      for (int i = 0; i < input_size; i++)
-      {
-         const double d = x(i) - x_HO(i);
-         res += d * d;
-      }
-      return 0.5 * res;
-   }
-
-   virtual void CalcObjectiveGrad(const Vector &x, Vector &grad) const
-   {
-      for (int i = 0; i < input_size; i++) { grad(i) = x(i) - x_HO(i); }
-   }
-};
-
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
-    and advection matrices, and b describes the flow on the boundary. This can
-    be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
-    used to evaluate the right-hand side. */
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   SparseMatrix &M, &K;
-   const Vector &b;
-   DSmoother M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-   double dt;
-   BilinearForm &bf;
-   Vector &M_rowsums;
-
-public:
-   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
-                BilinearForm &_bf, Vector &M_rs);
-
-   void SetTimeStep(double _dt) { dt = _dt; }
-   void SetK(SparseMatrix &_K) { K = _K; }
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution() { }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   problem = 0;
-   optimizer_type = 1;
-   const char *mesh_file = "../../data/periodic-hexagon.mesh";
-   int ref_levels = 2;
-   int order = 3;
-   int ode_solver_type = 3;
-   double t_final = 1.0;
-   double dt = 0.01;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&optimizer_type, "-opt", "--optimizer",
-                  "Nonlinear optimizer: 1 - SLBQP,\n\t"
-                  "                     2 - HIOP.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle geometrically
-   //    periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Define the ODE solver used for time integration. Several explicit
-   //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter. If the mesh is of NURBS type, we convert it to
-   //    a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::Positive);
-   FiniteElementSpace fes(mesh, &fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   // 6. Set up and assemble the bilinear and linear forms corresponding to the
-   //    DG discretization. The DGTraceIntegrator involves integrals over mesh
-   //    interior faces.
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   BilinearForm m(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
-   BilinearForm k(&fes);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-   k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-
-   LinearForm b(&fes);
-   b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
-
-   m.Assemble();
-   m.Finalize();
-   int skip_zeros = 0;
-   k.Assemble(skip_zeros);
-   k.Finalize(skip_zeros);
-   b.Assemble();
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u(&fes);
-   u.ProjectCoefficient(u0);
-
-   {
-      ofstream omesh("ex9.mesh");
-      omesh.precision(precision);
-      mesh->Print(omesh);
-      ofstream osol("ex9-init.gf");
-      osol.precision(precision);
-      u.Save(osol);
-   }
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9", mesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example9", mesh);
-         dc->SetPrecision(precision);
-      }
-      dc->RegisterField("solution", &u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *mesh << u;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   Vector M_rowsums(m.Size());
-   m.SpMat().GetRowSums(M_rowsums);
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m.SpMat(), k.SpMat(), b, k, M_rowsums);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   // Compute initial volume.
-   const double vol0 = M_rowsums * u;
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-      adv.SetTimeStep(dt_real);
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-
-         if (visualization)
-         {
-            sout << "solution\n" << *mesh << u << flush;
-         }
-
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-      }
-   }
-
-   // Print the error vs exact solution.
-   const double max_error = u.ComputeMaxError(u0),
-                l1_error  = u.ComputeL1Error(u0),
-                l2_error  = u.ComputeL2Error(u0);
-   std::cout << "Linf error = " << max_error << endl
-             << "L1   error = " << l1_error << endl
-             << "L2   error = " << l2_error << endl;
-
-   // Print error in volume.
-   const double vol = M_rowsums * u;
-   std::cout << "Vol  error = " << vol - vol0 << endl;
-
-   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m ex9.mesh -g ex9-final.gf".
-   {
-      ofstream osol("ex9-final.gf");
-      osol.precision(precision);
-      u.Save(osol);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete dc;
-   delete mesh;
-
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K,
-                           const Vector &_b, BilinearForm &_bf, Vector &M_rs)
-   : TimeDependentOperator(_M.Size()),
-     M(_M), K(_K), b(_b), M_prec(), M_solver(), z(_M.Size()),
-     bf(_bf), M_rowsums(M_rs)
-{
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // Compute bounds y_min, y_max for y from x on the ldofs.
-   const int dofs = x.Size();
-   Vector y_min(dofs), y_max(dofs);
-   const int *In = bf.SpMat().GetI(), *Jn = bf.SpMat().GetJ();
-   for (int i = 0, k = 0; i < dofs; i++)
-   {
-      double x_i_min = +std::numeric_limits<double>::infinity();
-      double x_i_max = -std::numeric_limits<double>::infinity();
-      for (int end = In[i+1]; k < end; k++)
-      {
-         const int j = Jn[k];
-         if (x(j) > x_i_max) { x_i_max = x(j); }
-         if (x(j) < x_i_min) { x_i_min = x(j); }
-      }
-      y_min(i) = x_i_min;
-      y_max(i) = x_i_max;
-   }
-   for (int i = 0; i < dofs; i++)
-   {
-      y_min(i) = (y_min(i) - x(i) ) / dt;
-      y_max(i) = (y_max(i) - x(i) ) / dt;
-   }
-
-   // Compute the high-order solution y = M^{-1} (K x + b).
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-
-   // The solution y is an increment; it should not introduce new mass.
-   const double mass_y = 0.0;
-
-   // Perform optimization.
-   Vector y_out(dofs);
-   const int max_iter = 500;
-   const double rtol = 1.e-7;
-   double atol = 1.e-7;
-
-   OptimizationSolver *optsolver = NULL;
-   if (optimizer_type == 2)
-   {
-#ifdef MFEM_USE_HIOP
-      HiopNlpOptimizer *tmp_opt_ptr = new HiopNlpOptimizer();
-      optsolver = tmp_opt_ptr;
-#else
-      MFEM_ABORT("MFEM is not built with HiOp support!");
-#endif
-   }
-   else
-   {
-      SLBQPOptimizer *slbqp = new SLBQPOptimizer();
-      slbqp->SetBounds(y_min, y_max);
-      slbqp->SetLinearConstraint(M_rowsums, mass_y);
-      atol = 1.e-15;
-      optsolver = slbqp;
-   }
-
-   OptimizedTransportProblem ot_prob(y, M_rowsums, mass_y, y_min, y_max);
-   optsolver->SetOptimizationProblem(ot_prob);
-
-   optsolver->SetMaxIter(max_iter);
-   optsolver->SetAbsTol(atol);
-   optsolver->SetRelTol(rtol);
-   optsolver->SetPrintLevel(0);
-   optsolver->Mult(y, y_out);
-
-   y = y_out;
-
-   delete optsolver;
-}
-
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = (invert_velocity) ? -1.0 : 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return (X(0) > -0.15 && X(0) < 0.15) ? 1.0 : 0.0;
-            //return exp(-40.*pow(X(0)-0.0,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}
@@ -1,802 +0,0 @@
-//  MFEM Example 9 with Nonlinear Constrained Optimization - Parallel Version
-//
-// Compile with: make ex9p
-//
-// Sample runs:
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 0 -rs 2 -dt 0.01 -tf 10 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 0 -rs 2 -dt 0.01 -tf 10 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh -p 1 -rs 1 -dt 0.002 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh -p 1 -rs 1 -dt 0.002 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 2 -rs 2 -dt 0.01 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 2 -rs 2 -dt 0.01 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 3 -rs 3 -dt 0.0025 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 3 -rs 3 -dt 0.0025 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.02 -tf 8 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.02 -tf 8 -opt 2
-
-// Description:  This example modifies the standard MFEM ex9 by adding nonlinear
-//               constrained optimization capabilities through the SLBQP and
-//               HIOP solvers. It demonstrates how a user can define a custom
-//               class OptimizationProblem that includes linear/nonlinear
-//               equality/inequality constraints. This optimization is applied
-//               as post-processing to the solution of the transport equation.
-//
-//               Description of ex9:
-//               This example code solves the time-dependent advection equation
-//               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem;
-
-// Nonlinear optimizer.
-int optimizer_type;
-
-// Velocity coefficient
-bool invert_velocity = false;
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-/// Computes C(x) = sum w_i x_i, where w is a given Vector.
-class LinearScaleOperator : public Operator
-{
-private:
-   ParFiniteElementSpace &pfes;
-   // Local weights.
-   const Vector &w;
-   // Gradient for the tdofs.
-   mutable DenseMatrix grad;
-
-public:
-   LinearScaleOperator(ParFiniteElementSpace &space, const Vector &weight)
-      : Operator(1, space.TrueVSize()),
-        pfes(space), w(weight), grad(1, width)
-   {
-      Vector w_glob(width);
-      pfes.Dof_TrueDof_Matrix()->MultTranspose(w, w_glob);
-      for (int i = 0; i < width; i++) { grad(0, i) = w_glob(i); }
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      Vector x_loc(w.Size());
-      pfes.GetProlongationMatrix()->Mult(x, x_loc);
-      const double loc_res = w * x_loc;
-      MPI_Allreduce(&loc_res, &y(0), 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      return grad;
-   }
-};
-
-/// Nonlinear monotone bounded operator to test nonlinear ineq constraints.
-/// Computes D(x) = tanh(sum(x_i)).
-class TanhSumOperator : public Operator
-{
-private:
-   // Gradient for the tdofs.
-   mutable DenseMatrix grad;
-
-public:
-   TanhSumOperator(ParFiniteElementSpace &space)
-      : Operator(1, space.TrueVSize()), grad(1, width) { }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      double sum_loc = x.Sum();
-      MPI_Allreduce(&sum_loc, &y(0), 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      y(0) = std::tanh(y(0));
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      double sum_loc = x.Sum();
-      double dtanh;
-      MPI_Allreduce(&sum_loc, &dtanh, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      dtanh = 1.0 - pow(std::tanh(dtanh), 2);
-
-      for (int i = 0; i < width; i++) { grad(0, i) = dtanh; }
-      return grad;
-   }
-};
-
-
-/** Monotone and conservative a-posteriori correction for transport solutions:
- *  Find x that minimizes 0.5 || x - x_HO ||^2, subject to
- *  sum w_i x_i = mass,
- *  tanh(sum(x_i_min)) <= tanh(sum(x_i)) <= tanh(sum(x_i_max)),
- *  x_i_min <= x_i <= x_i_max,
- */
-class OptimizedTransportProblem : public OptimizationProblem
-{
-private:
-   const Vector &x_HO;
-   Vector massvec, d_lo, d_hi;
-   const LinearScaleOperator LSoper;
-   const TanhSumOperator TSoper;
-
-public:
-   OptimizedTransportProblem(ParFiniteElementSpace &space,
-                             const Vector &xho, const Vector &w, double mass,
-                             const Vector &xmin, const Vector &xmax)
-      : OptimizationProblem(xho.Size(), NULL, NULL),
-        x_HO(xho), massvec(1), d_lo(1), d_hi(1),
-        LSoper(space, w), TSoper(space)
-   {
-      C = &LSoper;
-      massvec(0) = mass;
-      SetEqualityConstraint(massvec);
-
-      D = &TSoper;
-      double lsums[2], gsums[2];
-      lsums[0] = xmin.Sum();
-      lsums[1] = xmax.Sum();
-      MPI_Allreduce(lsums, gsums, 2, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      d_lo(0) = std::tanh(gsums[0]);
-      d_hi(0) = std::tanh(gsums[1]);
-      MFEM_ASSERT(d_lo(0) < d_hi(0),
-                  "The bounds produce an infeasible optimization problem");
-      SetInequalityConstraint(d_lo, d_hi);
-
-      SetSolutionBounds(xmin, xmax);
-   }
-
-   virtual double CalcObjective(const Vector &x) const
-   {
-      double loc_res = 0.0;
-      for (int i = 0; i < input_size; i++)
-      {
-         const double d = x(i) - x_HO(i);
-         loc_res += d * d;
-      }
-      loc_res *= 0.5;
-      double res;
-      MPI_Allreduce(&loc_res, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      return res;
-   }
-
-   virtual void CalcObjectiveGrad(const Vector &x, Vector &grad) const
-   {
-      for (int i = 0; i < input_size; i++) { grad(i) = x(i) - x_HO(i); }
-   }
-};
-
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
-    and advection matrices, and b describes the flow on the boundary. This can
-    be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
-    used to evaluate the right-hand side. */
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   HypreParMatrix &M, &K;
-   const Vector &b;
-   HypreSmoother M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-   double dt;
-   ParBilinearForm &pbf;
-   Vector &M_rowsums;
-
-public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                const Vector &_b, ParBilinearForm &_pbf, Vector &M_rs);
-
-   void SetTimeStep(double _dt) { dt = _dt; }
-   void SetK(HypreParMatrix &_K) { K = _K; }
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution() { }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   problem = 0;
-   optimizer_type = 1;
-   const char *mesh_file = "../../data/periodic-hexagon.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 0;
-   int order = 3;
-   int ode_solver_type = 3;
-   double t_final = 1.0;
-   double dt = 0.01;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&optimizer_type, "-opt", "--optimizer",
-                  "Nonlinear optimizer: 1 - SLBQP,\n\t"
-                  "                     2 - HIOP.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0) { args.PrintUsage(cout); }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0) { args.PrintOptions(cout); }
-
-   // 3. Read the serial mesh from the given mesh file on all processors. We can
-   //    handle geometrically periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Define the ODE solver used for time integration. Several explicit
-   //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      default:
-         if (myid == 0)
-         {
-            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         }
-         delete mesh;
-         MPI_Finalize();
-         return 3;
-   }
-
-   // 5. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter. If the mesh is of NURBS type, we convert it
-   //    to a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 7. Define the parallel discontinuous DG finite element space on the
-   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim, BasisType::Positive);
-   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
-
-   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << global_vSize << endl;
-   }
-
-   // 8. Set up and assemble the parallel bilinear and linear forms (and the
-   //    parallel hypre matrices) corresponding to the DG discretization. The
-   //    DGTraceIntegrator involves integrals over mesh interior faces.
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   ParBilinearForm *m = new ParBilinearForm(fes);
-   m->AddDomainIntegrator(new MassIntegrator);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-   k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-
-   ParLinearForm *b = new ParLinearForm(fes);
-   b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
-
-   m->Assemble();
-   m->Finalize();
-   int skip_zeros = 0;
-   k->Assemble(skip_zeros);
-   k->Finalize(skip_zeros);
-   b->Assemble();
-
-   HypreParMatrix *M = m->ParallelAssemble();
-   HypreParMatrix *K = k->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-
-   // 9. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   ParGridFunction *u = new ParGridFunction(fes);
-   u->ProjectCoefficient(u0);
-   HypreParVector *U = u->GetTrueDofs();
-
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "ex9-mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex9-init." << setfill('0') << setw(6) << myid;
-      ofstream omesh(mesh_name.str().c_str());
-      omesh.precision(precision);
-      pmesh->Print(omesh);
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u->Save(osol);
-   }
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9-Parallel", pmesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example9-Parallel", pmesh);
-         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
-      }
-      dc->RegisterField("solution", u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (myid == 0)
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-         visualization = false;
-         if (myid == 0)
-         {
-            cout << "GLVis visualization disabled.\n";
-         }
-      }
-      else
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout.precision(precision);
-         sout << "solution\n" << *pmesh << *u;
-         sout << "pause\n";
-         sout << flush;
-         if (myid == 0)
-            cout << "GLVis visualization paused."
-                 << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   Vector M_rowsums(m->Size());
-   m->SpMat().GetRowSums(M_rowsums);
-
-   // 10. Define the time-dependent evolution operator describing the ODE
-   //     right-hand side, and perform time-integration (looping over the time
-   //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*M, *K, *B, *k, M_rowsums);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   *u = *U;
-
-   // Compute initial volume.
-   const double vol0_loc = M_rowsums * (*u);
-   double vol0;
-   MPI_Allreduce(&vol0_loc, &vol0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-      adv.SetTimeStep(dt_real);
-      ode_solver->Step(*U, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         if (myid == 0)
-         {
-            cout << "time step: " << ti << ", time: " << t << endl;
-         }
-
-         // 11. Extract the parallel grid function corresponding to the finite
-         //     element approximation U (the local solution on each processor).
-         *u = *U;
-
-         if (visualization)
-         {
-            sout << "parallel " << num_procs << " " << myid << "\n";
-            sout << "solution\n" << *pmesh << *u << flush;
-         }
-
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-      }
-   }
-
-   // Print the error vs exact solution.
-   const double max_error = u->ComputeMaxError(u0),
-                l1_error  = u->ComputeL1Error(u0),
-                l2_error  = u->ComputeL2Error(u0);
-   if (myid == 0)
-   {
-      std::cout << "Linf error = " << max_error << endl
-                << "L1   error = " << l1_error << endl
-                << "L2   error = " << l2_error << endl;
-   }
-
-   // Print error in volume.
-   const double vol_loc = M_rowsums * (*u);
-   double vol;
-   MPI_Allreduce(&vol_loc, &vol, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-   if (myid == 0)
-   {
-      std::cout << "Vol  error = " << vol - vol0 << endl;
-   }
-
-   // 12. Save the final solution in parallel. This output can be viewed later
-   //     using GLVis: "glvis -np <np> -m ex9-mesh -g ex9-final".
-   {
-      *u = *U;
-      ostringstream sol_name;
-      sol_name << "ex9-final." << setfill('0') << setw(6) << myid;
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u->Save(osol);
-   }
-
-   // 13. Free the used memory.
-   delete U;
-   delete u;
-   delete B;
-   delete b;
-   delete K;
-   delete k;
-   delete M;
-   delete m;
-   delete fes;
-   delete pmesh;
-   delete ode_solver;
-   delete dc;
-
-   MPI_Finalize();
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                           const Vector &_b, ParBilinearForm &_pbf,
-                           Vector &M_rs)
-   : TimeDependentOperator(_M.Height()),
-     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height()),
-     pbf(_pbf), M_rowsums(M_rs)
-{
-   M_prec.SetType(HypreSmoother::Jacobi);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // Get values on the ldofs.
-   ParFiniteElementSpace *pfes = pbf.ParFESpace();
-   ParGridFunction x_gf(pfes);
-   pfes->GetProlongationMatrix()->Mult(x, x_gf);
-
-   // Compute bounds y_min, y_max for y from from x on the ldofs.
-   const int ldofs = x_gf.Size();
-   Vector y_min(ldofs), y_max(ldofs);
-   x_gf.ExchangeFaceNbrData();
-   Vector &x_nd = x_gf.FaceNbrData();
-   const int *In = pbf.SpMat().GetI(), *Jn = pbf.SpMat().GetJ();
-   for (int i = 0, k = 0; i < ldofs; i++)
-   {
-      double x_i_min = +std::numeric_limits<double>::infinity();
-      double x_i_max = -std::numeric_limits<double>::infinity();
-      for (int end = In[i+1]; k < end; k++)
-      {
-         const int j = Jn[k];
-         const double x_j = (j < ldofs) ? x(j): x_nd(j-ldofs);
-
-         if (x_j > x_i_max) { x_i_max = x_j; }
-         if (x_j < x_i_min) { x_i_min = x_j; }
-      }
-      y_min(i) = x_i_min;
-      y_max(i) = x_i_max;
-   }
-   for (int i = 0; i < ldofs; i++)
-   {
-      y_min(i) = (y_min(i) - x_gf(i) ) / dt;
-      y_max(i) = (y_max(i) - x_gf(i) ) / dt;
-   }
-   Vector y_min_tdofs(y.Size()), y_max_tdofs(y.Size());
-   // Move the bounds to the tdofs.
-   pfes->GetRestrictionMatrix()->Mult(y_min, y_min_tdofs);
-   pfes->GetRestrictionMatrix()->Mult(y_max, y_max_tdofs);
-
-   // Compute the high-order solution y = M^{-1} (K x + b) on the tdofs.
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-
-   // The solution y is an increment; it should not introduce new mass.
-   const double mass_y = 0.0;
-
-   // Perform optimization on the tdofs.
-   Vector y_out(y.Size());
-   const int max_iter = 500;
-   const double rtol = 1.e-7;
-   double atol = 1.e-7;
-
-   OptimizationSolver* optsolver = NULL;
-   if (optimizer_type == 2)
-   {
-#ifdef MFEM_USE_HIOP
-      HiopNlpOptimizer *tmp_opt_ptr = new HiopNlpOptimizer(MPI_COMM_WORLD);
-      optsolver = tmp_opt_ptr;
-#else
-      MFEM_ABORT("MFEM is not built with HiOp support!");
-#endif
-   }
-   else
-   {
-      SLBQPOptimizer *slbqp = new SLBQPOptimizer(MPI_COMM_WORLD);
-      slbqp->SetBounds(y_min_tdofs, y_max_tdofs);
-      slbqp->SetLinearConstraint(M_rowsums, mass_y);
-      atol = 1.e-15;
-      optsolver = slbqp;
-   }
-
-   OptimizedTransportProblem ot_prob(*pfes, y, M_rowsums, mass_y,
-                                     y_min_tdofs, y_max_tdofs);
-   optsolver->SetOptimizationProblem(ot_prob);
-
-   optsolver->SetMaxIter(max_iter);
-   optsolver->SetAbsTol(atol);
-   optsolver->SetRelTol(rtol);
-   optsolver->SetPrintLevel(0);
-   optsolver->Mult(y, y_out);
-
-   y = y_out;
-
-   delete optsolver;
-}
-
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = (invert_velocity) ? -1.0 : 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return (X(0) > -0.15 && X(0) < 0.15) ? 1.0 : 0.0;
-            //return exp(-40.*pow(X(0)-0.0,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}
@@ -1,66 +0,0 @@
-# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
-# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
-# See file COPYRIGHT for details.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability see http://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License (as published by the Free
-# Software Foundation) version 2.1 dated February 1999.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/hiop/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex9
-PAR_EXAMPLES = ex9p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_HIOP),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with HIOP)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.* Example9*
@@ -22,9 +22,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21 ex22
+  ex18 ex19 ex20 ex22
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
- ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p
+ ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex22p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -41,13 +41,6 @@ endif
 ifeq ($(MFEM_USE_PUMI),YES)
   SUBDIRS += pumi
 endif
-ifeq ($(MFEM_USE_HIOP),YES)
-   SUBDIRS += hiop
-endif
-ifeq ($(MFEM_USE_GINKGO),YES)
-   SUBDIRS += ginkgo
-endif
-
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
@@ -124,12 +117,12 @@ clean-build:

 clean-exec:
 	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
-	@rm -rf Example5* Example9* Example15* Example16* PVExample*
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
+	@rm -rf Example5* Example9* Example15* Example16*
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_*
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*
 	@rm -f vortex-mesh.* vortex.mesh vortex-?-init.* vortex-?-final.*
 	@rm -f deformation.* pressure.*
 	@rm -f ex20.dat ex20p_?????.dat gnuplot_ex20.inp gnuplot_ex20p.inp
-	@rm -f ex21*.mesh ex21*.sol ex21p_*.*
+	@rm -f ex22*.mesh ex22*.sol ex22p_*.*
@@ -27,11 +27,8 @@
 //               method HyperelasticOperator::ImplicitSolve is the only
 //               requirement for high-order implicit (SDIRK) time integration.
 //               If using PETSc to solve the nonlinear problem, use the option
-//               files provided (see rc_ex10p, rc_ex10p_mf, rc_ex10p_mfop) that
-//               customize the Newton-Krylov method.
-//               When option --jfnk is used, PETSc will use a Jacobian-free
-//               Newton-Krylov method, using a user-defined preconditioner
-//               constructed with the PetscPreconditionerFactory class.
+//               file provided (rc_ex10p) that customizes the
+//               Newton-Krylov method.
 //
 //               We recommend viewing examples 2 and 9 before viewing this
 //               example.
@@ -89,15 +86,12 @@ protected:
   Solver *J_solver;
   /// Preconditioner for the Jacobian solve in the Newton method
   Solver *J_prec;
-   /// Preconditioner factory for JFNK
-   PetscPreconditionerFactory *J_factory;

   mutable Vector z; // auxiliary vector

 public:
   HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
-                        double visc, double mu, double K,
-                        bool use_petsc, bool petsc_use_jfnk);
+                        double visc, double mu, double K, bool use_petsc);

   /// Compute the right-hand side of the ODE system.
   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
@@ -142,21 +136,8 @@ public:
   virtual Operator &GetGradient(const Vector &k) const;

   virtual ~ReducedSystemOperator();
-
 };

-/** Auxiliary class to provide preconditioners for matrix-free methods */
-class PreconditionerFactory : public PetscPreconditionerFactory
-{
-private:
-   // const ReducedSystemOperator& op; // unused for now (generates warning)
-
-public:
-   PreconditionerFactory(const ReducedSystemOperator& op_, const string& name_)
-      : PetscPreconditionerFactory(name_) /* , op(op_) */ {}
-   virtual mfem::Solver* NewPreconditioner(const mfem::OperatorHandle&);
-   virtual ~PreconditionerFactory() {}
-};

 /** Function representing the elastic energy density for the given hyperelastic
    model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -206,7 +187,6 @@ int main(int argc, char *argv[])
   int vis_steps = 1;
   bool use_petsc = true;
   const char *petscrc_file = "";
-   bool petsc_use_jfnk = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -241,9 +221,6 @@ int main(int argc, char *argv[])
                  "Use or not PETSc to solve the nonlinear system.");
   args.AddOption(&petscrc_file, "-petscopts", "--petscopts",
                  "PetscOptions file to use.");
-   args.AddOption(&petsc_use_jfnk, "-jfnk", "--jfnk", "-no-jfnk",
-                  "--no-jfnk",
-                  "Use JFNK with user-defined preconditioner factory.");
   args.Parse();
   if (!args.Good())
   {
@@ -367,8 +344,7 @@ int main(int argc, char *argv[])
   // 9. Initialize the hyperelastic operator, the GLVis visualization and print
   //    the initial energies.
   HyperelasticOperator *oper = new HyperelasticOperator(fespace, ess_bdr, visc,
-                                                         mu, K, use_petsc,
-                                                         petsc_use_jfnk);
+                                                         mu, K, use_petsc);

   socketstream vis_v, vis_w;
   if (visualization)
@@ -544,7 +520,7 @@ Operator &ReducedSystemOperator::GetGradient(const Vector &k) const
   add(*v, dt, k, w);
   add(*x, dt, w, z);
   localJ->Add(dt*dt, H->GetLocalGradient(z));
-   // if we are using PETSc, the HypreParCSR Jacobian will be converted to
+   // if we are using PETSc, the HypreParCSR jacobian will be converted to
   // PETSc's AIJ on the fly
   Jacobian = M->ParallelAssemble(localJ);
   delete localJ;
@@ -561,8 +537,7 @@ ReducedSystemOperator::~ReducedSystemOperator()

 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
                                           Array<int> &ess_bdr, double visc,
-                                           double mu, double K, bool use_petsc,
-                                           bool use_petsc_factory)
+                                           double mu, double K, bool use_petsc)
   : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
     M(&fespace), S(&fespace), H(&fespace),
     viscosity(visc), M_solver(f.GetComm()),
@@ -615,8 +590,6 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
      J_minres->SetPreconditioner(*J_prec);
      J_solver = J_minres;

-      J_factory = NULL;
-
      newton_solver.iterative_mode = false;
      newton_solver.SetSolver(*J_solver);
      newton_solver.SetOperator(*reduced_oper);
@@ -627,20 +600,12 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
   }
   else
   {
-      // if using PETSc, we create the same solver (Newton + MINRES + Jacobi)
+      // if using PETSc, we create the same solver (NEWTON+MINRES+Jacobi)
      // by command line options (see rc_ex10p)
      J_solver = NULL;
      J_prec = NULL;
-      J_factory = NULL;
      pnewton_solver = new PetscNonlinearSolver(f.GetComm(),
                                                *reduced_oper);
-
-      // we can setup a factory to construct a "physics-based" preconditioner
-      if (use_petsc_factory)
-      {
-         J_factory = new PreconditionerFactory(*reduced_oper, "JFNK preconditioner");
-         pnewton_solver->SetPreconditionerFactory(J_factory);
-      }
      pnewton_solver->SetPrintLevel(1); // print Newton iterations
      pnewton_solver->SetRelTol(rel_tol);
      pnewton_solver->SetAbsTol(0.0);
@@ -726,26 +691,12 @@ HyperelasticOperator::~HyperelasticOperator()
 {
   delete J_solver;
   delete J_prec;
-   delete J_factory;
   delete reduced_oper;
   delete model;
   delete Mmat;
   delete pnewton_solver;
 }

-// This method gets called every time we need a preconditioner "oh"
-// contains the PetscParMatrix that wraps the operator constructed in
-// the GetGradient() method (see also PetscSolver::SetJacobianType()).
-// In this example, we just return a customizable PetscPreconditioner
-// using that matrix. However, the OperatorHandle argument can be
-// ignored, and any "physics-based" solver can be constructed since we
-// have access to the HyperElasticOperator class.
-Solver* PreconditionerFactory::NewPreconditioner(const mfem::OperatorHandle& oh)
-{
-   PetscParMatrix *pP;
-   oh.Get(pP);
-   return new PetscPreconditioner(*pP,"jfnk_");
-}

 double ElasticEnergyCoefficient::Eval(ElementTransformation &T,
                                      const IntegrationPoint &ip)
@@ -759,8 +710,8 @@ double ElasticEnergyCoefficient::Eval(ElementTransformation &T,

 void InitialDeformation(const Vector &x, Vector &y)
 {
-   // set the initial configuration to be the same as the reference,
-   // stress free, configuration
+   // set the initial configuration to be the same as the reference, stress
+   // free, configuration
   y = x;
 }

@@ -84,10 +84,6 @@ EX9_E_ARGS       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts r
 EX9_ES_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
 EX9_IS_ARGS      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
 EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
-EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
-EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
-EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
@@ -111,9 +107,6 @@ ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_IS_ARGS))
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -1,5 +0,0 @@
-# matrix-free Jacobian action, preconditioner constructed using PetscPreconditionerFactory
-snes_monitor
-snes_mf_operator
-ksp_type minres
-jfnk_pc_type jacobi
@@ -1,4 +0,0 @@
-# matrix free -> no preconditioner
-snes_monitor
-snes_mf
-ksp_type minres
@@ -1,5 +0,0 @@
-# matrix-free Jacobian action, preconditioner constructed from the matrix obtained by the GetGradient() method
-snes_monitor
-snes_mf_operator
-ksp_type minres
-pc_type jacobi
@@ -42,12 +42,12 @@ add_mfem_examples(SUNDIALS_EXAMPLES_SRCS ${PFX} "" test_sundials)
 #   ctest -R sundials

 # Command line options for the tests.
-# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 7)
+# Example 9: test explicit CVODE time stepping
+set(EX9_COMMON_OPTS -m ../../data/periodic-hexagon.mesh -p 0 -s 11)
 set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 -dt 0.0018 -vs 25)
 set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS} -rp 1 -dt 0.0009 -vs 50)
-# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
-set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10)
+# Example 10: test implicit CVODE time stepping
+set(EX10_COMMON_OPTS -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -vs 10)
 set(EX10_TEST_OPTS ${EX10_COMMON_OPTS}  -r  2)
 set(EX10P_TEST_OPTS ${EX10_COMMON_OPTS} -rp 1)
 # Example 16: use the default options
@@ -4,16 +4,16 @@
 // Compile with: make ex10
 //
 // Sample runs:
-//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10
-//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 16 -dt 0.3  -vs 5
-//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 12 -dt 0.2  -vs 5
+//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  5 -dt 0.15 -vs 10
+//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  7 -dt 0.3  -vs 5
+//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  5 -dt 0.2  -vs 5
 //    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s  2 -dt 3 -nls kinsol
 //    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s  2 -dt 3 -nls kinsol
 //    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s  2 -dt 3 -nls kinsol
-//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 14 -dt 0.15 -vs 10
-//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 17 -dt 0.01 -vs 30
-//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 14 -dt 0.15 -vs 10
-//    ex10 -m ../../data/beam-quad-amr.mesh -r 2 -o 2 -s 12 -dt 0.15 -vs 10
+//    ex10 -m ../../data/beam-quad.mesh -r 2 -o 2 -s 15 -dt 5e-3 -vs 60
+//    ex10 -m ../../data/beam-tri.mesh  -r 2 -o 2 -s 16 -dt 0.01 -vs 30
+//    ex10 -m ../../data/beam-hex.mesh  -r 1 -o 2 -s 15 -dt 0.01 -vs 30
+//    ex10 -m ../../data/beam-quad-amr.mesh -r 2 -o 2 -s 5 -dt 0.15 -vs 10
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -53,6 +53,7 @@ using namespace std;
 using namespace mfem;

 class ReducedSystemOperator;
+class SundialsJacSolver;

 /** After spatial discretization, the hyperelastic model can be written as a
 *  system of ODEs:
@@ -91,17 +92,12 @@ protected:

   mutable Vector z; // auxiliary vector

-   SparseMatrix *grad_H;
-   SparseMatrix *Jacobian;
-
-   double saved_gamma; // saved gamma value from implicit setup
-
 public:
   /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
   enum NonlinearSolverType
   {
      NEWTON = 0, ///< Use MFEM's plain NewtonSolver
-      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KinSolver)
   };

   HyperelasticOperator(FiniteElementSpace &f, Array<int> &ess_bdr,
@@ -110,41 +106,15 @@ public:

   /// Compute the right-hand side of the ODE system.
   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
-
   /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-
-   /// Custom Jacobian system solver for the SUNDIALS time integrators.
-   /** For the ODE system represented by HyperelasticOperator
-
-          M dv/dt = -(H(x) + S*v)
-          dx/dt = v,
-
-       this class facilitates the solution of linear systems of the form
-
-           (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
-                - γ yv +    yx =   bx
-
-       for given bv, bx, x, and γ = GetTimeStep(). */
-
-   /** Linear solve applicable to the SUNDIALS format.
-       Solves (Mass - dt J) y = Mass b, where in our case:
-       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
-              | 0  I |      |  I     0    |      | x_hat |      | b_x |
-       The result replaces the rhs b.
-       We substitute x_hat = b_x + dt v_hat and solve
-       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
-
-   /** Setup the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &y, const Vector &fy,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Connect the Jacobian linear system solver (SundialsJacSolver) used by
+       SUNDIALS' CVODE and ARKODE time integrators to the internal objects
+       created by HyperelasticOperator. This method is called by the InitSystem
+       method of SundialsJacSolver. */
+   void InitSundialsJacSolver(SundialsJacSolver &sjsolv);

   double ElasticEnergy(const Vector &x) const;
   double KineticEnergy(const Vector &v) const;
@@ -182,6 +152,53 @@ public:
   virtual ~ReducedSystemOperator();
 };

+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by HyperelasticOperator
+
+        M dv/dt = -(H(x) + S*v)
+          dx/dt = v,
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
+             - γ yv +    yx =   bx
+
+    for given bv, bx, x, and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   BilinearForm *M, *S;
+   NonlinearForm *H;
+   SparseMatrix *grad_H, *Jacobian;
+   Solver *J_solver;
+
+public:
+   SundialsJacSolver()
+      : M(), S(), H(), grad_H(), Jacobian(), J_solver() { }
+
+   /// Connect the solver to the objects created inside HyperelasticOperator.
+   void SetOperators(BilinearForm &M_, BilinearForm &S_,
+                     NonlinearForm &H_, Solver &solver)
+   {
+      M = &M_; S = &S_; H = &H_; J_solver = &solver;
+   }
+
+   /** Linear solve applicable to the SUNDIALS format.
+       Solves (Mass - dt J) y = Mass b, where in our case:
+       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
+              | 0  I |      |  I     0    |      | x_hat |      | b_x |
+       The result replaces the rhs b.
+       We substitute x_hat = b_x + dt v_hat and solve
+       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+

 /** Function representing the elastic energy density for the given hyperelastic
    model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -226,12 +243,6 @@ int main(int argc, char *argv[])

   // Relative and absolute tolerances for CVODE and ARKODE.
   const double reltol = 1e-1, abstol = 1e-1;
-   // Since this example uses the loose tolerances defined above, it is
-   // necessary to lower the linear solver tolerance for CVODE which is relative
-   // to the above tolerances.
-   const double cvode_eps_lin = 1e-4;
-   // Similarly, the nonlinear tolerance for ARKODE needs to be tightened.
-   const double arkode_eps_nonlin = 1e-6;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -241,24 +252,15 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Backward Euler,\n\t"
-                  "2  - SDIRK2, L-stable\n\t"
-                  "3  - SDIRK3, L-stable\n\t"
-                  "4  - Implicit Midpoint,\n\t"
-                  "5  - SDIRK2, A-stable,\n\t"
-                  "6  - SDIRK3, A-stable,\n\t"
-                  "7  - Forward Euler,\n\t"
-                  "8  - RK2,\n\t"
-                  "9  - RK3 SSP,\n\t"
-                  "10 - RK4,\n\t"
-                  "11 - CVODE implicit BDF, approximate Jacobian,\n\t"
-                  "12 - CVODE implicit BDF, specified Jacobian,\n\t"
-                  "13 - CVODE implicit ADAMS, approximate Jacobian,\n\t"
-                  "14 - CVODE implicit ADAMS, specified Jacobian,\n\t"
-                  "15 - ARKODE implicit, approximate Jacobian,\n\t"
-                  "16 - ARKODE implicit, specified Jacobian,\n\t"
-                  "17 - ARKODE explicit, 4th order.");
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            4 - CVODE implicit, approximate Jacobian,\n\t"
+                  "            5 - CVODE implicit, specified Jacobian,\n\t"
+                  "            6 - ARKODE implicit, approximate Jacobian,\n\t"
+                  "            7 - ARKODE implicit, specified Jacobian,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4,\n\t"
+                  "            15 - CVODE (adaptive order) explicit,\n\t"
+                  "            16 - ARKODE default (4th order) explicit.");
   args.AddOption(&nls, "-nls", "--nonlinear-solver",
                  "Nonlinear systems solver: "
                  "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
@@ -285,19 +287,72 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 17)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 1;
-   }
-
   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 3. Setup the nonlinear solver
+   // 3. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
+   ODESolver *ode_solver;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver *sjsolver = NULL;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1: ode_solver = new BackwardEulerSolver; break;
+      case 2: ode_solver = new SDIRK23Solver(2); break;
+      case 3: ode_solver = new SDIRK33Solver; break;
+      case 4:
+      case 5:
+         cvode = new CVODESolver(CV_BDF, CV_NEWTON);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         if (ode_solver_type == 5)
+         {
+            sjsolver = new SundialsJacSolver;
+            cvode->SetLinearSolver(*sjsolver);
+         }
+         ode_solver = cvode; break;
+      case 6:
+      case 7:
+         arkode = new ARKODESolver(ARKODESolver::IMPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 7)
+         {
+            // Custom Jacobian inversion.
+            sjsolver = new SundialsJacSolver;
+            arkode->SetLinearSolver(*sjsolver);
+         }
+         ode_solver = arkode; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15:
+         cvode = new CVODESolver(CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 16:
+         arkode = new ARKODESolver(ARKODESolver::IMPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
   map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
   nls_map["newton"] = HyperelasticOperator::NEWTON;
   nls_map["kinsol"] = HyperelasticOperator::KINSOL;
@@ -384,82 +439,11 @@ int main(int argc, char *argv[])
   cout << "initial kinetic energy (KE) = " << ke0 << endl;
   cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;

-   // 8. Define the ODE solver used for time integration. Several implicit
-   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //    explicit Runge-Kutta methods are available.
   double t = 0.0;
   oper.SetTime(t);
+   ode_solver->Init(oper);

-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 4:  ode_solver = new ImplicitMidpointSolver; break;
-      case 5:  ode_solver = new SDIRK23Solver; break;
-      case 6:  ode_solver = new SDIRK34Solver; break;
-      // Explicit methods
-      case 7:  ode_solver = new ForwardEulerSolver; break;
-      case 8:  ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 9:  ode_solver = new RK3SSPSolver; break;
-      case 10: ode_solver = new RK4Solver; break;
-      // CVODE BDF
-      case 11:
-      case 12:
-         cvode = new CVODESolver(CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // CVODE Adams
-      case 13:
-      case 14:
-         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 13)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // ARKStep Implicit methods
-      case 15:
-      case 16:
-         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 15)
-         {
-            arkode->UseSundialsLinearSolver();
-         }
-         ode_solver = arkode; break;
-      // ARKStep Explicit methods
-      case 17:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(oper); }
-
-   // 9. Perform time-integration (looping over the time iterations, ti, with a
+   // 8. Perform time-integration (looping over the time iterations, ti, with a
   //    time-step dt).
   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -494,7 +478,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 10. Save the displaced mesh, the velocity and elastic energy.
+   // 9. Save the displaced mesh, the velocity and elastic energy.
   {
      v.SetFromTrueVector(); x.SetFromTrueVector();
      GridFunction *nodes = &x;
@@ -513,8 +497,9 @@ int main(int argc, char *argv[])
      w.Save(ee_ofs);
   }

-   // 11. Free the used memory.
+   // 10. Free the used memory.
   delete ode_solver;
+   delete sjsolver;
   delete mesh;

   return 0;
@@ -594,14 +579,81 @@ ReducedSystemOperator::~ReducedSystemOperator()
 }


+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+   HyperelasticOperator *he_oper;
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   he_oper = dynamic_cast<HyperelasticOperator*>(td_oper);
+   MFEM_VERIFY(he_oper, "operator is not HyperelasticOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // he_oper = static_cast<HyperelasticOperator*>(td_oper);
+
+   he_oper->InitSundialsJacSolver(*this);
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   int sc = y_pred.Size() / 2;
+   const Vector x(y_pred.GetData() + sc, sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // J = M + dt*(S + dt*grad(H))
+   delete Jacobian;
+   Jacobian = Add(1.0, M->SpMat(), dt, S->SpMat());
+   grad_H = dynamic_cast<SparseMatrix *>(&H->GetGradient(x));
+   Jacobian->Add(dt * dt, *grad_H);
+
+   J_solver->SetOperator(*Jacobian);
+
+   jac_cur = 1;
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   int sc = b.Size() / 2;
+   // Vector x(y_cur.GetData() + sc, sc);
+   Vector b_v(b.GetData() +  0, sc);
+   Vector b_x(b.GetData() + sc, sc);
+   Vector rhs(sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // rhs = M b_v - dt*grad(H) b_x
+   grad_H->Mult(b_x, rhs);
+   rhs *= -dt;
+   M->AddMult(b_v, rhs);
+
+   J_solver->iterative_mode = false;
+   J_solver->Mult(rhs, b_v);
+
+   b_x.Add(dt, b_v);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   delete Jacobian;
+   return 0;
+}
+
+
 HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,
                                           Array<int> &ess_bdr, double visc,
                                           double mu, double K,
                                           NonlinearSolverType nls_type)
   : TimeDependentOperator(2*f.GetTrueVSize(), 0.0), fespace(f),
     M(&fespace), S(&fespace), H(&fespace),
-     viscosity(visc), z(height/2),
-     grad_H(NULL), Jacobian(NULL)
+     viscosity(visc), z(height/2)
 {
   const double rel_tol = 1e-8;
   const int skip_zero_entries = 0;
@@ -650,24 +702,23 @@ HyperelasticOperator::HyperelasticOperator(FiniteElementSpace &f,

   if (nls_type == KINSOL)
   {
-      KINSolver *kinsolver = new KINSolver(KIN_NONE, true);
+      KinSolver *kinsolver = new KinSolver(KIN_NONE, true);
+      kinsolver->SetMaxSetupCalls(4);
      newton_solver = kinsolver;
-      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(200);
      newton_solver->SetRelTol(rel_tol);
      newton_solver->SetPrintLevel(0);
-      kinsolver->SetMaxSetupCalls(4);
   }
   else
   {
      newton_solver = new NewtonSolver();
-      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(10);
      newton_solver->SetRelTol(rel_tol);
      newton_solver->SetPrintLevel(-1);
   }
   newton_solver->SetSolver(*J_solver);
   newton_solver->iterative_mode = false;
+   newton_solver->SetOperator(*reduced_oper);
 }

 void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const
@@ -717,53 +768,9 @@ void HyperelasticOperator::ImplicitSolve(const double dt,
   add(v, dt, dv_dt, dx_dt);
 }

-int HyperelasticOperator::SUNImplicitSetup(const Vector &y,
-                                           const Vector &fy, int jok, int *jcur,
-                                           double gamma)
+void HyperelasticOperator::InitSundialsJacSolver(SundialsJacSolver &sjsolv)
 {
-   int sc = y.Size() / 2;
-   const Vector x(y.GetData() + sc, sc);
-
-   // J = M + dt*(S + dt*grad(H))
-   if (Jacobian) { delete Jacobian; }
-   Jacobian = Add(1.0, M.SpMat(), gamma, S.SpMat());
-   grad_H = dynamic_cast<SparseMatrix *>(&H.GetGradient(x));
-   Jacobian->Add(gamma * gamma, *grad_H);
-
-   // Set Jacobian solve operator
-   J_solver->SetOperator(*Jacobian);
-
-   // Indicate that the Jacobian was updated
-   *jcur = 1;
-
-   // Save gamma for use in solve
-   saved_gamma = gamma;
-
-   // Return success
-   return 0;
-}
-
-int HyperelasticOperator::SUNImplicitSolve(const Vector &b, Vector &x,
-                                           double tol)
-{
-   int sc = b.Size() / 2;
-   Vector b_v(b.GetData() +  0, sc);
-   Vector b_x(b.GetData() + sc, sc);
-   Vector x_v(x.GetData() +  0, sc);
-   Vector x_x(x.GetData() + sc, sc);
-   Vector rhs(sc);
-
-   // rhs = M b_v - dt*grad(H) b_x
-   grad_H->Mult(b_x, rhs);
-   rhs *= -saved_gamma;
-   M.AddMult(b_v, rhs);
-
-   J_solver->iterative_mode = false;
-   J_solver->Mult(rhs, x_v);
-
-   add(b_x, saved_gamma, x_v, x_x);
-
-   return 0;
+   sjsolv.SetOperators(M, S, H, *J_solver);
 }

 double HyperelasticOperator::ElasticEnergy(const Vector &x) const
@@ -785,7 +792,6 @@ void HyperelasticOperator::GetElasticEnergyDensity(

 HyperelasticOperator::~HyperelasticOperator()
 {
-   delete Jacobian;
   delete newton_solver;
   delete J_solver;
   delete J_prec;
@@ -4,16 +4,16 @@
 // Compile with: make ex10p
 //
 // Sample runs:
-//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 12 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 16 -dt 0.25 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 12 -dt 0.15 -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s  5 -dt 0.15 -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s  7 -dt 0.25  -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s  5 -dt 0.15  -vs 10
 //    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s  2 -dt 3 -nls kinsol
 //    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rs 1 -o 2 -s  2 -dt 3 -nls kinsol
-//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 14 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 17 -dt 5e-3 -vs 60
-//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 14 -dt 0.15 -vs 10
-//    mpirun -np 4 ex10p -m ../../data/beam-quad-amr.mesh -rp 1 -o 2 -s 12 -dt 0.15 -vs 10
+//    mpirun -np 4 ex10p -m ../../data/beam-quad.mesh -rp 1 -o 2 -s 15 -dt 3e-3 -vs 120
+//    mpirun -np 4 ex10p -m ../../data/beam-tri.mesh  -rp 1 -o 2 -s 16 -dt 5e-3 -vs 60
+//    mpirun -np 4 ex10p -m ../../data/beam-hex.mesh  -rp 0 -o 2 -s 15 -dt 5e-3 -vs 60
+//    mpirun -np 4 ex10p -m ../../data/beam-quad-amr.mesh -rp 1 -o 2 -s 5 -dt 0.15 -vs 10
 //
 // Description:  This examples solves a time dependent nonlinear elasticity
 //               problem of the form dv/dt = H(x) + S v, dx/dt = v, where H is a
@@ -53,6 +53,7 @@ using namespace std;
 using namespace mfem;

 class ReducedSystemOperator;
+class SundialsJacSolver;

 /** After spatial discretization, the hyperelastic model can be written as a
 *  system of ODEs:
@@ -93,17 +94,12 @@ protected:

   mutable Vector z; // auxiliary vector

-   const SparseMatrix *local_grad_H;
-   HypreParMatrix *Jacobian;
-
-   double saved_gamma; // saved gamma value from implicit setup
-
 public:
   /// Solver type to use in the ImplicitSolve() method, used by SDIRK methods.
   enum NonlinearSolverType
   {
      NEWTON = 0, ///< Use MFEM's plain NewtonSolver
-      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KINSolver)
+      KINSOL = 1  ///< Use SUNDIALS' KINSOL (through MFEM's class KinSolver)
   };

   HyperelasticOperator(ParFiniteElementSpace &f, Array<int> &ess_bdr,
@@ -112,41 +108,15 @@ public:

   /// Compute the right-hand side of the ODE system.
   virtual void Mult(const Vector &vx, Vector &dvx_dt) const;
-
   /** Solve the Backward-Euler equation: k = f(x + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-
-   /// Custom Jacobian system solver for the SUNDIALS time integrators.
-   /** For the ODE system represented by HyperelasticOperator
-
-          M dv/dt = -(H(x) + S*v)
-          dx/dt = v,
-
-       this class facilitates the solution of linear systems of the form
-
-           (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
-                - γ yv +    yx =   bx
-
-       for given bv, bx, x, and γ = GetTimeStep(). */
-
-   /** Linear solve applicable to the SUNDIALS format.
-       Solves (Mass - dt J) y = Mass b, where in our case:
-       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
-              | 0  I |      |  I     0    |      | x_hat |      | b_x |
-       The result replaces the rhs b.
-       We substitute x_hat = b_x + dt v_hat and solve
-       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
-
-   /** Setup the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &y, const Vector &fy,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the linear system. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Connect the Jacobian linear system solver (SundialsJacSolver) used by
+       SUNDIALS' CVODE and ARKODE time integrators to the internal objects
+       created by HyperelasticOperator. This method is called by the InitSystem
+       method of SundialsJacSolver. */
+   void InitSundialsJacSolver(SundialsJacSolver &sjsolv);

   double ElasticEnergy(const ParGridFunction &x) const;
   double KineticEnergy(const ParGridFunction &v) const;
@@ -187,6 +157,57 @@ public:
   virtual ~ReducedSystemOperator();
 };

+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by HyperelasticOperator
+
+        M dv/dt = -(H(x) + S*v)
+          dx/dt = v,
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γS) yv + γJ yx = M bv,   J=(dH/dx)(x)
+             - γ yv +    yx =   bx
+
+    for given bv, bx, x, and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   ParBilinearForm *M, *S;
+   ParNonlinearForm *H;
+   const SparseMatrix *local_grad_H;
+   HypreParMatrix *Jacobian;
+   Solver *J_solver;
+   const Array<int> *ess_tdof_list;
+
+public:
+   SundialsJacSolver()
+      : M(), S(), H(), local_grad_H(), Jacobian(), J_solver() { }
+
+   /// Connect the solver to the objects created inside HyperelasticOperator.
+   void SetOperators(ParBilinearForm &M_, ParBilinearForm &S_,
+                     ParNonlinearForm &H_, Solver &solver,
+                     const Array<int> &ess_tdof_list_)
+   {
+      M = &M_; S = &S_; H = &H_; J_solver = &solver;
+      ess_tdof_list = &ess_tdof_list_;
+   }
+
+   /** Linear solve applicable to the SUNDIALS format.
+       Solves (Mass - dt J) y = Mass b, where in our case:
+       Mass = | M  0 |  J = | -S  -grad_H |  y = | v_hat |  b = | b_v |
+              | 0  I |      |  I     0    |      | x_hat |      | b_x |
+       The result replaces the rhs b.
+       We substitute x_hat = b_x + dt v_hat and solve
+       (M + dt S + dt^2 grad_H) v_hat = M b_v - dt grad_H b_x. */
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+

 /** Function representing the elastic energy density for the given hyperelastic
    model+deformation. Used in HyperelasticOperator::GetElasticEnergyDensity. */
@@ -238,12 +259,6 @@ int main(int argc, char *argv[])

   // Relative and absolute tolerances for CVODE and ARKODE.
   const double reltol = 1e-1, abstol = 1e-1;
-   // Since this example uses the loose tolerances defined above, it is
-   // necessary to lower the linear solver tolerance for CVODE which is relative
-   // to the above tolerances.
-   const double cvode_eps_lin = 1e-4;
-   // Similarly, the nonlinear tolerance for ARKODE needs to be tightened.
-   const double arkode_eps_nonlin = 1e-6;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -255,24 +270,15 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Backward Euler,\n\t"
-                  "2  - SDIRK2, L-stable\n\t"
-                  "3  - SDIRK3, L-stable\n\t"
-                  "4  - Implicit Midpoint,\n\t"
-                  "5  - SDIRK2, A-stable,\n\t"
-                  "6  - SDIRK3, A-stable,\n\t"
-                  "7  - Forward Euler,\n\t"
-                  "8  - RK2,\n\t"
-                  "9  - RK3 SSP,\n\t"
-                  "10 - RK4,\n\t"
-                  "11 - CVODE implicit BDF, approximate Jacobian,\n\t"
-                  "12 - CVODE implicit BDF, specified Jacobian,\n\t"
-                  "13 - CVODE implicit ADAMS, approximate Jacobian,\n\t"
-                  "14 - CVODE implicit ADAMS, specified Jacobian,\n\t"
-                  "15 - ARKODE implicit, approximate Jacobian,\n\t"
-                  "16 - ARKODE implicit, specified Jacobian,\n\t"
-                  "17 - ARKODE explicit, 4th order.");
+                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
+                  "            4 - CVODE implicit, approximate Jacobian,\n\t"
+                  "            5 - CVODE implicit, specified Jacobian,\n\t"
+                  "            6 - ARKODE implicit, approximate Jacobian,\n\t"
+                  "            7 - ARKODE implicit, specified Jacobian,\n\t"
+                  "            11 - Forward Euler, 12 - RK2,\n\t"
+                  "            13 - RK3 SSP, 14 - RK4,\n\t"
+                  "            15 - CVODE (adaptive order) explicit,\n\t"
+                  "            16 - ARKODE default (4th order) explicit.");
   args.AddOption(&nls, "-nls", "--nonlinear-solver",
                  "Nonlinear systems solver: "
                  "\"newton\" (plain Newton) or \"kinsol\" (KINSOL).");
@@ -306,24 +312,76 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 17)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 1;
-   }
-
   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Nonlinear solver
+   // 4. Define the ODE solver used for time integration. Several implicit
+   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
+   //    explicit Runge-Kutta methods are available.
+   ODESolver *ode_solver;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver *sjsolver = NULL;
+   switch (ode_solver_type)
+   {
+      // Implicit L-stable methods
+      case 1: ode_solver = new BackwardEulerSolver; break;
+      case 2: ode_solver = new SDIRK23Solver(2); break;
+      case 3: ode_solver = new SDIRK33Solver; break;
+      case 4:
+      case 5:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF, CV_NEWTON);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         if (ode_solver_type == 5)
+         {
+            sjsolver = new SundialsJacSolver;
+            cvode->SetLinearSolver(*sjsolver); // Custom Jacobian inversion.
+         }
+         ode_solver = cvode; break;
+      case 6:
+      case 7:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::IMPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 7)
+         {
+            sjsolver = new SundialsJacSolver;
+            arkode->SetLinearSolver(*sjsolver); // Custom Jacobian inversion.
+         }
+         ode_solver = arkode; break;
+      // Explicit methods
+      case 11: ode_solver = new ForwardEulerSolver; break;
+      case 12: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 13: ode_solver = new RK3SSPSolver; break;
+      case 14: ode_solver = new RK4Solver; break;
+      case 15:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 16:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Implicit A-stable methods (not L-stable)
+      case 22: ode_solver = new ImplicitMidpointSolver; break;
+      case 23: ode_solver = new SDIRK23Solver; break;
+      case 24: ode_solver = new SDIRK34Solver; break;
+      default:
+         if (myid == 0)
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         MPI_Finalize();
+         return 3;
+   }
+
   map<string,HyperelasticOperator::NonlinearSolverType> nls_map;
   nls_map["newton"] = HyperelasticOperator::NEWTON;
   nls_map["kinsol"] = HyperelasticOperator::KINSOL;
@@ -333,6 +391,7 @@ int main(int argc, char *argv[])
      {
         cout << "Unknown type of nonlinear solver: " << nls << endl;
      }
+      delete ode_solver;
      delete mesh;
      MPI_Finalize();
      return 4;
@@ -436,82 +495,11 @@ int main(int argc, char *argv[])
      cout << "initial   total energy (TE) = " << (ee0 + ke0) << endl;
   }

-   // 10. Define the ODE solver used for time integration. Several implicit
-   //     singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //     explicit Runge-Kutta methods are available.
   double t = 0.0;
   oper.SetTime(t);
+   ode_solver->Init(oper);

-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // Implicit L-stable methods
-      case 1:  ode_solver = new BackwardEulerSolver; break;
-      case 2:  ode_solver = new SDIRK23Solver(2); break;
-      case 3:  ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 4:  ode_solver = new ImplicitMidpointSolver; break;
-      case 5:  ode_solver = new SDIRK23Solver; break;
-      case 6:  ode_solver = new SDIRK34Solver; break;
-      // Explicit methods
-      case 7:  ode_solver = new ForwardEulerSolver; break;
-      case 8:  ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 9:  ode_solver = new RK3SSPSolver; break;
-      case 10: ode_solver = new RK4Solver; break;
-      // CVODE BDF
-      case 11:
-      case 12:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // CVODE Adams
-      case 13:
-      case 14:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         CVodeSetEpsLin(cvode->GetMem(), cvode_eps_lin);
-         cvode->SetMaxStep(dt);
-         if (ode_solver_type == 13)
-         {
-            cvode->UseSundialsLinearSolver();
-         }
-         ode_solver = cvode; break;
-      // ARKStep Implicit methods
-      case 15:
-      case 16:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         ARKStepSetNonlinConvCoef(arkode->GetMem(), arkode_eps_nonlin);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 15)
-         {
-            arkode->UseSundialsLinearSolver();
-         }
-         ode_solver = arkode; break;
-      // ARKStep Explicit methods
-      case 17:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 11) { ode_solver->Init(oper); }
-
-   // 11. Perform time-integration
+   // 10. Perform time-integration
   //     (looping over the time iterations, ti, with a time-step dt).
   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -550,7 +538,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 12. Save the displaced mesh, the velocity and elastic energy.
+   // 11. Save the displaced mesh, the velocity and elastic energy.
   {
      v_gf.SetFromTrueVector(); x_gf.SetFromTrueVector();
      GridFunction *nodes = &x_gf;
@@ -575,8 +563,9 @@ int main(int argc, char *argv[])
      w_gf.Save(ee_ofs);
   }

-   // 13. Free the used memory.
+   // 12. Free the used memory.
   delete ode_solver;
+   delete sjsolver;
   delete pmesh;

   MPI_Finalize();
@@ -664,14 +653,92 @@ ReducedSystemOperator::~ReducedSystemOperator()
 }


+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+   HyperelasticOperator *he_oper;
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   he_oper = dynamic_cast<HyperelasticOperator*>(td_oper);
+   MFEM_VERIFY(he_oper, "operator is not HyperelasticOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // he_oper = static_cast<HyperelasticOperator*>(td_oper);
+
+   he_oper->InitSundialsJacSolver(*this);
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   int sc = y_pred.Size() / 2;
+   const Vector x(y_pred.GetData() + sc, sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // J = M + dt*(S + dt*grad(H))
+   delete Jacobian;
+   SparseMatrix *localJ = Add(1.0, M->SpMat(), dt, S->SpMat());
+   local_grad_H = &H->GetLocalGradient(x);
+   localJ->Add(dt*dt, *local_grad_H);
+   Jacobian = M->ParallelAssemble(localJ);
+   delete localJ;
+   HypreParMatrix *Je = Jacobian->EliminateRowsCols(*ess_tdof_list);
+   delete Je;
+
+   J_solver->SetOperator(*Jacobian);
+
+   jac_cur = 1;
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   int sc = b.Size() / 2;
+   ParFiniteElementSpace *fes = H->ParFESpace();
+   // Vector x(y_cur.GetData() + sc, sc);
+   Vector b_v(b.GetData() +  0, sc);
+   Vector b_x(b.GetData() + sc, sc);
+   Vector rhs(sc);
+   double dt = GetTimeStep(sundials_mem);
+
+   // We can assume that b_v and b_x have zeros at essential tdofs.
+
+   // rhs = M b_v - dt*grad(H) b_x
+   ParGridFunction lb_x(fes), lrhs(fes);
+   lb_x.Distribute(b_x);
+   local_grad_H->Mult(lb_x, lrhs);
+   lrhs.ParallelAssemble(rhs);
+   rhs *= -dt;
+   M->TrueAddMult(b_v, rhs);
+   rhs.SetSubVector(*ess_tdof_list, 0.0);
+
+   J_solver->iterative_mode = false;
+   J_solver->Mult(rhs, b_v);
+
+   b_x.Add(dt, b_v);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   delete Jacobian;
+   return 0;
+}
+
+
 HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,
                                           Array<int> &ess_bdr, double visc,
                                           double mu, double K,
                                           NonlinearSolverType nls_type)
   : TimeDependentOperator(2*f.TrueVSize(), 0.0), fespace(f),
     M(&fespace), S(&fespace), H(&fespace),
-     viscosity(visc), M_solver(f.GetComm()), z(height/2),
-     local_grad_H(NULL), Jacobian(NULL)
+     viscosity(visc), M_solver(f.GetComm()), z(height/2)
 {
   const double rel_tol = 1e-8;
   const int skip_zero_entries = 0;
@@ -721,24 +788,23 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,

   if (nls_type == KINSOL)
   {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_NONE, true);
+      KinSolver *kinsolver = new KinSolver(f.GetComm(), KIN_NONE, true);
+      kinsolver->SetMaxSetupCalls(4);
      newton_solver = kinsolver;
-      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(200);
      newton_solver->SetRelTol(rel_tol);
      newton_solver->SetPrintLevel(0);
-      kinsolver->SetMaxSetupCalls(4);
   }
   else
   {
      newton_solver = new NewtonSolver(f.GetComm());
-      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(10);
      newton_solver->SetRelTol(rel_tol);
      newton_solver->SetPrintLevel(-1);
   }
   newton_solver->SetSolver(*J_solver);
   newton_solver->iterative_mode = false;
+   newton_solver->SetOperator(*reduced_oper);
 }

 void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const
@@ -792,64 +858,9 @@ void HyperelasticOperator::ImplicitSolve(const double dt,
   add(v, dt, dv_dt, dx_dt);
 }

-int HyperelasticOperator::SUNImplicitSetup(const Vector &y,
-                                           const Vector &fy, int jok, int *jcur,
-                                           double gamma)
+void HyperelasticOperator::InitSundialsJacSolver(SundialsJacSolver &sjsolv)
 {
-   int sc = y.Size() / 2;
-   const Vector x(y.GetData() + sc, sc);
-
-   // J = M + dt*(S + dt*grad(H))
-   if (Jacobian) { delete Jacobian; }
-   SparseMatrix *localJ = Add(1.0, M.SpMat(), gamma, S.SpMat());
-   local_grad_H = &H.GetLocalGradient(x);
-   localJ->Add(gamma*gamma, *local_grad_H);
-   Jacobian = M.ParallelAssemble(localJ);
-   delete localJ;
-   HypreParMatrix *Je = Jacobian->EliminateRowsCols(ess_tdof_list);
-   delete Je;
-
-   // Set Jacobian solve operator
-   J_solver->SetOperator(*Jacobian);
-
-   // Indicate that the Jacobian was updated
-   *jcur = 1;
-
-   // Save gamma for use in solve
-   saved_gamma = gamma;
-
-   // Return success
-   return 0;
-}
-
-int HyperelasticOperator::SUNImplicitSolve(const Vector &b, Vector &x,
-                                           double tol)
-{
-   int sc = b.Size() / 2;
-   ParFiniteElementSpace *fes = H.ParFESpace();
-   Vector b_v(b.GetData() +  0, sc);
-   Vector b_x(b.GetData() + sc, sc);
-   Vector x_v(x.GetData() +  0, sc);
-   Vector x_x(x.GetData() + sc, sc);
-   Vector rhs(sc);
-
-   // We can assume that b_v and b_x have zeros at essential tdofs.
-
-   // rhs = M b_v - dt*grad(H) b_x
-   ParGridFunction lb_x(fes), lrhs(fes);
-   lb_x.Distribute(b_x);
-   local_grad_H->Mult(lb_x, lrhs);
-   lrhs.ParallelAssemble(rhs);
-   rhs *= -saved_gamma;
-   M.TrueAddMult(b_v, rhs);
-   rhs.SetSubVector(ess_tdof_list, 0.0);
-
-   J_solver->iterative_mode = false;
-   J_solver->Mult(rhs, x_v);
-
-   add(b_x, saved_gamma, x_v, x_x);
-
-   return 0;
+   sjsolv.SetOperators(M, S, H, *J_solver, ess_tdof_list);
 }

 double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const
@@ -875,7 +886,6 @@ void HyperelasticOperator::GetElasticEnergyDensity(

 HyperelasticOperator::~HyperelasticOperator()
 {
-   delete Jacobian;
   delete newton_solver;
   delete J_solver;
   delete J_prec;
@@ -7,9 +7,9 @@
 //               ex16 -m ../../data/inline-tri.mesh
 //               ex16 -m ../../data/disc-nurbs.mesh -tf 2
 //               ex16 -s 12 -a 0.0 -k 1.0
-//               ex16 -s 8 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
-//               ex16 -s 9 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
-//               ex16 -s 10 -dt 1.0e-4 -tf 4.0e-2 -vs 40
+//               ex16 -s 1 -a 1.0 -k 0.0 -dt 1e-4 -tf 5e-2 -vs 25
+//               ex16 -s 2 -a 0.5 -k 0.5 -o 4 -dt 1e-4 -tf 2e-2 -vs 25
+//               ex16 -s 3 -dt 1.0e-4 -tf 4.0e-2 -vs 40
 //               ex16 -m ../../data/fichera-q2.mesh
 //               ex16 -m ../../data/escher.mesh
 //               ex16 -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -58,6 +58,7 @@ protected:

   SparseMatrix Mmat, Kmat;
   SparseMatrix *T; // T = M + dt K
+   double current_dt;

   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
   DSmoother M_prec;  // Preconditioner for the mass matrix M
@@ -74,30 +75,13 @@ public:
                      const Vector &u);

   virtual void Mult(const Vector &u, Vector &du_dt) const;
-
   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);

-   /// Custom Jacobian system solver for the SUNDIALS time integrators.
-   /** For the ODE system represented by ConductionOperator
-
-           M du/dt = -K(u),
-
-       this class facilitates the solution of linear systems of the form
-
-           (M + γK) y = M b,
-
-       for given b, u (not used), and γ = GetTimeStep(). */
-
-   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Solve the system (M + dt K) y = M b. The result y replaces the input b.
+       This method is used by the implicit SUNDIALS solvers. */
+   void SundialsSolve(const double dt, Vector &b);

   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
   void SetParameters(const Vector &u);
@@ -105,6 +89,33 @@ public:
   virtual ~ConductionOperator();
 };

+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by ConductionOperator
+
+        M du/dt = -K(u),
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γK) y = M b,
+
+    for given b, u (not used), and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   ConductionOperator *oper;
+
+public:
+   SundialsJacSolver() : oper(NULL) { }
+
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+
 double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
@@ -113,7 +124,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../../data/star.mesh";
   int ref_levels = 2;
   int order = 2;
-   int ode_solver_type = 9; // CVODE implicit BDF
+   int ode_solver_type = 11; // 11 = CVODE implicit
   double t_final = 0.5;
   double dt = 1.0e-2;
   double alpha = 1.0e-2;
@@ -136,19 +147,12 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Forward Euler,\n\t"
-                  "2  - RK2,\n\t"
-                  "3  - RK3 SSP,\n\t"
-                  "4  - RK4,\n\t"
-                  "5  - Backward Euler,\n\t"
-                  "6  - SDIRK 2,\n\t"
-                  "7  - SDIRK 3,\n\t"
-                  "8  - CVODE (implicit Adams),\n\t"
-                  "9  - CVODE (implicit BDF),\n\t"
-                  "10 - ARKODE (default explicit),\n\t"
-                  "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default impicit).");
+                  "ODE solver:\n"
+                  "\t 1/11 - CVODE (explicit/implicit),\n"
+                  "\t 2/12 - ARKODE (default explicit/implicit),\n"
+                  "\t 3 - ARKODE (Fehlberg-6-4-5)\n"
+                  "\t 4 - Forward Euler, 5 - RK2, 6 - RK3 SSP, 7 - RK4,\n"
+                  "\t 8 - Backward Euler, 9 - SDIRK23, 10 - SDIRK33.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -171,11 +175,6 @@ int main(int argc, char *argv[])
      args.PrintUsage(cout);
      return 1;
   }
-   if (ode_solver_type < 1 || ode_solver_type > 12)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 3;
-   }
   args.PrintOptions(cout);

   // 2. Read the mesh from the given mesh file. We can handle triangular,
@@ -183,7 +182,61 @@ int main(int argc, char *argv[])
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 3. Define the ODE solver used for time integration. Several
+   // SUNDIALS solvers are available, as well as included both
+   // explicit and implicit MFEM ODE solvers.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver sun_solver; // Used by the implicit SUNDIALS ode solvers.
+   switch (ode_solver_type)
+   {
+      // SUNDIALS solvers
+      case 1:
+         cvode = new CVODESolver(CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 11:
+         cvode = new CVODESolver(CV_BDF, CV_NEWTON);
+         cvode->SetLinearSolver(sun_solver);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 2:
+      case 3:
+         arkode = new ARKODESolver(ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 3) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKODESolver(ARKODESolver::IMPLICIT);
+         arkode->SetLinearSolver(sun_solver);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Other MFEM explicit methods
+      case 4: ode_solver = new ForwardEulerSolver; break;
+      case 5: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 6: ode_solver = new RK3SSPSolver; break;
+      case 7: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 8:  ode_solver = new BackwardEulerSolver; break;
+      case 9:  ode_solver = new SDIRK23Solver(2); break;
+      case 10: ode_solver = new SDIRK33Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // Since we want to update the diffusion coefficient after every time step,
+   // we need to use the "one-step" mode of the SUNDIALS solvers.
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
   //    command-line parameter.
   for (int lev = 0; lev < ref_levels; lev++)
@@ -191,7 +244,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 4. Define the vector finite element space representing the current and the
+   // 5. Define the vector finite element space representing the current and the
   //    initial temperature, u_ref.
   H1_FECollection fe_coll(order, dim);
   FiniteElementSpace fespace(mesh, &fe_coll);
@@ -201,14 +254,14 @@ int main(int argc, char *argv[])

   GridFunction u_gf(&fespace);

-   // 5. Set the initial conditions for u. All boundaries are considered
+   // 6. Set the initial conditions for u. All boundaries are considered
   //    natural.
   FunctionCoefficient u_0(InitialTemperature);
   u_gf.ProjectCoefficient(u_0);
   Vector u;
   u_gf.GetTrueDofs(u);

-   // 6. Initialize the conduction operator and the visualization.
+   // 7. Initialize the conduction operator and the visualization.
   ConductionOperator oper(fespace, alpha, kappa, u);

   u_gf.SetFromTrueDofs(u);
@@ -254,65 +307,13 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define the ODE solver used for time integration.
-   double t = 0.0;
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // MFEM explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      // MFEM implicit L-stable methods
-      case 5: ode_solver = new BackwardEulerSolver; break;
-      case 6: ode_solver = new SDIRK23Solver(2); break;
-      case 7: ode_solver = new SDIRK33Solver; break;
-      // CVODE
-      case 8:
-         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      case 9:
-         cvode = new CVODESolver(CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      // ARKODE
-      case 10:
-      case 11:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-      case 12:
-         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 8) { ode_solver->Init(oper); }
-
-   // Since we want to update the diffusion coefficient after every time step,
-   // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
-   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
-
   // 8. Perform time-integration (looping over the time iterations, ti, with a
   //    time-step dt).
   cout << "Integrating the ODE ..." << endl;
   tic_toc.Clear();
   tic_toc.Start();
+   ode_solver->Init(oper);
+   double t = 0.0;

   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -370,7 +371,7 @@ int main(int argc, char *argv[])
 ConductionOperator::ConductionOperator(FiniteElementSpace &f, double al,
                                       double kap, const Vector &u)
   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL), z(height)
+     T(NULL), current_dt(0.0), z(height)
 {
   const double rel_tol = 1e-8;

@@ -416,14 +417,32 @@ void ConductionOperator::ImplicitSolve(const double dt,
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
   // for du_dt
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, dt, Kmat);
-   T_solver.SetOperator(*T);
+   if (!T)
+   {
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
   Kmat.Mult(u, z);
   z.Neg();
   T_solver.Mult(z, du_dt);
 }

+void ConductionOperator::SundialsSolve(const double dt, Vector &b)
+{
+   // Solve the system (M + dt K) y = M b. The result y replaces the input b.
+   if (!T || dt != current_dt)
+   {
+      delete T;
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   Mmat.Mult(b, z);
+   T_solver.Mult(z, b);
+}
+
 void ConductionOperator::SetParameters(const Vector &u)
 {
   GridFunction u_alpha_gf(&fespace);
@@ -441,26 +460,8 @@ void ConductionOperator::SetParameters(const Vector &u)
   K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
   K->Assemble();
   K->FormSystemMatrix(ess_tdof_list, Kmat);
-}
-
-int ConductionOperator::SUNImplicitSetup(const Vector &x,
-                                         const Vector &fx, int jok, int *jcur,
-                                         double gamma)
-{
-   // Setup the ODE Jacobian T = M + gamma K.
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, gamma, Kmat);
-   T_solver.SetOperator(*T);
-   *jcur = 1;
-   return (0);
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
-{
-   // Solve the system A x = z => (M - gamma K) x = M b.
-   Mmat.Mult(b, z);
-   T_solver.Mult(z, x);
-   return (0);
+   delete T;
+   T = NULL; // re-compute T on the next ImplicitSolve or SundialsSolve
 }

 ConductionOperator::~ConductionOperator()
@@ -470,6 +471,46 @@ ConductionOperator::~ConductionOperator()
   delete K;
 }

+
+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   oper = dynamic_cast<ConductionOperator*>(td_oper);
+   MFEM_VERIFY(oper, "operator is not ConductionOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // oper = static_cast<ConductionOperator*>(td_oper);
+
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   jac_cur = 1;
+
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   oper->SundialsSolve(GetTimeStep(sundials_mem), b);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   return 0;
+}
+
+
 double InitialTemperature(const Vector &x)
 {
   if (x.Norml2() < 0.5)
@@ -8,9 +8,9 @@
 //     mpirun -np 4 ex16p -m ../../data/inline-tri.mesh
 //     mpirun -np 4 ex16p -m ../../data/disc-nurbs.mesh -tf 2
 //     mpirun -np 4 ex16p -s 12 -a 0.0 -k 1.0
-//     mpirun -np 4 ex16p -s 8 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
-//     mpirun -np 8 ex16p -s 9 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
-//     mpirun -np 4 ex16p -s 10 -dt 2.0e-4 -tf 4.0e-2
+//     mpirun -np 4 ex16p -s 1 -a 1.0 -k 0.0 -dt 4e-6 -tf 2e-2 -vs 50
+//     mpirun -np 8 ex16p -s 2 -a 0.5 -k 0.5 -o 4 -dt 8e-6 -tf 2e-2 -vs 50
+//     mpirun -np 4 ex16p -s 3 -dt 2.0e-4 -tf 4.0e-2
 //     mpirun -np 16 ex16p -m ../../data/fichera-q2.mesh
 //     mpirun -np 16 ex16p -m ../../data/escher-p2.mesh
 //     mpirun -np 8 ex16p -m ../../data/beam-tet.mesh -tf 10 -dt 0.1
@@ -77,19 +77,13 @@ public:
                      const Vector &u);

   virtual void Mult(const Vector &u, Vector &du_dt) const;
-
   /** Solve the Backward-Euler equation: k = f(u + dt*k, t), for the unknown k.
       This is the only requirement for high-order SDIRK implicit integration.*/
   virtual void ImplicitSolve(const double dt, const Vector &u, Vector &k);

-   /** Setup the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSetup(const Vector &x, const Vector &fx,
-                                int jok, int *jcur, double gamma);
-
-   /** Solve the system (M + dt K) x = M b. This method is used by the implicit
-       SUNDIALS solvers. */
-   virtual int SUNImplicitSolve(const Vector &b, Vector &x, double tol);
+   /** Solve the system (M + dt K) y = M b. The result y replaces the input b.
+       This method is used by the implicit SUNDIALS solvers. */
+   void SundialsSolve(const double dt, Vector &b);

   /// Update the diffusion BilinearForm K using the given true-dof vector `u`.
   void SetParameters(const Vector &u);
@@ -97,6 +91,33 @@ public:
   virtual ~ConductionOperator();
 };

+/// Custom Jacobian system solver for the SUNDIALS time integrators.
+/** For the ODE system represented by ConductionOperator
+
+        M du/dt = -K(u),
+
+    this class facilitates the solution of linear systems of the form
+
+        (M + γK) y = M b,
+
+    for given b, u (not used), and γ = GetTimeStep(). */
+class SundialsJacSolver : public SundialsODELinearSolver
+{
+private:
+   ConductionOperator *oper;
+
+public:
+   SundialsJacSolver() : oper(NULL) { }
+
+   int InitSystem(void *sundials_mem);
+   int SetupSystem(void *sundials_mem, int conv_fail,
+                   const Vector &y_pred, const Vector &f_pred, int &jac_cur,
+                   Vector &v_temp1, Vector &v_temp2, Vector &v_temp3);
+   int SolveSystem(void *sundials_mem, Vector &b, const Vector &weight,
+                   const Vector &y_cur, const Vector &f_cur);
+   int FreeSystem(void *sundials_mem);
+};
+
 double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
@@ -112,7 +133,7 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 1;
   int order = 2;
-   int ode_solver_type = 9; // CVODE implicit BDF
+   int ode_solver_type = 11; // 11 = CVODE implicit
   double t_final = 0.5;
   double dt = 1.0e-2;
   double alpha = 1.0e-2;
@@ -137,19 +158,12 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1  - Forward Euler,\n\t"
-                  "2  - RK2,\n\t"
-                  "3  - RK3 SSP,\n\t"
-                  "4  - RK4,\n\t"
-                  "5  - Backward Euler,\n\t"
-                  "6  - SDIRK 2,\n\t"
-                  "7  - SDIRK 3,\n\t"
-                  "8  - CVODE (implicit Adams),\n\t"
-                  "9  - CVODE (implicit BDF),\n\t"
-                  "10 - ARKODE (default explicit),\n\t"
-                  "11 - ARKODE (explicit Fehlberg-6-4-5),\n\t"
-                  "12 - ARKODE (default impicit).");
+                  "ODE solver:\n"
+                  "\t 1/11 - CVODE (explicit/implicit),\n"
+                  "\t 2/12 - ARKODE (default explicit/implicit),\n"
+                  "\t 3 - ARKODE (Fehlberg-6-4-5)\n"
+                  "\t 4 - Forward Euler, 5 - RK2, 6 - RK3 SSP, 7 - RK4,\n"
+                  "\t 8 - Backward Euler, 9 - SDIRK23, 10 - SDIRK33.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -179,24 +193,67 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 12)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 1;
-   }
-
   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle triangular, quadrilateral, tetrahedral and hexahedral meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh in serial to increase the resolution. In this example
+   // 4. Define the ODE solver used for time integration. Several
+   // SUNDIALS solvers are available, as well as included both
+   // explicit and implicit MFEM ODE solvers.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   SundialsJacSolver sun_solver; // Used by the implicit SUNDIALS ode solvers.
+   switch (ode_solver_type)
+   {
+      // SUNDIALS solvers
+      case 1:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 11:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF, CV_NEWTON);
+         cvode->SetLinearSolver(sun_solver);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 2:
+      case 3:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 3) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      case 12:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::IMPLICIT);
+         arkode->SetLinearSolver(sun_solver);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         ode_solver = arkode; break;
+      // Other MFEM explicit methods
+      case 4: ode_solver = new ForwardEulerSolver; break;
+      case 5: ode_solver = new RK2Solver(0.5); break; // midpoint method
+      case 6: ode_solver = new RK3SSPSolver; break;
+      case 7: ode_solver = new RK4Solver; break;
+      // MFEM implicit L-stable methods
+      case 8:  ode_solver = new BackwardEulerSolver; break;
+      case 9:  ode_solver = new SDIRK23Solver(2); break;
+      case 10: ode_solver = new SDIRK33Solver; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // Since we want to update the diffusion coefficient after every time step,
+   // we need to use the "one-step" mode of the SUNDIALS solvers.
+   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
+   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
+
+   // 5. Refine the mesh in serial to increase the resolution. In this example
   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
   //    a command-line parameter.
   for (int lev = 0; lev < ser_ref_levels; lev++)
@@ -204,7 +261,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -214,7 +271,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 6. Define the vector finite element space representing the current and the
+   // 7. Define the vector finite element space representing the current and the
   //    initial temperature, u_ref.
   H1_FECollection fe_coll(order, dim);
   ParFiniteElementSpace fespace(pmesh, &fe_coll);
@@ -227,14 +284,14 @@ int main(int argc, char *argv[])

   ParGridFunction u_gf(&fespace);

-   // 7. Set the initial conditions for u. All boundaries are considered
+   // 8. Set the initial conditions for u. All boundaries are considered
   //    natural.
   FunctionCoefficient u_0(InitialTemperature);
   u_gf.ProjectCoefficient(u_0);
   Vector u;
   u_gf.GetTrueDofs(u);

-   // 8. Initialize the conduction operator and the VisIt visualization.
+   // 9. Initialize the conduction operator and the VisIt visualization.
   ConductionOperator oper(fespace, alpha, kappa, u);

   u_gf.SetFromTrueDofs(u);
@@ -293,60 +350,6 @@ int main(int argc, char *argv[])
      }
   }

-   // 9. Define the ODE solver used for time integration.
-   double t = 0.0;
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      // MFEM explicit methods
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(0.5); break; // midpoint method
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      // MFEM implicit L-stable methods
-      case 5: ode_solver = new BackwardEulerSolver; break;
-      case 6: ode_solver = new SDIRK23Solver(2); break;
-      case 7: ode_solver = new SDIRK33Solver; break;
-      // CVODE
-      case 8:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      case 9:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_BDF);
-         cvode->Init(oper);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         ode_solver = cvode; break;
-      // ARKODE
-      case 10:
-      case 11:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-      case 12:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
-         arkode->Init(oper);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 8) { ode_solver->Init(oper); }
-
-   // Since we want to update the diffusion coefficient after every time step,
-   // we need to use the "one-step" mode of the SUNDIALS solvers.
-   if (cvode) { cvode->SetStepMode(CV_ONE_STEP); }
-   if (arkode) { arkode->SetStepMode(ARK_ONE_STEP); }
-
   // 10. Perform time-integration (looping over the time iterations, ti, with a
   //     time-step dt).
   if (myid == 0)
@@ -355,6 +358,8 @@ int main(int argc, char *argv[])
   }
   tic_toc.Clear();
   tic_toc.Start();
+   ode_solver->Init(oper);
+   double t = 0.0;

   bool last_step = false;
   for (int ti = 1; !last_step; ti++)
@@ -423,7 +428,7 @@ int main(int argc, char *argv[])
 ConductionOperator::ConductionOperator(ParFiniteElementSpace &f, double al,
                                       double kap, const Vector &u)
   : TimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL), K(NULL),
-     T(NULL),
+     T(NULL), current_dt(0.0),
     M_solver(f.GetComm()), T_solver(f.GetComm()), z(height)
 {
   const double rel_tol = 1e-8;
@@ -471,32 +476,30 @@ void ConductionOperator::ImplicitSolve(const double dt,
   // Solve the equation:
   //    du_dt = M^{-1}*[-K(u + dt*du_dt)]
   // for du_dt
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, dt, Kmat);
-   T_solver.SetOperator(*T);
+   if (!T)
+   {
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
+   MFEM_VERIFY(dt == current_dt, ""); // SDIRK methods use the same dt
   Kmat.Mult(u, z);
   z.Neg();
   T_solver.Mult(z, du_dt);
 }

-int ConductionOperator::SUNImplicitSetup(const Vector &x,
-                                         const Vector &fx, int jok, int *jcur,
-                                         double gamma)
+void ConductionOperator::SundialsSolve(const double dt, Vector &b)
 {
-   // Setup the ODE Jacobian T = M + gamma K.
-   if (T) { delete T; }
-   T = Add(1.0, Mmat, gamma, Kmat);
-   T_solver.SetOperator(*T);
-   *jcur = 1;
-   return (0);
-}
-
-int ConductionOperator::SUNImplicitSolve(const Vector &b, Vector &x, double tol)
-{
-   // Solve the system A x = z => (M - gamma K) x = M b.
+   // Solve the system (M + dt K) y = M b. The result y replaces the input b.
+   if (!T || dt != current_dt)
+   {
+      delete T;
+      T = Add(1.0, Mmat, dt, Kmat);
+      current_dt = dt;
+      T_solver.SetOperator(*T);
+   }
   Mmat.Mult(b, z);
-   T_solver.Mult(z, x);
-   return (0);
+   T_solver.Mult(z, b);
 }

 void ConductionOperator::SetParameters(const Vector &u)
@@ -516,6 +519,8 @@ void ConductionOperator::SetParameters(const Vector &u)
   K->AddDomainIntegrator(new DiffusionIntegrator(u_coeff));
   K->Assemble(0); // keep sparsity pattern of M and K the same
   K->FormSystemMatrix(ess_tdof_list, Kmat);
+   delete T;
+   T = NULL; // re-compute T on the next ImplicitSolve or SundialsSolve
 }

 ConductionOperator::~ConductionOperator()
@@ -525,6 +530,46 @@ ConductionOperator::~ConductionOperator()
   delete K;
 }

+
+int SundialsJacSolver::InitSystem(void *sundials_mem)
+{
+   TimeDependentOperator *td_oper = GetTimeDependentOperator(sundials_mem);
+
+   // During development, we use dynamic_cast<> to ensure the setup is correct:
+   oper = dynamic_cast<ConductionOperator*>(td_oper);
+   MFEM_VERIFY(oper, "operator is not ConductionOperator");
+
+   // When the implementation is finalized, we can switch to static_cast<>:
+   // oper = static_cast<ConductionOperator*>(td_oper);
+
+   return 0;
+}
+
+int SundialsJacSolver::SetupSystem(void *sundials_mem, int conv_fail,
+                                   const Vector &y_pred, const Vector &f_pred,
+                                   int &jac_cur, Vector &v_temp1,
+                                   Vector &v_temp2, Vector &v_temp3)
+{
+   jac_cur = 1;
+
+   return 0;
+}
+
+int SundialsJacSolver::SolveSystem(void *sundials_mem, Vector &b,
+                                   const Vector &weight, const Vector &y_cur,
+                                   const Vector &f_cur)
+{
+   oper->SundialsSolve(GetTimeStep(sundials_mem), b);
+
+   return 0;
+}
+
+int SundialsJacSolver::FreeSystem(void *sundials_mem)
+{
+   return 0;
+}
+
+
 double InitialTemperature(const Vector &x)
 {
   if (x.Norml2() < 0.5)
@@ -4,14 +4,14 @@
 // Compile with: make ex9
 //
 // Sample runs:
-//    ex9 -m ../../data/periodic-segment.mesh -p 0 -r 2 -s 7 -dt 0.005
-//    ex9 -m ../../data/periodic-square.mesh  -p 1 -r 2 -s 8 -dt 0.005 -tf 9
-//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 7 -dt 0.0018 -vs 25
-//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 9 -dt 0.01 -vs 15
-//    ex9 -m ../../data/amr-quad.mesh         -p 1 -r 2 -s 9 -dt 0.002 -tf 9
-//    ex9 -m ../../data/star-q3.mesh          -p 1 -r 2 -s 9 -dt 0.005 -tf 9
-//    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 7 -dt 0.005 -tf 9
-//    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 8 -dt 0.02 -tf 8 -o 2
+//    ex9 -m ../../data/periodic-segment.mesh -p 0 -r 2 -s 11 -dt 0.005
+//    ex9 -m ../../data/periodic-square.mesh  -p 1 -r 2 -s 12 -dt 0.005 -tf 9
+//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 11 -dt 0.0018 -vs 25
+//    ex9 -m ../../data/periodic-hexagon.mesh -p 0 -r 2 -s 13 -dt 0.01 -vs 15
+//    ex9 -m ../../data/amr-quad.mesh         -p 1 -r 2 -s 13 -dt 0.002 -tf 9
+//    ex9 -m ../../data/star-q3.mesh          -p 1 -r 2 -s 13 -dt 0.005 -tf 9
+//    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 11 -dt 0.005 -tf 9
+//    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 12 -dt 0.02 -tf 8 -o 2
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -109,15 +109,11 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1 - Forward Euler,\n\t"
-                  "2 - RK2 SSP,\n\t"
-                  "3 - RK3 SSP,\n\t"
-                  "4 - RK4,\n\t"
-                  "6 - RK6,\n\t"
-                  "7 - CVODE (adaptive order implicit Adams),\n\t"
-                  "8 - ARKODE default (4th order) explicit,\n\t"
-                  "9 - ARKODE RK8.");
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - CVODE (adaptive order) explicit,\n\t"
+                  "            12 - ARKODE default (4th order) explicit,\n\t"
+                  "            13 - ARKODE RK8.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -139,41 +135,65 @@ int main(int argc, char *argv[])
      args.PrintUsage(cout);
      return 1;
   }
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 9)
-   {
-      cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      return 3;
-   }
   args.PrintOptions(cout);

   // 2. Read the mesh from the given mesh file. We can handle geometrically
   //    periodic meshes in this code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

-   // 3. Refine the mesh to increase the resolution. In this example we do
+   // 3. Define the ODE solver used for time integration. Several explicit
+   //    Runge-Kutta methods are available.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      case 11:
+         cvode = new CVODESolver(CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 12:
+      case 13:
+         arkode = new ARKODESolver(ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 13) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      default:
+         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         delete mesh;
+         return 3;
+   }
+
+   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
   //    command-line parameter. If the mesh is of NURBS type, we convert it to
   //    a (piecewise-polynomial) high-order mesh.
   for (int lev = 0; lev < ref_levels; lev++)
   {
-      mesh.UniformRefinement();
+      mesh->UniformRefinement();
   }
-   if (mesh.NURBSext)
+   if (mesh->NURBSext)
   {
-      mesh.SetCurvature(max(order, 1));
+      mesh->SetCurvature(max(order, 1));
   }
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
+   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));

-   // 4. Define the discontinuous DG finite element space of the given
+   // 5. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
   DG_FECollection fec(order, dim);
-   FiniteElementSpace fes(&mesh, &fec);
+   FiniteElementSpace fes(mesh, &fec);

   cout << "Number of unknowns: " << fes.GetVSize() << endl;

-   // 5. Set up and assemble the bilinear and linear forms corresponding to the
+   // 6. Set up and assemble the bilinear and linear forms corresponding to the
   //    DG discretization. The DGTraceIntegrator involves integrals over mesh
   //    interior faces.
   VectorFunctionCoefficient velocity(dim, velocity_function);
@@ -200,7 +220,7 @@ int main(int argc, char *argv[])
   k.Finalize(skip_zeros);
   b.Assemble();

-   // 6. Define the initial conditions, save the corresponding grid function to
+   // 7. Define the initial conditions, save the corresponding grid function to
   //    a file and (optionally) save data in the VisIt format and initialize
   //    GLVis visualization.
   GridFunction u(&fes);
@@ -209,7 +229,7 @@ int main(int argc, char *argv[])
   {
      ofstream omesh("ex9.mesh");
      omesh.precision(precision);
-      mesh.Print(omesh);
+      mesh->Print(omesh);
      ofstream osol("ex9-init.gf");
      osol.precision(precision);
      u.Save(osol);
@@ -223,14 +243,14 @@ int main(int argc, char *argv[])
      if (binary)
      {
 #ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9", &mesh);
+         dc = new SidreDataCollection("Example9", mesh);
 #else
         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
 #endif
      }
      else
      {
-         dc = new VisItDataCollection("Example9", &mesh);
+         dc = new VisItDataCollection("Example9", mesh);
         dc->SetPrecision(precision);
      }
      dc->RegisterField("solution", &u);
@@ -255,7 +275,7 @@ int main(int argc, char *argv[])
      else
      {
         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
+         sout << "solution\n" << *mesh << u;
         sout << "pause\n";
         sout << flush;
         cout << "GLVis visualization paused."
@@ -263,46 +283,15 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and define the ODE solver used for time integration.
+   // 8. Define the time-dependent evolution operator describing the ODE
+   //    right-hand side, and perform time-integration (looping over the time
+   //    iterations, ti, with a time-step dt).
   FE_Evolution adv(m.SpMat(), k.SpMat(), b);

   double t = 0.0;
   adv.SetTime(t);
+   ode_solver->Init(adv);

-   // Create the time integrator
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      case 7:
-         cvode = new CVODESolver(CV_ADAMS);
-         cvode->Init(adv);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         cvode->UseSundialsLinearSolver();
-         ode_solver = cvode; break;
-      case 8:
-      case 9:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 7) { ode_solver->Init(adv); }
-
-   // 8. Perform time-integration (looping over the time iterations, ti,
-   //    with a time-step dt).
   bool done = false;
   for (int ti = 0; !done; )
   {
@@ -320,7 +309,7 @@ int main(int argc, char *argv[])

         if (visualization)
         {
-            sout << "solution\n" << mesh << u << flush;
+            sout << "solution\n" << *mesh << u << flush;
         }

         if (visit)
@@ -4,14 +4,14 @@
 // Compile with: make ex9p
 //
 // Sample runs:
-//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -p 1 -rp 1 -s 7 -dt 0.0025
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh  -p 1 -rp 1 -s 8 -dt 0.0025 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 7 -dt 0.0009 -vs 25
-//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 9 -dt 0.005 -vs 15
-//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh         -p 1 -rp 1 -s 9 -dt 0.001 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/star-q3.mesh          -p 1 -rp 1 -s 9 -dt 0.0025 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 7 -dt 0.0025 -tf 9
-//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 8 -dt 0.01 -tf 8 -o 2
+//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -p 1 -rp 1 -s 11 -dt 0.0025
+//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh  -p 1 -rp 1 -s 12 -dt 0.0025 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 11 -dt 0.0009 -vs 25
+//    mpirun -np 4 ex9p -m ../../data/periodic-hexagon.mesh -p 0 -rp 1 -s 13 -dt 0.005 -vs 15
+//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh         -p 1 -rp 1 -s 13 -dt 0.001 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/star-q3.mesh          -p 1 -rp 1 -s 13 -dt 0.0025 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 11 -dt 0.0025 -tf 9
+//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 12 -dt 0.01 -tf 8 -o 2
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -117,15 +117,11 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver:\n\t"
-                  "1 - Forward Euler,\n\t"
-                  "2 - RK2 SSP,\n\t"
-                  "3 - RK3 SSP,\n\t"
-                  "4 - RK4,\n\t"
-                  "6 - RK6,\n\t"
-                  "7 - CVODE (adaptive order implicit Adams),\n\t"
-                  "8 - ARKODE default (4th order) explicit,\n\t"
-                  "9 - ARKODE RK8.");
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
+                  "            11 - CVODE (adaptive order) explicit,\n\t"
+                  "            12 - ARKODE default (4th order) explicit,\n\t"
+                  "            13 - ARKODE RK8.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -155,23 +151,47 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   // check for vaild ODE solver option
-   if (ode_solver_type < 1 || ode_solver_type > 9)
-   {
-      if (myid == 0)
-      {
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-      }
-      MPI_Finalize();
-      return 3;
-   }

   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle geometrically periodic meshes in this code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh in serial to increase the resolution. In this example
+   // 4. Define the ODE solver used for time integration. Several explicit
+   //    Runge-Kutta methods are available.
+   ODESolver *ode_solver = NULL;
+   CVODESolver *cvode = NULL;
+   ARKODESolver *arkode = NULL;
+   switch (ode_solver_type)
+   {
+      case 1: ode_solver = new ForwardEulerSolver; break;
+      case 2: ode_solver = new RK2Solver(1.0); break;
+      case 3: ode_solver = new RK3SSPSolver; break;
+      case 4: ode_solver = new RK4Solver; break;
+      case 6: ode_solver = new RK6Solver; break;
+      case 11:
+         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS, CV_FUNCTIONAL);
+         cvode->SetSStolerances(reltol, abstol);
+         cvode->SetMaxStep(dt);
+         ode_solver = cvode; break;
+      case 12:
+      case 13:
+         arkode = new ARKODESolver(MPI_COMM_WORLD, ARKODESolver::EXPLICIT);
+         arkode->SetSStolerances(reltol, abstol);
+         arkode->SetMaxStep(dt);
+         if (ode_solver_type == 13) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         ode_solver = arkode; break;
+      default:
+         if (myid == 0)
+         {
+            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
+         }
+         delete mesh;
+         MPI_Finalize();
+         return 3;
+   }
+
+   // 5. Refine the mesh in serial to increase the resolution. In this example
   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
   //    a command-line parameter. If the mesh is of NURBS type, we convert it
   //    to a (piecewise-polynomial) high-order mesh.
@@ -185,7 +205,7 @@ int main(int argc, char *argv[])
   }
   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));

-   // 5. Define the parallel mesh by a partitioning of the serial mesh. Refine
+   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -195,7 +215,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 6. Define the parallel discontinuous DG finite element space on the
+   // 7. Define the parallel discontinuous DG finite element space on the
   //    parallel refined mesh of the given polynomial order.
   DG_FECollection fec(order, dim);
   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
@@ -206,7 +226,7 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << global_vSize << endl;
   }

-   // 7. Set up and assemble the parallel bilinear and linear forms (and the
+   // 8. Set up and assemble the parallel bilinear and linear forms (and the
   //    parallel hypre matrices) corresponding to the DG discretization. The
   //    DGTraceIntegrator involves integrals over mesh interior faces.
   VectorFunctionCoefficient velocity(dim, velocity_function);
@@ -237,7 +257,7 @@ int main(int argc, char *argv[])
   HypreParMatrix *K = k->ParallelAssemble();
   HypreParVector *B = b->ParallelAssemble();

-   // 8. Define the initial conditions, save the corresponding grid function to
+   // 9. Define the initial conditions, save the corresponding grid function to
   //    a file and (optionally) save data in the VisIt format and initialize
   //    GLVis visualization.
   ParGridFunction *u = new ParGridFunction(fes);
@@ -310,46 +330,15 @@ int main(int argc, char *argv[])
      }
   }

-   // 9. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and define the ODE solver used for time integration.
+   // 10. Define the time-dependent evolution operator describing the ODE
+   //     right-hand side, and perform time-integration (looping over the time
+   //     iterations, ti, with a time-step dt).
   FE_Evolution adv(*M, *K, *B);

   double t = 0.0;
   adv.SetTime(t);
+   ode_solver->Init(adv);

-   // Create the time integrator
-   ODESolver *ode_solver = NULL;
-   CVODESolver *cvode = NULL;
-   ARKStepSolver *arkode = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      case 7:
-         cvode = new CVODESolver(MPI_COMM_WORLD, CV_ADAMS);
-         cvode->Init(adv);
-         cvode->SetSStolerances(reltol, abstol);
-         cvode->SetMaxStep(dt);
-         cvode->UseSundialsLinearSolver();
-         ode_solver = cvode; break;
-      case 8:
-      case 9:
-         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
-         ode_solver = arkode; break;
-   }
-
-   // Initialize MFEM integrators, SUNDIALS integrators are initialized above
-   if (ode_solver_type < 7) { ode_solver->Init(adv); }
-
-   // 10. Perform time-integration (looping over the time iterations, ti,
-   //     with a time-step dt).
   bool done = false;
   for (int ti = 0; !done; )
   {
@@ -60,15 +60,15 @@ PARALLEL_NAME := Parallel SUNDIALS example
 	@$(call mfem-test,$<,, $(SERIAL_NAME))

 # Testing: Specific execution options:
-# Example 9: test CVODE with CV_ADAMS (non-stiff implicit) time stepping
-EX9_COMMON_ARGS := -m ../../data/periodic-hexagon.mesh -p 0 -s 7
+# Example 9: test explicit CVODE time stepping
+EX9_COMMON_ARGS := -m ../../data/periodic-hexagon.mesh -p 0 -s 11
 EX9_ARGS  := $(EX9_COMMON_ARGS) -r  2 -dt 0.0018 -vs 25
 EX9P_ARGS := $(EX9_COMMON_ARGS) -rp 1 -dt 0.0009 -vs 50
 ex9-test-seq: ex9
 	@$(call mfem-test,$<,, $(SERIAL_NAME),$(EX9_ARGS))
 ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX9P_ARGS))
-# Example 10: test CVODE with CV_BDF (stiff implicit) time stepping
+# Example 10: test implicit CVODE time stepping
 EX10_COMMON_ARGS := -m ../../data/beam-quad.mesh -o 2 -s 5 -dt 0.15 -tf 6 -vs 10
 EX10_ARGS  := $(EX10_COMMON_ARGS) -r  2
 EX10P_ARGS := $(EX10_COMMON_ARGS) -rp 1
@@ -13,17 +13,8 @@ set(SRCS
  bilinearform.cpp
  bilinearform_ext.cpp
  bilininteg.cpp
-  bilininteg_diffusion.cpp
-  bilininteg_divergence.cpp
-  bilininteg_gradient.cpp
-  bilininteg_mass.cpp
-  bilininteg_divergence.cpp
-  bilininteg_gradient.cpp
-  bilininteg_convection.cpp
-  bilininteg_vecdiffusion.cpp
-  bilininteg_vecmass.cpp
+  bilininteg_ext.cpp
  coefficient.cpp
-  complex_fem.cpp
  datacollection.cpp
  eltrans.cpp
  estimators.cpp
@@ -37,21 +28,17 @@ set(SRCS
  linearform.cpp
  lininteg.cpp
  nonlinearform.cpp
-  nonlinearform_ext.cpp
  nonlininteg.cpp
-  nonlininteg_vectorconvection.cpp
  staticcond.cpp
  tmop.cpp
-  tmop_tools.cpp
-  gslib.cpp
  )

 set(HDRS
  bilinearform.hpp
  bilinearform_ext.hpp
  bilininteg.hpp
+  bilininteg_ext.hpp
  coefficient.hpp
-  complex_fem.hpp
  datacollection.hpp
  eltrans.hpp
  estimators.hpp
@@ -66,7 +53,6 @@ set(HDRS
  linearform.hpp
  lininteg.hpp
  nonlinearform.hpp
-  nonlinearform_ext.hpp
  nonlininteg.hpp
  staticcond.hpp
  tbilinearform.hpp
@@ -78,8 +64,6 @@ set(HDRS
  tfespace.hpp
  tintrules.hpp
  tmop.hpp
-  tmop_tools.hpp
-  gslib.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -109,17 +93,6 @@ if (MFEM_USE_MPI)
    pnonlinearform.hpp)
 endif()

-if (MFEM_USE_CEED)
-   list(APPEND SRCS
-     libceed/ceed.cpp
-     libceed/diffusion.cpp
-     libceed/mass.cpp)
-   list(APPEND HDRS
-     libceed/ceed.hpp
-     libceed/diffusion.hpp
-     libceed/mass.hpp)
-endif()
-
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)

@@ -55,7 +55,7 @@ void BilinearForm::AllocMat()

   int *I = dof_dof.GetI();
   int *J = dof_dof.GetJ();
-   double *data = new double[I[height]];
+   double *data = mfem::New<double>(I[height]);

   mat = new SparseMatrix(I, J, data, height, height, true, true, true);
   *mat = 0.0;
@@ -122,7 +122,11 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   switch (assembly)
   {
      case AssemblyLevel::FULL:
-         // ext = new FABilinearFormExtension(this);
+         if (Device::IsEnabled())
+         {
+            mfem_error("Full assembly not supported yet in device mode!");
+            // ext = new FABilinearFormExtension(this);
+         }
         // Use the original BilinearForm implementation for now
         break;
      case AssemblyLevel::ELEMENT:
@@ -204,7 +208,7 @@ void BilinearForm::UseSparsity(SparseMatrix &A)
               << A.Height() << " x " << A.Width());
   MFEM_ASSERT(A.Finalized(), "matrix A must be Finalized");

-   UseSparsity(A.GetI(), A.GetJ(), A.ColumnsAreSorted());
+   UseSparsity(A.GetI(), A.GetJ(), A.areColumnsSorted());
 }

 double& BilinearForm::Elem (int i, int j)
@@ -294,33 +298,6 @@ void BilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
   }
 }

-void BilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (bbfi.Size())
-   {
-      const FiniteElement &be = *fes->GetBE(i);
-      ElementTransformation *eltrans = fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix(be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
-      {
-         bbfi[k]->AssembleElementMatrix(be, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      fes->GetBdrElementVDofs(i, vdofs);
-      elmat.SetSize(vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void BilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleElementMatrix(i, elmat, vdofs, skip_zeros);
-}
-
 void BilinearForm::AssembleElementMatrix(
   int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
 {
@@ -343,12 +320,6 @@ void BilinearForm::AssembleElementMatrix(
   }
 }

-void BilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleBdrElementMatrix(i, elmat, vdofs, skip_zeros);
-}
-
 void BilinearForm::AssembleBdrElementMatrix(
   int i, const DenseMatrix &elmat, Array<int> &vdofs, int skip_zeros)
 {
@@ -373,6 +344,11 @@ void BilinearForm::AssembleBdrElementMatrix(

 void BilinearForm::Assemble(int skip_zeros)
 {
+   if (Device::IsEnabled() && (assembly != AssemblyLevel::PARTIAL))
+   {
+      mfem_error("Chosen assembly level not supported yet in device mode!");
+   }
+
   if (ext)
   {
      ext->Assemble();
@@ -608,41 +584,22 @@ void BilinearForm::ConformingAssemble()
   width = mat->Width();
 }

-void BilinearForm::AssembleDiagonal(Vector &diag) const
-{
-   if (ext)
-   {
-      MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      const Operator *P = fes->GetProlongationMatrix();
-      if (!IsIdentityProlongation(P))
-      {
-         Vector local_diag(P->Height());
-         ext->AssembleDiagonal(local_diag);
-         P->MultTranspose(local_diag, diag);
-      }
-      else
-      {
-         ext->AssembleDiagonal(diag);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
-                 "matrix and use SparseMatrix::GetDiag?");
-   }
-}
-
 void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
                                    Vector &b, OperatorHandle &A, Vector &X,
                                    Vector &B, int copy_interior)
 {
+   const SparseMatrix *P = fes->GetConformingProlongation();
+
   if (ext)
   {
+      if (P != NULL && assembly != AssemblyLevel::FULL && Device::IsEnabled())
+      {
+         P->BuildTranspose();
+      }
      ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
      return;
   }
-   const SparseMatrix *P = fes->GetConformingProlongation();
+
   FormSystemMatrix(ess_tdof_list, A);

   // Transform the system and perform the elimination in B, based on the
@@ -668,8 +625,8 @@ void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
      {
         // A, X and B point to the same data as mat, x and b
         EliminateVDofsInRHS(ess_tdof_list, x, b);
-         X.NewMemoryAndSize(x.GetMemory(), x.Size(), false);
-         B.NewMemoryAndSize(b.GetMemory(), b.Size(), false);
+         X.NewDataAndSize(x.GetData(), x.Size());
+         B.NewDataAndSize(b.GetData(), b.Size());
         if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
      }
   }
@@ -770,10 +727,6 @@ void BilinearForm::RecoverFEMSolution(const Vector &X,
      else
      {
         // X and x point to the same data
-
-         // If the validity flags of X's Memory were changed (e.g. if it was
-         // moved to device memory) then we need to tell x about that.
-         x.SyncMemory(X);
      }
   }
   else // non-conforming space
@@ -983,18 +936,6 @@ void BilinearForm::EliminateVDofsInRHS(
   mat->PartMult(vdofs, x, b);
 }

-void BilinearForm::Mult(const Vector &x, Vector &y) const
-{
-   if (ext)
-   {
-      ext->Mult(x, y);
-   }
-   else
-   {
-      mat->Mult(x, y);
-   }
-}
-
 void BilinearForm::Update(FiniteElementSpace *nfes)
 {
   bool full_update;
@@ -1070,11 +1011,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   trial_fes = tr_fes;
   test_fes = te_fes;
   mat = NULL;
-   mat_e = NULL;
   extern_bfs = 0;
-
-   assembly = AssemblyLevel::FULL;
-   ext = NULL;
 }

 MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
@@ -1085,49 +1022,12 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   trial_fes = tr_fes;
   test_fes = te_fes;
   mat = NULL;
-   mat_e = NULL;
   extern_bfs = 1;

   // Copy the pointers to the integrators
-   dbfi = mbf->dbfi;
-   bbfi = mbf->bbfi;
-   tfbfi = mbf->tfbfi;
-   btfbfi = mbf->btfbfi;
-
-   bbfi_marker = mbf->bbfi_marker;
-   btfbfi_marker = mbf->btfbfi_marker;
-
-   assembly = AssemblyLevel::FULL;
-   ext = NULL;
-}
-
-void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
-{
-   if (ext)
-   {
-      MFEM_ABORT("the assembly level has already been set!");
-   }
-   assembly = assembly_level;
-   switch (assembly)
-   {
-      case AssemblyLevel::FULL:
-         // ext = new FAMixedBilinearFormExtension(this);
-         // Use the original BilinearForm implementation for now
-         break;
-      case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
-         // ext = new EAMixedBilinearFormExtension(this);
-         break;
-      case AssemblyLevel::PARTIAL:
-         ext = new PAMixedBilinearFormExtension(this);
-         break;
-      case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         // ext = new MFMixedBilinearFormExtension(this);
-         break;
-      default:
-         mfem_error("Unknown assembly level");
-   }
+   dom = mbf->dom;
+   bdr = mbf->bdr;
+   skt = mbf->skt;
 }

 double & MixedBilinearForm::Elem (int i, int j)
@@ -1140,63 +1040,31 @@ const double & MixedBilinearForm::Elem (int i, int j) const
   return (*mat)(i, j);
 }

-void MixedBilinearForm::Mult(const Vector & x, Vector & y) const
+void MixedBilinearForm::Mult (const Vector & x, Vector & y) const
 {
-   y = 0.0;
-   AddMult(x, y);
+   mat -> Mult (x, y);
 }

-void MixedBilinearForm::AddMult(const Vector & x, Vector & y,
-                                const double a) const
+void MixedBilinearForm::AddMult (const Vector & x, Vector & y,
+                                 const double a) const
 {
-   if (ext)
-   {
-      ext->AddMult(x, y, a);
-   }
-   else
-   {
-      mat->AddMult(x, y, a);
-   }
+   mat -> AddMult (x, y, a);
 }

-void MixedBilinearForm::MultTranspose(const Vector & x, Vector & y) const
+void MixedBilinearForm::AddMultTranspose (const Vector & x, Vector & y,
+                                          const double a) const
 {
-   y = 0.0;
-   AddMultTranspose(x, y);
-}
-
-void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,
-                                         const double a) const
-{
-   if (ext)
-   {
-      ext->AddMultTranspose(x, y, a);
-   }
-   else
-   {
-      mat->AddMultTranspose(x, y, a);
-   }
+   mat -> AddMultTranspose (x, y, a);
 }

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::FULL)
-   {
-      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
-      return NULL;
-   }
-   else
-   {
-      return mat -> Inverse ();
-   }
+   return mat -> Inverse ();
 }

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::FULL)
-   {
-      mat -> Finalize (skip_zeros);
-   }
+   mat -> Finalize (skip_zeros);
 }

 void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
@@ -1213,48 +1081,22 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const

 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
-   dbfi.Append (bfi);
+   dom.Append (bfi);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
 {
-   bbfi.Append (bfi);
-   bbfi_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                                               Array<int> &bdr_marker)
-{
-   bbfi.Append (bfi);
-   bbfi_marker.Append(&bdr_marker);
+   bdr.Append (bfi);
 }

 void MixedBilinearForm::AddTraceFaceIntegrator (BilinearFormIntegrator * bfi)
 {
-   tfbfi.Append (bfi);
-}
-
-void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi)
-{
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(NULL); // NULL marker means apply everywhere
-}
-
-void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
-                                                  Array<int> &bdr_marker)
-{
-   btfbfi.Append(bfi);
-   btfbfi_marker.Append(&bdr_marker);
+   skt.Append (bfi);
 }

 void MixedBilinearForm::Assemble (int skip_zeros)
 {
-   if (ext)
-   {
-      ext->Assemble();
-      return;
-   }
-
+   int i, k;
   Array<int> tr_vdofs, te_vdofs;
   ElementTransformation *eltrans;
   DenseMatrix elemmat;
@@ -1266,75 +1108,48 @@ void MixedBilinearForm::Assemble (int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (dbfi.Size())
+   if (dom.Size())
   {
-      for (int i = 0; i < test_fes -> GetNE(); i++)
+      for (i = 0; i < test_fes -> GetNE(); i++)
      {
         trial_fes -> GetElementVDofs (i, tr_vdofs);
         test_fes  -> GetElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
-         for (int k = 0; k < dbfi.Size(); k++)
+         for (k = 0; k < dom.Size(); k++)
         {
-            dbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                               *test_fes  -> GetFE(i),
-                                               *eltrans, elemmat);
+            dom[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                              *test_fes  -> GetFE(i),
+                                              *eltrans, elemmat);
            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (bbfi.Size())
+   if (bdr.Size())
   {
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < bbfi.Size(); k++)
+      for (i = 0; i < test_fes -> GetNBE(); i++)
      {
-         if (bbfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *bbfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < test_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
         trial_fes -> GetBdrElementVDofs (i, tr_vdofs);
         test_fes  -> GetBdrElementVDofs (i, te_vdofs);
         eltrans = test_fes -> GetBdrElementTransformation (i);
-         for (int k = 0; k < bbfi.Size(); k++)
+         for (k = 0; k < bdr.Size(); k++)
         {
-            if (bbfi_marker[k] &&
-                (*bbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-            bbfi[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
-                                               *test_fes  -> GetBE(i),
-                                               *eltrans, elemmat);
+            bdr[k] -> AssembleElementMatrix2 (*trial_fes -> GetBE(i),
+                                              *test_fes  -> GetBE(i),
+                                              *eltrans, elemmat);
            mat -> AddSubMatrix (te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }

-   if (tfbfi.Size())
+   if (skt.Size())
   {
      FaceElementTransformations *ftr;
      Array<int> te_vdofs2;
      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;

      int nfaces = mesh->GetNumFaces();
-      for (int i = 0; i < nfaces; i++)
+      for (i = 0; i < nfaces; i++)
      {
         ftr = mesh->GetFaceElementTransformations(i);
         trial_fes->GetFaceVDofs(i, tr_vdofs);
@@ -1354,80 +1169,18 @@ void MixedBilinearForm::Assemble (int skip_zeros)
            // want to actually make a fake element.
            test_fe2 = test_fe1;
         }
-         for (int k = 0; k < tfbfi.Size(); k++)
+         for (int k = 0; k < skt.Size(); k++)
         {
-            tfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                         *ftr, elemmat);
+            skt[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
+                                       *ftr, elemmat);
            mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
         }
      }
   }
-
-   if (btfbfi.Size())
-   {
-      FaceElementTransformations *ftr;
-      Array<int> te_vdofs2;
-      const FiniteElement *trial_face_fe, *test_fe1, *test_fe2;
-
-      // Which boundary attributes need to be processed?
-      Array<int> bdr_attr_marker(mesh->bdr_attributes.Size() ?
-                                 mesh->bdr_attributes.Max() : 0);
-      bdr_attr_marker = 0;
-      for (int k = 0; k < btfbfi.Size(); k++)
-      {
-         if (btfbfi_marker[k] == NULL)
-         {
-            bdr_attr_marker = 1;
-            break;
-         }
-         Array<int> &bdr_marker = *btfbfi_marker[k];
-         MFEM_ASSERT(bdr_marker.Size() == bdr_attr_marker.Size(),
-                     "invalid boundary marker for boundary trace face integrator #"
-                     << k << ", counting from zero");
-         for (int i = 0; i < bdr_attr_marker.Size(); i++)
-         {
-            bdr_attr_marker[i] |= bdr_marker[i];
-         }
-      }
-
-      for (int i = 0; i < trial_fes -> GetNBE(); i++)
-      {
-         const int bdr_attr = mesh->GetBdrAttribute(i);
-         if (bdr_attr_marker[bdr_attr-1] == 0) { continue; }
-
-         ftr = mesh->GetBdrFaceTransformations(i);
-         if (ftr)
-         {
-            trial_fes->GetFaceVDofs(i, tr_vdofs);
-            test_fes->GetElementVDofs(ftr->Elem1No, te_vdofs);
-            trial_face_fe = trial_fes->GetFaceElement(i);
-            test_fe1 = test_fes->GetFE(ftr->Elem1No);
-            // The test_fe2 object is really a dummy and not used on the
-            // boundaries, but we can't dereference a NULL pointer, and we don't
-            // want to actually make a fake element.
-            test_fe2 = test_fe1;
-            for (int k = 0; k < btfbfi.Size(); k++)
-            {
-               if (btfbfi_marker[k] &&
-                   (*btfbfi_marker[k])[bdr_attr-1] == 0) { continue; }
-
-               btfbfi[k]->AssembleFaceMatrix(*trial_face_fe, *test_fe1, *test_fe2,
-                                             *ftr, elemmat);
-               mat->AddSubMatrix(te_vdofs, tr_vdofs, elemmat, skip_zeros);
-            }
-         }
-      }
-   }
 }

 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::FULL)
-   {
-      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
-      return;
-   }
-
   Finalize();

   const SparseMatrix *P2 = test_fes->GetConformingProlongation();
@@ -1452,93 +1205,8 @@ void MixedBilinearForm::ConformingAssemble()
   width = mat->Width();
 }

-
-void MixedBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (dbfi.Size())
-   {
-      const FiniteElement &trial_fe = *trial_fes->GetFE(i);
-      const FiniteElement &test_fe = *test_fes->GetFE(i);
-      ElementTransformation *eltrans = test_fes->GetElementTransformation(i);
-      dbfi[0]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elmat);
-      for (int k = 1; k < dbfi.Size(); k++)
-      {
-         dbfi[k]->AssembleElementMatrix2(trial_fe, test_fe, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      trial_fes->GetElementVDofs(i, trial_vdofs);
-      test_fes->GetElementVDofs(i, test_vdofs);
-      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::ComputeBdrElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (bbfi.Size())
-   {
-      const FiniteElement &trial_be = *trial_fes->GetBE(i);
-      const FiniteElement &test_be = *test_fes->GetBE(i);
-      ElementTransformation *eltrans = test_fes->GetBdrElementTransformation(i);
-      bbfi[0]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elmat);
-      for (int k = 1; k < bbfi.Size(); k++)
-      {
-         bbfi[k]->AssembleElementMatrix2(trial_be, test_be, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-      test_fes->GetBdrElementVDofs(i, test_vdofs);
-      elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
-      elmat = 0.0;
-   }
-}
-
-void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleElementMatrix(i, elmat, trial_vdofs, test_vdofs, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
-{
-   trial_fes->GetElementVDofs(i, trial_vdofs);
-   test_fes->GetElementVDofs(i, test_vdofs);
-   if (mat == NULL)
-   {
-      mat = new SparseMatrix(height, width);
-   }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, int skip_zeros)
-{
-   AssembleBdrElementMatrix(i, elmat, trial_vdofs, test_vdofs, skip_zeros);
-}
-
-void MixedBilinearForm::AssembleBdrElementMatrix(
-   int i, const DenseMatrix &elmat, Array<int> &trial_vdofs,
-   Array<int> &test_vdofs, int skip_zeros)
-{
-   trial_fes->GetBdrElementVDofs(i, trial_vdofs);
-   test_fes->GetBdrElementVDofs(i, test_vdofs);
-   if (mat == NULL)
-   {
-      mat = new SparseMatrix(height, width);
-   }
-   mat->AddSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
-}
-
 void MixedBilinearForm::EliminateTrialDofs (
-   const Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
+   Array<int> &bdr_attr_is_ess, const Vector &sol, Vector &rhs )
 {
   int i, j, k;
   Array<int> tr_vdofs, cols_marker (trial_fes -> GetVSize());
@@ -1561,12 +1229,12 @@ void MixedBilinearForm::EliminateTrialDofs (
 }

 void MixedBilinearForm::EliminateEssentialBCFromTrialDofs (
-   const Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
+   Array<int> &marked_vdofs, const Vector &sol, Vector &rhs)
 {
   mat -> EliminateCols (marked_vdofs, &sol, &rhs);
 }

-void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
+void MixedBilinearForm::EliminateTestDofs (Array<int> &bdr_attr_is_ess)
 {
   int i, j, k;
   Array<int> te_vdofs;
@@ -1586,100 +1254,24 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
      }
 }

-void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    OperatorHandle &A)
-
-{
-   if (ext)
-   {
-      ext->FormRectangularSystemOperator(trial_tdof_list, test_tdof_list, A);
-      return;
-   }
-
-   const SparseMatrix *test_P = test_fes->GetConformingProlongation();
-   const SparseMatrix *trial_P = trial_fes->GetConformingProlongation();
-
-   mat->Finalize();
-
-   if (test_P) // TODO: Must actually check for trial_P too
-   {
-      SparseMatrix *m = RAP(*test_P, *mat, *trial_P);
-      delete mat;
-      mat = m;
-   }
-
-   Array<int> ess_trial_tdof_marker, ess_test_tdof_marker;
-   FiniteElementSpace::ListToMarker(trial_tdof_list, trial_fes->GetTrueVSize(),
-                                    ess_trial_tdof_marker);
-   FiniteElementSpace::ListToMarker(test_tdof_list, test_fes->GetTrueVSize(),
-                                    ess_test_tdof_marker);
-
-   mat_e = new SparseMatrix(mat->Height(), mat->Width());
-   mat->EliminateCols(ess_trial_tdof_marker, *mat_e);
-
-   for (int i=0; i<test_tdof_list.Size(); ++i)
-   {
-      mat->EliminateRow(test_tdof_list[i]);
-   }
-   mat_e->Finalize();
-   A.Reset(mat, false);
-}
-
-void MixedBilinearForm::FormRectangularLinearSystem(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    Vector &x, Vector &b,
-                                                    OperatorHandle &A,
-                                                    Vector &X, Vector &B)
-{
-   if (ext)
-   {
-      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
-                                       B);
-      return;
-   }
-
-   const Operator *Po = this->GetOutputProlongation();
-   const Operator *Ri = this->GetRestriction();
-   InitTVectors(Po, Ri, x, b, X, B);
-
-   if (!mat_e)
-   {
-      FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list,
-                                  A); // Set A = mat_e
-   }
-   // Eliminate essential BCs with B -= Ab xb
-   mat_e->AddMult(X, B, -1.0);
-
-   B.SetSubVector(test_tdof_list, 0.0);
-}
-
 void MixedBilinearForm::Update()
 {
   delete mat;
   mat = NULL;
-   delete mat_e;
-   mat_e = NULL;
   height = test_fes->GetVSize();
   width = trial_fes->GetVSize();
-   if (ext) { ext->Update(); }
 }

 MixedBilinearForm::~MixedBilinearForm()
 {
   if (mat) { delete mat; }
-   if (mat_e) { delete mat_e; }
   if (!extern_bfs)
   {
      int i;
-      for (i = 0; i < dbfi.Size(); i++) { delete dbfi[i]; }
-      for (i = 0; i < bbfi.Size(); i++) { delete bbfi[i]; }
-      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
-      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
+      for (i = 0; i < dom.Size(); i++) { delete dom[i]; }
+      for (i = 0; i < bdr.Size(); i++) { delete bdr[i]; }
+      for (i = 0; i < skt.Size(); i++) { delete skt[i]; }
   }
-   delete ext;
 }


@@ -1695,7 +1287,7 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      mat = new SparseMatrix(height, width);
   }

-   if (dbfi.Size() > 0)
+   if (dom.Size() > 0)
   {
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
@@ -1705,17 +1297,17 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
         dom_fe = trial_fes->GetFE(i);
         ran_fe = test_fes->GetFE(i);

-         dbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < dbfi.Size(); j++)
+         dom[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < dom.Size(); j++)
         {
-            dbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            dom[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
            totelmat += elmat;
         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
      }
   }

-   if (tfbfi.Size())
+   if (skt.Size())
   {
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
@@ -1726,10 +1318,10 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
         dom_fe = trial_fes->GetFaceElement(i);
         ran_fe = test_fes->GetFaceElement(i);

-         tfbfi[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
-         for (int j = 1; j < tfbfi.Size(); j++)
+         skt[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, totelmat);
+         for (int j = 1; j < skt.Size(); j++)
         {
-            tfbfi[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
+            skt[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T, elmat);
            totelmat += elmat;
         }
         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
@@ -58,7 +58,7 @@ protected:
   /// FE space on which the form lives. Not owned.
   FiniteElementSpace *fes;

-   /// The assembly level of the form (full, partial, etc.)
+   /// The form assembly level (full, partial, etc.)
   AssemblyLevel assembly;
   /// Element batch size used in the form action (1, 8, num_elems, etc.)
   int batch;
@@ -153,9 +153,6 @@ public:
   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

-   /// Get the assembly level
-   AssemblyLevel GetAssemblyLevel() {return assembly;}
-
   /** Enable the use of static condensation. For details see the description
       for class StaticCondensation in fem/staticcond.hpp This method should be
       called before assembly. If the number of unknowns after static
@@ -230,7 +227,7 @@ public:
   virtual const double &Elem(int i, int j) const;

   /// Matrix vector multiplication.
-   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void Mult(const Vector &x, Vector &y) const { mat->Mult(x, y); }

   void FullMult(const Vector &x, Vector &y) const
   { mat->Mult(x, y); mat_e->AddMult(x, y); }
@@ -322,26 +319,12 @@ public:
   /// Assembles the form i.e. sums over all domain/bdr integrators.
   void Assemble(int skip_zeros = 1);

-   /** @brief Assemble the diagonal of the bilinear form into diag
-
-       For adaptively refined meshes, this returns P^T d_e, where d_e is the
-       locally assembled diagonal on each element and P^T is the transpose of
-       the conforming prolongation. In general this is not the correct diagonal
-       for an AMR mesh. */
-   void AssembleDiagonal(Vector &diag) const;
-
   /// Get the finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return fes->GetConformingProlongation(); }
   /// Get the finite element space restriction matrix
   virtual const Operator *GetRestriction() const
   { return fes->GetConformingRestriction(); }
-   /// Get the output finite element space prolongation matrix
-   virtual const Operator *GetOutputProlongation() const
-   { return GetProlongation(); }
-   /// Get the output finite element space restriction matrix
-   virtual const Operator *GetOutputRestriction() const
-   { return GetRestriction(); }

   /** @brief Form the linear system A X = B, corresponding to this bilinear
       form and the linear form @a b(.). */
@@ -430,49 +413,9 @@ public:
   void FreeElementMatrices()
   { delete element_matrices; element_matrices = NULL; }

-   /// Compute the element matrix of the given element
-   /** The element matrix is computed by calling the domain integrators
-       or the one stored internally by a prior call of ComputeElementMatrices()
-       is returned when available.
-   */
   void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Compute the boundary element matrix of the given boundary element
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The flag @a skip_zeros skips the zero
-       elements of the matrix, unless they are breaking the symmetry of
-       the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              int skip_zeros = 1);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The vdofs of the element are returned
-       in @a vdofs. The flag @a skip_zeros skips the zero elements of the
-       matrix, unless they are breaking the symmetry of the system matrix.
-   */
   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
                              Array<int> &vdofs, int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The vdofs of the element
-       are returned in @a vdofs. The flag @a skip_zeros skips the zero elements
-       of the matrix, unless they are breaking the symmetry of the system matrix.
-   */
   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
                                 Array<int> &vdofs, int skip_zeros = 1);

@@ -542,9 +485,6 @@ public:
   /// Sets diagonal policy used upon construction of the linear system
   void SetDiagonalPolicy(DiagonalPolicy policy);

-   /// Indicate that integrators are not owned by the BilinearForm
-   void UseExternalIntegrators() { extern_bfs = 1; };
-
   /// Destroys bilinear form.
   virtual ~BilinearForm();
 };
@@ -569,37 +509,20 @@ class MixedBilinearForm : public Matrix
 {
 protected:
   SparseMatrix *mat; ///< Owned.
-   SparseMatrix *mat_e; ///< Owned.

   FiniteElementSpace *trial_fes, ///< Not owned
                      *test_fes;  ///< Not owned

-   /// The form assembly level (full, partial, etc.)
-   AssemblyLevel assembly;
-   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
-       Partial Assembly (PA), or Matrix Free assembly (MF). */
-   MixedBilinearFormExtension *ext;
-
-   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
-       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
+   /** @brief Indicates the BilinearFormIntegrator%s stored in #dom, #bdr, and
+       #skt are owned by another MixedBilinearForm. */
   int extern_bfs;

   /// Domain integrators.
-   Array<BilinearFormIntegrator*> dbfi;
-
+   Array<BilinearFormIntegrator*> dom;
   /// Boundary integrators.
-   Array<BilinearFormIntegrator*> bbfi;
-   Array<Array<int>*>             bbfi_marker;///< Entries are not owned.
-
+   Array<BilinearFormIntegrator*> bdr;
   /// Trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> tfbfi;
-
-   /// Boundary trace face (skeleton) integrators.
-   Array<BilinearFormIntegrator*> btfbfi;
-   Array<Array<int>*>             btfbfi_marker;///< Entries are not owned.
-
-   DenseMatrix elemmat;
-   Array<int>  trial_vdofs, test_vdofs;
+   Array<BilinearFormIntegrator*> skt;

 private:
   /// Copy construction is not supported; body is undefined.
@@ -634,13 +557,16 @@ public:
   virtual const double &Elem(int i, int j) const;

   virtual void Mult(const Vector & x, Vector & y) const;
+
   virtual void AddMult(const Vector & x, Vector & y,
                        const double a = 1.0) const;

-   virtual void MultTranspose(const Vector & x, Vector & y) const;
   virtual void AddMultTranspose(const Vector & x, Vector & y,
                                 const double a = 1.0) const;

+   virtual void MultTranspose(const Vector & x, Vector & y) const
+   { y = 0.0; AddMultTranspose (x, y); }
+
   virtual MatrixInverse *Inverse() const;

   virtual void Finalize(int skip_zeros = 1);
@@ -660,10 +586,6 @@ public:
   /// Adds a boundary integrator. Assumes ownership of @a bfi.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);

-   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                               Array<int> &bdr_marker);
-
   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.

       This type of integrator assembles terms over all faces of the mesh using
@@ -671,57 +593,19 @@ public:
       test space. */
   void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);

-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);
-
-   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
-                                   Array<int> &bdr_marker);
-
   /// Access all integrators added with AddDomainIntegrator().
-   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDBFI() { return &dom; }

   /// Access all integrators added with AddBoundaryIntegrator().
-   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
-   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBBFI_Marker() { return &bbfi_marker; }
+   Array<BilinearFormIntegrator*> *GetBBFI() { return &bdr; }

   /// Access all integrators added with AddTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetTFBFI() { return &tfbfi; }
-
-   /// Access all integrators added with AddBdrTraceFaceIntegrator().
-   Array<BilinearFormIntegrator*> *GetBTFBFI() { return &btfbfi; }
-   /** @brief Access all boundary markers added with AddBdrTraceFaceIntegrator().
-       If no marker was specified when the integrator was added, the
-       corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }
+   Array<BilinearFormIntegrator*> *GetTFBFI() { return &skt; }

   void operator=(const double a) { *mat = a; }

-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
-   /** This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level);
-
   void Assemble(int skip_zeros = 1);

-   /// Get the input finite element space prolongation matrix
-   virtual const Operator *GetProlongation() const
-   { return trial_fes->GetProlongationMatrix(); }
-
-   /// Get the input finite element space restriction matrix
-   virtual const Operator *GetRestriction() const
-   { return trial_fes->GetRestrictionMatrix(); }
-
-   /// Get the test finite element space prolongation matrix
-   virtual const Operator *GetOutputProlongation() const
-   { return test_fes->GetProlongationMatrix(); }
-
-   /// Get the test finite element space restriction matrix
-   virtual const Operator *GetOutputRestriction() const
-   { return test_fes->GetRestrictionMatrix(); }
-
   /** For partially conforming trial and/or test FE spaces, complete the
       assembly process by performing A := P2^t A P1 where A is the internal
       sparse matrix; P1 and P2 are the conforming prolongation matrices of the
@@ -729,135 +613,16 @@ public:
       MixedBilinearForm becomes an operator on the conforming FE spaces. */
   void ConformingAssemble();

-   /// Compute the element matrix of the given element
-   void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Compute the boundary element matrix of the given boundary element
-   void ComputeBdrElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The flag @a skip_zeros skips the zero
-       elements of the matrix, unless they are breaking the symmetry of
-       the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              int skip_zeros = 1);
-
-   /// Assemble the given element matrix
-   /** The element matrix @a elmat is assembled for the element @a i, i.e.
-       added to the system matrix. The vdofs of the element are returned
-       in @a trial_vdofs and @a test_vdofs. The flag @a skip_zeros skips
-       the zero elements of the matrix, unless they are breaking the symmetry
-       of the system matrix.
-   */
-   void AssembleElementMatrix(int i, const DenseMatrix &elmat,
-                              Array<int> &trial_vdofs, Array<int> &test_vdofs,
-                              int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 int skip_zeros = 1);
-
-   /// Assemble the given boundary element matrix
-   /** The boundary element matrix @a elmat is assembled for the boundary
-       element @a i, i.e. added to the system matrix. The vdofs of the element
-       are returned in @a trial_vdofs and @a test_vdofs. The flag @a skip_zeros
-       skips the zero elements of the matrix, unless they are breaking the
-       symmetry of the system matrix.
-   */
-   void AssembleBdrElementMatrix(int i, const DenseMatrix &elmat,
-                                 Array<int> &trial_vdofs, Array<int> &test_vdofs,
-                                 int skip_zeros = 1);
-
-   void EliminateTrialDofs(const Array<int> &bdr_attr_is_ess,
+   void EliminateTrialDofs(Array<int> &bdr_attr_is_ess,
                           const Vector &sol, Vector &rhs);

-   void EliminateEssentialBCFromTrialDofs(const Array<int> &marked_vdofs,
+   void EliminateEssentialBCFromTrialDofs(Array<int> &marked_vdofs,
                                          const Vector &sol, Vector &rhs);

-   virtual void EliminateTestDofs(const Array<int> &bdr_attr_is_ess);
-
-   /** @brief Return in @a A a parallel (on truedofs) version of this operator.
-
-      This returns the same operator as FormRectangularLinearSystem(), but does
-      without the transformations of the right-hand side. */
-   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    OperatorHandle &A);
-
-   /** @brief Form the column-constrained linear system matrix A.
-       See FormRectangularSystemMatrix() for details.
-
-       Version of the method FormRectangularSystemMatrix() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
-   template <typename OpType>
-   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list, OpType &A)
-   {
-      OperatorHandle Ah;
-      FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, Ah);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   /** @brief Form the linear system A X = B, corresponding to this mixed bilinear
-       form and the linear form @a b(.).
-
-       Return in @a A a *reference* to the system matrix that is column-constrained.
-       The reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
-   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OperatorHandle &A, Vector &X, Vector &B);
-
-   /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.).
-
-       Version of the method FormRectangularLinearSystem() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
-   template <typename OpType>
-   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OpType &A, Vector &X, Vector &B)
-   {
-      OperatorHandle Ah;
-      FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, Ah, X, B);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
+   virtual void EliminateTestDofs(Array<int> &bdr_attr_is_ess);

   void Update();

-   /// Return the trial FE space associated with the BilinearForm.
-   FiniteElementSpace *TrialFESpace() { return trial_fes; }
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return trial_fes; }
-
-   /// Return the test FE space associated with the BilinearForm.
-   FiniteElementSpace *TestFESpace() { return test_fes; }
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return test_fes; }
-
   virtual ~MixedBilinearForm();
 };

@@ -919,7 +684,7 @@ public:
   { AddTraceFaceIntegrator(di); }

   /// Access all interpolators added with AddDomainInterpolator().
-   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }
+   Array<BilinearFormIntegrator*> *GetDI() { return &dom; }

   /** @brief Construct the internal matrix representation of the discrete
       linear operator. */
@@ -14,7 +14,6 @@

 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
-#include "libceed/ceed.hpp"

 namespace mfem
 {
@@ -37,19 +36,16 @@ const Operator *BilinearFormExtension::GetRestriction() const


 // Data and methods for partially-assembled bilinear forms
-PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
-   : BilinearFormExtension(form),
-     trialFes(a->FESpace()),
-     testFes(a->FESpace())
+PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form) :
+   BilinearFormExtension(form),
+   trialFes(a->FESpace()), testFes(a->FESpace()),
+   localX(trialFes->GetNE() * trialFes->GetFE(0)->GetDof() * trialFes->GetVDim()),
+   localY( testFes->GetNE() * testFes->GetFE(0)->GetDof() * testFes->GetVDim()),
+   elem_restrict(new ElemRestriction(*a->FESpace())) { }
+
+PABilinearFormExtension::~PABilinearFormExtension()
 {
-   elem_restrict_lex = trialFes->GetElementRestriction(
-                          ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_lex)
-   {
-      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
-      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
-   }
+   delete elem_restrict;
 }

 void PABilinearFormExtension::Assemble()
@@ -58,32 +54,7 @@ void PABilinearFormExtension::Assemble()
   const int integratorCount = integrators.Size();
   for (int i = 0; i < integratorCount; ++i)
   {
-      integrators[i]->AssemblePA(*a->FESpace());
-   }
-}
-
-void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (elem_restrict_lex)
-   {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalPA(localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalPA(y);
-      }
+      integrators[i]->Assemble(*a->FESpace());
   }
 }

@@ -93,21 +64,23 @@ void PABilinearFormExtension::Update()
   height = width = fes->GetVSize();
   trialFes = fes;
   testFes = fes;
-   elem_restrict_lex = trialFes->GetElementRestriction(
-                          ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_lex)
-   {
-      localX.SetSize(elem_restrict_lex->Height());
-      localY.SetSize(elem_restrict_lex->Height());
-   }
+   localX.SetSize(trialFes->GetNE() * trialFes->GetFE(0)->GetDof() *
+                  trialFes->GetVDim());
+   localY.SetSize(testFes->GetNE() * testFes->GetFE(0)->GetDof() *
+                  testFes->GetVDim());
+   delete elem_restrict;
+   elem_restrict = new ElemRestriction(*fes);
 }

 void PABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                               OperatorHandle &A)
 {
-   Operator *oper;
-   Operator::FormSystemOperator(ess_tdof_list, oper);
-   A.Reset(oper); // A will own oper
+   const Operator* trialP = trialFes->GetProlongationMatrix();
+   const Operator* testP  = testFes->GetProlongationMatrix();
+   Operator *rap = this;
+   if (trialP) { rap = new RAPOperator(*testP, *this, *trialP); }
+   const bool own_A = (rap!=this);
+   A.Reset(new ConstrainedOperator(rap, ess_tdof_list, own_A));
 }

 void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
@@ -124,251 +97,140 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
+   elem_restrict->Mult(x, localX);
+   localY = 0.0;
   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict_lex)
+   for (int i = 0; i < iSz; ++i)
   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(x, y);
-      }
-   }
-   else
-   {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
+      integrators[i]->MultAssembled(localX, localY);
   }
+   elem_restrict->MultTranspose(localY, y);
 }

 void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
+   elem_restrict->Mult(x, localX);
+   localY = 0.0;
   const int iSz = integrators.Size();
-   if (elem_restrict_lex)
-   {
-      elem_restrict_lex->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposePA(localX, localY);
-      }
-      elem_restrict_lex->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true);
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposePA(x, y);
-      }
-   }
-}
-
-
-MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
-   : Operator(form->Height(), form->Width()), a(form)
-{
-   // empty
-}
-
-const Operator *MixedBilinearFormExtension::GetProlongation() const
-{
-   return a->GetProlongation();
-}
-
-const Operator *MixedBilinearFormExtension::GetRestriction() const
-{
-   return a->GetRestriction();
-}
-
-const Operator *MixedBilinearFormExtension::GetOutputProlongation() const
-{
-   return a->GetOutputProlongation();
-}
-
-const Operator *MixedBilinearFormExtension::GetOutputRestriction() const
-{
-   return a->GetOutputRestriction();
-}
-
-// Data and methods for partially-assembled bilinear forms
-
-PAMixedBilinearFormExtension::PAMixedBilinearFormExtension(
-   MixedBilinearForm *form)
-   : MixedBilinearFormExtension(form),
-     trialFes(form->TrialFESpace()),
-     testFes(form->TestFESpace()),
-     elem_restrict_trial(NULL),
-     elem_restrict_test(NULL)
-{
-   Update();
-}
-
-void PAMixedBilinearFormExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssemblePA(*trialFes, *testFes);
-   }
-}
-
-void PAMixedBilinearFormExtension::Update()
-{
-   trialFes = a->TrialFESpace();
-   testFes  = a->TestFESpace();
-   height = testFes->GetVSize();
-   width = trialFes->GetVSize();
-   elem_restrict_trial = trialFes->GetElementRestriction(
-                            ElementDofOrdering::LEXICOGRAPHIC);
-   elem_restrict_test  =  testFes->GetElementRestriction(
-                             ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_trial)
-   {
-      localTrial.UseDevice(true);
-      localTrial.SetSize(elem_restrict_trial->Height(), Device::GetMemoryType());
-
-   }
-   if (elem_restrict_test)
-   {
-      localTest.UseDevice(true); // ensure 'localY = 0.0' is done on device
-      localTest.SetSize(elem_restrict_test->Height(), Device::GetMemoryType());
-   }
-}
-
-void PAMixedBilinearFormExtension::FormRectangularSystemOperator(
-   const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list,
-   OperatorHandle &A)
-{
-   Operator * oper;
-   Operator::FormRectangularSystemOperator(trial_tdof_list, test_tdof_list, oper);
-   A.Reset(oper); // A will own oper
-}
-
-void PAMixedBilinearFormExtension::FormRectangularLinearSystem(
-   const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list,
-   Vector &x, Vector &b,
-   OperatorHandle &A,
-   Vector &X, Vector &B)
-{
-   Operator *oper;
-   Operator::FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b,
-                                         oper, X, B);
-   A.Reset(oper); // A will own oper
-}
-
-void PAMixedBilinearFormExtension::SetupMultInputs(const Operator
-                                                   *elem_restrict_x,
-                                                   const Vector &x,
-                                                   Vector &localX,
-                                                   const Operator *elem_restrict_y,
-                                                   Vector &y,
-                                                   Vector &localY,
-                                                   const double c) const
-{
-   // * G operation: localX = c*local(x)
-   if (elem_restrict_x)
-   {
-      elem_restrict_x->Mult(x, localX);
-      if (c != 1.0)
-      {
-         localX *= c;
-      }
-   }
-   else
-   {
-      if (c == 1.0)
-      {
-         localX.SyncAliasMemory(x);
-      }
-      else
-      {
-         localX.Set(c, x);
-      }
-   }
-   if (elem_restrict_y)
-   {
-      localY = 0.0;
-   }
-   else
-   {
-      y.UseDevice(true);
-      localY.SyncAliasMemory(y);
-   }
-}
-
-void PAMixedBilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   y = 0.0;
-   AddMult(x, y);
-}
-
-void PAMixedBilinearFormExtension::AddMult(const Vector &x, Vector &y,
-                                           const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // * G operation
-   SetupMultInputs(elem_restrict_trial, x, localTrial,
-                   elem_restrict_test, y, localTest, c);
-
-   // * B^TDB operation
   for (int i = 0; i < iSz; ++i)
   {
-      integrators[i]->AddMultPA(localTrial, localTest);
-   }
-
-   // * G^T operation
-   if (elem_restrict_test)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict_test->MultTranspose(localTest, tempY);
-      y += tempY;
+      integrators[i]->MultAssembledTranspose(localX, localY);
   }
+   elem_restrict->MultTranspose(localY, y);
 }

-void PAMixedBilinearFormExtension::MultTranspose(const Vector &x,
-                                                 Vector &y) const
+
+ElemRestriction::ElemRestriction(const FiniteElementSpace &f)
+   : fes(f),
+     ne(fes.GetNE()),
+     vdim(fes.GetVDim()),
+     byvdim(fes.GetOrdering() == Ordering::byVDIM),
+     ndofs(fes.GetNDofs()),
+     dof(fes.GetFE(0)->GetDof()),
+     nedofs(ne*dof),
+     offsets(ndofs+1),
+     indices(ne*dof)
 {
-   y = 0.0;
-   AddMultTranspose(x, y);
+   for (int e = 0; e < ne; ++e)
+   {
+      const FiniteElement *fe = fes.GetFE(e);
+      const TensorBasisElement* el =
+         dynamic_cast<const TensorBasisElement*>(fe);
+      if (el) { continue; }
+      mfem_error("Finite element not supported with partial assembly");
+   }
+   const FiniteElement *fe = fes.GetFE(0);
+   const TensorBasisElement* el = dynamic_cast<const TensorBasisElement*>(fe);
+   const Array<int> &dof_map = el->GetDofMap();
+   const bool dof_map_is_identity = (dof_map.Size()==0);
+   const Table& e2dTable = fes.GetElementToDofTable();
+   const int* elementMap = e2dTable.GetJ();
+   // We'll be keeping a count of how many local nodes point to its global dof
+   for (int i = 0; i <= ndofs; ++i)
+   {
+      offsets[i] = 0;
+   }
+   for (int e = 0; e < ne; ++e)
+   {
+      for (int d = 0; d < dof; ++d)
+      {
+         const int gid = elementMap[dof*e + d];
+         ++offsets[gid + 1];
+      }
+   }
+   // Aggregate to find offsets for each global dof
+   for (int i = 1; i <= ndofs; ++i)
+   {
+      offsets[i] += offsets[i - 1];
+   }
+   // For each global dof, fill in all local nodes that point   to it
+   for (int e = 0; e < ne; ++e)
+   {
+      for (int d = 0; d < dof; ++d)
+      {
+         const int did = dof_map_is_identity?d:dof_map[d];
+         const int gid = elementMap[dof*e + did];
+         const int lid = dof*e + d;
+         indices[offsets[gid]++] = lid;
+      }
+   }
+   // We shifted the offsets vector by 1 by using it as a counter
+   // Now we shift it back.
+   for (int i = ndofs; i > 0; --i)
+   {
+      offsets[i] = offsets[i - 1];
+   }
+   offsets[0] = 0;
 }

-void PAMixedBilinearFormExtension::AddMultTranspose(const Vector &x, Vector &y,
-                                                    const double c) const
+void ElemRestriction::Mult(const Vector& x, Vector& y) const
 {
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // * G operation
-   SetupMultInputs(elem_restrict_test, x, localTest,
-                   elem_restrict_trial, y, localTrial, c);
-
-   // * B^TD^TB operation
-   for (int i = 0; i < iSz; ++i)
+   const int vd = vdim;
+   const bool t = byvdim;
+   const DeviceArray d_offsets(offsets, ndofs+1);
+   const DeviceArray d_indices(indices, nedofs);
+   const DeviceMatrix d_x(x, t?vd:ndofs, t?ndofs:vd);
+   DeviceMatrix d_y(y, t?vd:nedofs, t?nedofs:vd);
+   MFEM_FORALL(i, ndofs,
   {
-      integrators[i]->AddMultTransposePA(localTest, localTrial);
-   }
+      const int offset = d_offsets[i];
+      const int nextOffset = d_offsets[i+1];
+      for (int c = 0; c < vd; ++c)
+      {
+         const double dofValue = d_x(t?c:i,t?i:c);
+         for (int j = offset; j < nextOffset; ++j)
+         {
+            const int idx_j = d_indices[j];
+            d_y(t?c:idx_j,t?idx_j:c) = dofValue;
+         }
+      }
+   });
+}

-   // * G^T operation
-   if (elem_restrict_trial)
+void ElemRestriction::MultTranspose(const Vector& x, Vector& y) const
+{
+   const int vd = vdim;
+   const bool t = byvdim;
+   const DeviceArray d_offsets(offsets, ndofs+1);
+   const DeviceArray d_indices(indices, nedofs);
+   const DeviceMatrix d_x(x, t?vd:nedofs, t?nedofs:vd);
+   DeviceMatrix d_y(y, t?vd:ndofs, t?ndofs:vd);
+   MFEM_FORALL(i, ndofs,
   {
-      tempY.SetSize(y.Size());
-      elem_restrict_trial->MultTranspose(localTrial, tempY);
-      y += tempY;
-   }
+      const int offset = d_offsets[i];
+      const int nextOffset = d_offsets[i + 1];
+      for (int c = 0; c < vd; ++c)
+      {
+         double dofValue = 0;
+         for (int j = offset; j < nextOffset; ++j)
+         {
+            const int idx_j = d_indices[j];
+            dofValue +=  d_x(t?c:idx_j,t?idx_j:c);
+         }
+         d_y(t?c:i,t?i:c) = dofValue;
+      }
+   });
 }

 } // namespace mfem
@@ -14,17 +14,32 @@

 #include "../config/config.hpp"
 #include "fespace.hpp"
-#include "../general/device.hpp"

 namespace mfem
 {

 class BilinearForm;
-class MixedBilinearForm;
+
+/// Element restriction operator
+class ElemRestriction: public Operator
+{
+public:
+   const FiniteElementSpace &fes;
+   const int ne;
+   const int vdim;
+   const bool byvdim;
+   const int ndofs;
+   const int dof;
+   const int nedofs;
+   Array<int> offsets;
+   Array<int> indices;
+public:
+   ElemRestriction(const FiniteElementSpace&);
+   void Mult(const Vector &x, Vector &y) const;
+   void MultTranspose(const Vector &x, Vector &y) const;
+};


-/** @brief Class extending the BilinearForm class to support the different
-    AssemblyLevel%s. */
 class BilinearFormExtension : public Operator
 {
 protected:
@@ -33,9 +48,6 @@ protected:
 public:
   BilinearFormExtension(BilinearForm *form);

-   virtual MemoryClass GetMemoryClass() const
-   { return Device::GetMemoryClass(); }
-
   /// Get the finite element space prolongation matrix
   virtual const Operator *GetProlongation() const;

@@ -43,10 +55,6 @@ public:
   virtual const Operator *GetRestriction() const;

   virtual void Assemble() = 0;
-   virtual void AssembleDiagonal(Vector &diag) const
-   {
-      MFEM_ABORT("AssembleDiagonal not implemented for this assembly level!");
-   }
   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
                                 OperatorHandle &A) = 0;
   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
@@ -72,7 +80,6 @@ public:
                         int copy_interior = 0) {}
   void Mult(const Vector &x, Vector &y) const {}
   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
   ~FABilinearFormExtension() {}
 };

@@ -92,7 +99,6 @@ public:
                         int copy_interior = 0) {}
   void Mult(const Vector &x, Vector &y) const {}
   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
   ~EABilinearFormExtension() {}
 };

@@ -100,15 +106,14 @@ public:
 class PABilinearFormExtension : public BilinearFormExtension
 {
 protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
+   const FiniteElementSpace *trialFes, *testFes;
   mutable Vector localX, localY;
-   const Operator *elem_restrict_lex; // Not owned
+   ElemRestriction *elem_restrict;

 public:
   PABilinearFormExtension(BilinearForm*);

   void Assemble();
-   void AssembleDiagonal(Vector &diag) const;
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
   void FormLinearSystem(const Array<int> &ess_tdof_list,
                         Vector &x, Vector &b,
@@ -118,8 +123,9 @@ public:
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
   void Update();
-};

+   ~PABilinearFormExtension();
+};

 /// Data and methods for matrix-free bilinear forms
 class MFBilinearFormExtension : public BilinearFormExtension
@@ -137,102 +143,9 @@ public:
                         int copy_interior = 0) {}
   void Mult(const Vector &x, Vector &y) const {}
   void MultTranspose(const Vector &x, Vector &y) const {}
-   void Update() {}
   ~MFBilinearFormExtension() {}
 };

-
-/** @brief Class extending the MixedBilinearForm class to support the different
-    AssemblyLevel%s. */
-class MixedBilinearFormExtension : public Operator
-{
-protected:
-   MixedBilinearForm *a; ///< Not owned
-
-public:
-   MixedBilinearFormExtension(MixedBilinearForm *form);
-
-   virtual MemoryClass GetMemoryClass() const
-   { return Device::GetMemoryClass(); }
-
-   /// Get the finite element space prolongation matrix
-   virtual const Operator *GetProlongation() const;
-
-   /// Get the finite element space restriction matrix
-   virtual const Operator *GetRestriction() const;
-
-   /// Get the output finite element space restriction matrix
-   virtual const Operator *GetOutputProlongation() const;
-
-   /// Get the output finite element space restriction matrix
-   virtual const Operator *GetOutputRestriction() const;
-
-   virtual void Assemble() = 0;
-   virtual void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
-                                              const Array<int> &test_tdof_list,
-                                              OperatorHandle &A) = 0;
-   virtual void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                            const Array<int> &test_tdof_list,
-                                            Vector &x, Vector &b,
-                                            OperatorHandle &A, Vector &X, Vector &B) = 0;
-
-   virtual void AddMult(const Vector &x, Vector &y, const double c=1.0) const = 0;
-   virtual void AddMultTranspose(const Vector &x, Vector &y,
-                                 const double c=1.0) const = 0;
-   virtual void Update() = 0;
-};
-
-/// Data and methods for partially-assembled mixed bilinear forms
-class PAMixedBilinearFormExtension : public MixedBilinearFormExtension
-{
-protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
-   mutable Vector localTrial, localTest, tempY;
-   const Operator *elem_restrict_trial; // Not owned
-   const Operator *elem_restrict_test;  // Not owned
-private:
-   /// Helper function to set up inputs/outputs for Mult or MultTranspose
-   void SetupMultInputs(const Operator *elem_restrict_x,
-                        const Vector &x, Vector &localX,
-                        const Operator *elem_restrict_y,
-                        Vector &y, Vector &localY, const double c) const;
-
-public:
-   PAMixedBilinearFormExtension(MixedBilinearForm *form);
-
-   /// Partial assembly of all internal integrators
-   void Assemble();
-   /**
-      @brief Setup OperatorHandle A to contain constrained linear operator
-
-      OperatorHandle A contains matrix-free constrained operator formed for RAP
-      system where ess_tdof_list are in trial space and eliminated from
-      "columns" of A.
-   */
-   void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
-                                      const Array<int> &test_tdof_list,
-                                      OperatorHandle &A);
-   /**
-      Setup OperatorHandle A to contain constrained linear operator and
-      eliminate columns corresponding to essential dofs from system,
-      updating RHS B vector with the results.
-   */
-   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OperatorHandle &A, Vector &X, Vector &B);
-   /// y = A*x
-   void Mult(const Vector &x, Vector &y) const;
-   /// y += c*A*x
-   void AddMult(const Vector &x, Vector &y, const double c=1.0) const;
-   /// y = A^T*x
-   void MultTranspose(const Vector &x, Vector &y) const;
-   /// y += c*A^T*x
-   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
-   /// Update internals for when a new MixedBilinearForm is given to this class
-   void Update();
-};
-
 }

 #endif
@@ -19,33 +19,19 @@ using namespace std;

 namespace mfem
 {
-
-void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
+void BilinearFormIntegrator::Assemble(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::Assemble (...)\n"
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
-                                        const FiniteElementSpace&)
-{
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
-{
-   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA (...)\n"
-              "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
+void BilinearFormIntegrator::MultAssembled(Vector&, Vector&)
 {
   mfem_error ("BilinearFormIntegrator::MultAssembled (...)\n"
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
+void BilinearFormIntegrator::MultAssembledTranspose(Vector&, Vector&)
 {
   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose (...)\n"
               "   is not implemented for this class.");
@@ -392,73 +378,6 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
   }
 }

-
-void GradientIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe,
-   ElementTransformation &Trans,  DenseMatrix &elmat)
-{
-   int dim = test_fe.GetDim();
-   int trial_dof = trial_fe.GetDof();
-   int test_dof = test_fe.GetDof();
-   double c;
-   Vector d_col;
-
-   dshape.SetSize(trial_dof, dim);
-   gshape.SetSize(trial_dof, dim);
-   Jadj.SetSize(dim);
-   shape.SetSize(test_dof);
-   elmat.SetSize(dim * test_dof, trial_dof);
-
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
-                                                            Trans);
-
-   elmat = 0.0;
-   elmat_comp.SetSize(test_dof, trial_dof);
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      trial_fe.CalcDShape(ip, dshape);
-      test_fe.CalcShape(ip, shape);
-
-      Trans.SetIntPoint(&ip);
-      CalcAdjugate(Trans.Jacobian(), Jadj);
-
-      Mult(dshape, Jadj, gshape);
-
-      c = ip.weight;
-      if (Q)
-      {
-         c *= Q->Eval(Trans, ip);
-      }
-      shape *= c;
-
-      for (int d = 0; d < dim; ++d)
-      {
-         gshape.GetColumnReference(d, d_col);
-         MultVWt(shape, d_col, elmat_comp);
-         for (int jj = 0; jj < trial_dof; ++jj)
-         {
-            for (int ii = 0; ii < test_dof; ++ii)
-            {
-               elmat(d * test_dof + ii, jj) += elmat_comp(ii, jj);
-            }
-         }
-      }
-   }
-}
-
-const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
-                                                   &trial_fe,
-                                                   const FiniteElement &test_fe,
-                                                   ElementTransformation &Trans)
-{
-   int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder() + Trans.OrderJ();
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
  DenseMatrix &elmat )
@@ -478,7 +397,29 @@ void DiffusionIntegrator::AssembleElementMatrix
 #endif
   elmat.SetSize(nd);

-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -534,7 +475,28 @@ void DiffusionIntegrator::AssembleElementMatrix2(
 #endif
   elmat.SetSize(te_nd, tr_nd);

-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (trial_fe.Space() == FunctionSpace::Pk)
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
+      }
+      else
+      {
+         order = trial_fe.GetOrder() + test_fe.GetOrder() + dim - 1;
+      }
+
+      if (trial_fe.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(trial_fe.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+      }
+   }

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -589,7 +551,29 @@ void DiffusionIntegrator::AssembleElementVector(

   elvect.SetSize(nd);

-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (el.Space() == FunctionSpace::Pk)
+      {
+         order = 2*el.GetOrder() - 2;
+      }
+      else
+         // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
+      {
+         order = 2*el.GetOrder() + dim - 1;
+      }
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }

   elvect = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -749,27 +733,6 @@ double DiffusionIntegrator::ComputeFluxEnergy
   return energy;
 }

-const IntegrationRule &DiffusionIntegrator::GetRule(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe)
-{
-   int order;
-   if (trial_fe.Space() == FunctionSpace::Pk)
-   {
-      order = trial_fe.GetOrder() + test_fe.GetOrder() - 2;
-   }
-   else
-   {
-      // order = 2*el.GetOrder() - 2;  // <-- this seems to work fine too
-      order = trial_fe.GetOrder() + test_fe.GetOrder() + trial_fe.GetDim() - 1;
-   }
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-

 void MassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
@@ -785,7 +748,21 @@ void MassIntegrator::AssembleElementMatrix
   elmat.SetSize(nd);
   shape.SetSize(nd);

-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      // int order = 2 * el.GetOrder();
+      int order = 2 * el.GetOrder() + Trans.OrderW();
+
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
+   }

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -820,8 +797,13 @@ void MassIntegrator::AssembleElementMatrix2(
   shape.SetSize(tr_nd);
   te_shape.SetSize(te_nd);

-   const IntegrationRule *ir = IntRule ? IntRule :
-                               &GetRule(trial_fe, test_fe, Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
+
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -842,20 +824,6 @@ void MassIntegrator::AssembleElementMatrix2(
   }
 }

-const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
-                                               const FiniteElement &test_fe,
-                                               ElementTransformation &Trans)
-{
-   // int order = trial_fe.GetOrder() + test_fe.GetOrder();
-   const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
-
-   if (trial_fe.Space() == FunctionSpace::rQk)
-   {
-      return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
-   }
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-

 void BoundaryMassIntegrator::AssembleFaceMatrix(
   const FiniteElement &el1, const FiniteElement &el2,
@@ -901,16 +869,6 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
 }


-const IntegrationRule &ConvectionIntegrator::GetRule(const FiniteElement
-                                                     &trial_fe,
-                                                     const FiniteElement &test_fe,
-                                                     ElementTransformation &Trans)
-{
-   int order = Trans.OrderGrad(&trial_fe) + Trans.Order() + test_fe.GetOrder();
-
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
 void ConvectionIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -937,7 +895,7 @@ void ConvectionIntegrator::AssembleElementMatrix(
      ir = &IntRules.Get(el.GetGeomType(), order);
   }

-   Q->Eval(Q_ir, Trans, *ir);
+   Q.Eval(Q_ir, Trans, *ir);

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -978,7 +936,7 @@ void GroupConvectionIntegrator::AssembleElementMatrix(
      ir = &IntRules.Get(el.GetGeomType(), order);
   }

-   Q->Eval(Q_nodal, Trans, el.GetNodes()); // sets the size of Q_nodal
+   Q.Eval(Q_nodal, Trans, el.GetNodes()); // sets the size of Q_nodal

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
@@ -1459,7 +1417,7 @@ void DerivativeIntegrator::AssembleElementMatrix2 (
         dshapedxi(l) = dshapedxt(l,xi);
      }

-      shape *= Q->Eval(Trans,ip) * det * ip.weight;
+      shape *= Q.Eval(Trans,ip) * det * ip.weight;
      AddMultVWt (shape, dshapedxi, elmat);
   }
 }
@@ -1841,26 +1799,81 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
   const FiniteElement &trial_fe, const FiniteElement &test_fe,
   ElementTransformation &Trans, DenseMatrix &elmat)
 {
-   if (test_fe.GetRangeType() == FiniteElement::SCALAR
-       && trial_fe.GetRangeType() == FiniteElement::VECTOR)
+   if ( test_fe.GetRangeType() == FiniteElement::SCALAR && VQ )
   {
      // assume test_fe is scalar FE and trial_fe is vector FE
      int dim  = test_fe.GetDim();
      int trial_dof = trial_fe.GetDof();
      int test_dof = test_fe.GetDof();
      double w;
-      double Kv;
+
+      if (MQ)
+         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                    "   is not implemented for tensor materials");
+
+#ifdef MFEM_THREAD_SAFE
+      DenseMatrix trial_vshape(trial_dof, dim);
+      Vector shape(test_dof);
+      Vector D(dim);
+#else
+      trial_vshape.SetSize(trial_dof, dim);
+      shape.SetSize(test_dof);
+      D.SetSize(dim);
+#endif
+
+      elmat.SetSize (test_dof, trial_dof);
+
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order = (Trans.OrderW() + test_fe.GetOrder() + trial_fe.GetOrder());
+         ir = &IntRules.Get(test_fe.GetGeomType(), order);
+      }
+
+      elmat = 0.0;
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+
+         Trans.SetIntPoint (&ip);
+
+         trial_fe.CalcVShape(Trans, trial_vshape);
+         test_fe.CalcShape(ip, shape);
+
+         w = ip.weight * Trans.Weight();
+         VQ->Eval(D, Trans, ip);
+         D *= w;
+
+         for (int d = 0; d < dim; d++)
+         {
+            for (int j = 0; j < test_dof; j++)
+            {
+               for (int k = 0; k < trial_dof; k++)
+               {
+                  elmat(j, k) += D[d] * shape(j) * trial_vshape(k, d);
+               }
+            }
+         }
+      }
+   }
+   else if ( test_fe.GetRangeType() == FiniteElement::SCALAR )
+   {
+      // assume test_fe is scalar FE and trial_fe is vector FE
+      int dim  = test_fe.GetDim();
+      int trial_dof = trial_fe.GetDof();
+      int test_dof = test_fe.GetDof();
+      double w;
+
+      if (VQ || MQ)
+         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                    "   is not implemented for vector/tensor permeability");

 #ifdef MFEM_THREAD_SAFE
      DenseMatrix trial_vshape(trial_dof, dim);
      Vector shape(test_dof);
-      Vector D(VQ ? VQ->GetVDim() : 0);
-      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
      trial_vshape.SetSize(trial_dof, dim);
      shape.SetSize(test_dof);
-      D.SetSize(VQ ? VQ->GetVDim() : 0);
-      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif

      elmat.SetSize (dim*test_dof, trial_dof);
@@ -1883,64 +1896,24 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
         test_fe.CalcShape(ip, shape);

         w = ip.weight * Trans.Weight();
-         if (VQ)
+         if (Q)
         {
-            VQ->Eval(D, Trans, ip);
-            D *= w;
-            for (int d = 0; d < dim; d++)
-            {
-               for (int j = 0; j < test_dof; j++)
-               {
-                  for (int k = 0; k < trial_dof; k++)
-                  {
-                     elmat(d * test_dof + j, k) +=
-                        shape(j) * D(d) * trial_vshape(k, d);
-                  }
-               }
-            }
+            w *= Q -> Eval (Trans, ip);
         }
-         else if (MQ)
+
+         for (int d = 0; d < dim; d++)
         {
-            MQ->Eval(K, Trans, ip);
-            K *= w;
-            for (int d = 0; d < dim; d++)
+            for (int j = 0; j < test_dof; j++)
            {
-               for (int j = 0; j < test_dof; j++)
+               for (int k = 0; k < trial_dof; k++)
               {
-                  for (int k = 0; k < trial_dof; k++)
-                  {
-                     Kv = 0.0;
-                     for (int vd = 0; vd < dim; vd++)
-                     {
-                        Kv += K(d, vd) * trial_vshape(k, vd);
-                     }
-                     elmat(d * test_dof + j, k) += shape(j) * Kv;
-                  }
-               }
-            }
-         }
-         else
-         {
-            if (Q)
-            {
-               w *= Q->Eval(Trans, ip);
-            }
-            for (int d = 0; d < dim; d++)
-            {
-               for (int j = 0; j < test_dof; j++)
-               {
-                  for (int k = 0; k < trial_dof; k++)
-                  {
-                     elmat(d * test_dof + j, k) +=
-                        w * shape(j) * trial_vshape(k, d);
-                  }
+                  elmat(d * test_dof + j, k) += w * shape(j) * trial_vshape(k, d);
               }
            }
         }
      }
   }
-   else if (test_fe.GetRangeType() == FiniteElement::VECTOR
-            && trial_fe.GetRangeType() == FiniteElement::VECTOR)
+   else
   {
      // assume both test_fe and trial_fe are vector FE
      int dim  = test_fe.GetDim();
@@ -1948,18 +1921,17 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
      int test_dof = test_fe.GetDof();
      double w;

+      if (VQ || MQ)
+         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                    "   is not implemented for vector/tensor permeability");
+
 #ifdef MFEM_THREAD_SAFE
      DenseMatrix trial_vshape(trial_dof, dim);
      DenseMatrix test_vshape(test_dof,dim);
-      Vector D(VQ ? VQ->GetVDim() : 0);
-      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #else
      trial_vshape.SetSize(trial_dof, dim);
      test_vshape.SetSize(test_dof,dim);
-      D.SetSize(VQ ? VQ->GetVDim() : 0);
-      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
-      DenseMatrix tmp(trial_vshape.Height(), K.Width());

      elmat.SetSize (test_dof, trial_dof);

@@ -1981,34 +1953,23 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
         test_fe.CalcVShape(Trans, test_vshape);

         w = ip.weight * Trans.Weight();
-         if (MQ)
+         if (Q)
         {
-            MQ->Eval(K, Trans, ip);
-            K *= w;
-            Mult(test_vshape,K,tmp);
-            AddMultABt(tmp,trial_vshape,elmat);
+            w *= Q -> Eval (Trans, ip);
         }
-         else if (VQ)
+
+         for (int d = 0; d < dim; d++)
         {
-            VQ->Eval(D, Trans, ip);
-            D *= w;
-            AddMultADBt(test_vshape,D,trial_vshape,elmat);
-         }
-         else
-         {
-            if (Q)
+            for (int j = 0; j < test_dof; j++)
            {
-               w *= Q -> Eval (Trans, ip);
+               for (int k = 0; k < trial_dof; k++)
+               {
+                  elmat(j, k) += w * test_vshape(j, d) * trial_vshape(k, d);
+               }
            }
-            AddMult_a_ABt(w,test_vshape,trial_vshape,elmat);
         }
      }
   }
-   else
-   {
-      mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
-                 "   is not implemented for given trial and test bases.");
-   }
 }

 void VectorDivergenceIntegrator::AssembleElementMatrix2(
@@ -2030,8 +1991,12 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(

   elmat.SetSize (test_dof, dim*trial_dof);

-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
-                                                            Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }

   elmat = 0.0;

@@ -2061,15 +2026,6 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
   }
 }

-const IntegrationRule &VectorDivergenceIntegrator::GetRule(
-   const FiniteElement &trial_fe,
-   const FiniteElement &test_fe,
-   ElementTransformation &Trans)
-{
-   int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder() + Trans.OrderJ();
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-

 void DivDivIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -3307,7 +3263,7 @@ ScalarProductInterpolator::AssembleElementMatrix2(const FiniteElement &dom_fe,
                                                  ElementTransformation &Trans,
                                                  DenseMatrix &elmat)
 {
-   internal::ShapeCoefficient dom_shape_coeff(*Q, dom_fe);
+   internal::ShapeCoefficient dom_shape_coeff(Q, dom_fe);

   elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());

@@ -3342,7 +3298,7 @@ ScalarVectorProductInterpolator::AssembleElementMatrix2(
      }
   };

-   VShapeCoefficient dom_shape_coeff(*Q, dom_fe, Trans.GetSpaceDim());
+   VShapeCoefficient dom_shape_coeff(Q, dom_fe, Trans.GetSpaceDim());

   elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());

@@ -3380,7 +3336,7 @@ VectorScalarProductInterpolator::AssembleElementMatrix2(
      }
   };

-   VecShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   VecShapeCoefficient dom_shape_coeff(VQ, dom_fe);

   elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());

@@ -3427,11 +3383,11 @@ VectorCrossProductInterpolator::AssembleElementMatrix2(
      }
   };

-   VCrossVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   VCrossVShapeCoefficient dom_shape_coeff(VQ, dom_fe);

   if (ran_fe.GetRangeType() == FiniteElement::SCALAR)
   {
-      elmat.SetSize(ran_fe.GetDof()*VQ->GetVDim(),dom_fe.GetDof());
+      elmat.SetSize(ran_fe.GetDof()*VQ.GetVDim(),dom_fe.GetDof());
   }
   else
   {
@@ -3480,7 +3436,7 @@ VectorInnerProductInterpolator::AssembleElementMatrix2(
   ElementTransformation &Trans,
   DenseMatrix &elmat)
 {
-   internal::VDotVShapeCoefficient dom_shape_coeff(*VQ, dom_fe);
+   internal::VDotVShapeCoefficient dom_shape_coeff(VQ, dom_fe);

   elmat.SetSize(ran_fe.GetDof(),dom_fe.GetDof());

@@ -15,7 +15,7 @@
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
 #include "fespace.hpp"
-#include "libceed/ceed.hpp"
+#include "bilininteg_ext.hpp"

 namespace mfem
 {
@@ -23,53 +23,19 @@ namespace mfem
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
-protected:
-   BilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : NonlinearFormIntegrator(ir) { }
+public:
+   BilinearFormIntegrator(const IntegrationRule *ir = NULL) :
+      NonlinearFormIntegrator(ir) { }

 public:
-   // TODO: add support for other assembly levels (in addition to PA) and their
-   // actions.
-
-   // TODO: for mixed meshes the quadrature rules to be used by methods like
-   // AssemblePA() can be given as a QuadratureSpace, e.g. using a new method:
-   // SetQuadratureSpace().
-
-   // TODO: the methods for the various assembly levels make sense even in the
-   // base class NonlinearFormIntegrator, except that not all assembly levels
-   // make sense for the action of the nonlinear operator (but they all make
-   // sense for its Jacobian).
-
-   using NonlinearFormIntegrator::AssemblePA;
-
   /// Method defining partial assembly.
-   /** The result of the partial assembly is stored internally so that it can be
-       used later in the methods AddMultPA() and AddMultTransposePA(). */
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   /** Used with BilinearFormIntegrators that have different spaces. */
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   /// Assemble diagonal and add it to Vector @a diag.
-   virtual void AssembleDiagonalPA(Vector &diag);
+   virtual void Assemble(const FiniteElementSpace&);

   /// Method for partially assembled action.
-   /** Perform the action of integrator on the input @a x and add the result to
-       the output @a y. Both @a x and @a y are E-vectors, i.e. they represent
-       the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void MultAssembled(Vector&, Vector&);

   /// Method for partially assembled transposed action.
-   /** Perform the transpose action of integrator on the input @a x and add the
-       result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
-       represent the element-wise discontinuous version of the FE space.
-
-       This method can be called only after the method AssemblePA() has been
-       called. */
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
+   virtual void MultAssembledTranspose(Vector&, Vector&);

   /// Given a particular Finite Element computes the element matrix elmat.
   virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -318,10 +284,10 @@ protected:
                                      Vector & shape)
   { trial_fe.CalcPhysShape(Trans, shape); }

-   Coefficient *Q;
-
 private:

+   Coefficient *Q;
+
 #ifndef MFEM_THREAD_SAFE
   Vector test_shape;
   Vector trial_shape;
@@ -392,13 +358,13 @@ protected:
                                      DenseMatrix & shape)
   { trial_fe.CalcVShape(Trans, shape); }

+private:
+
   Coefficient *Q;
   VectorCoefficient *VQ;
   VectorCoefficient *DQ;
   MatrixCoefficient *MQ;

-private:
-
 #ifndef MFEM_THREAD_SAFE
   Vector V;
   Vector D;
@@ -473,12 +439,12 @@ protected:
                                 Vector & shape)
   { scalar_fe.CalcPhysShape(Trans, shape); }

+private:
+
   VectorCoefficient *VQ;
   bool transpose;
   bool cross_2d;  // In 2D use a cross product rather than a dot product

-private:
-
 #ifndef MFEM_THREAD_SAFE
   Vector V;
   DenseMatrix vshape;
@@ -1667,125 +1633,31 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same space
-    as u. */
-class GradientIntegrator : public BilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-
-private:
-   Vector shape;
-   DenseMatrix dshape;
-   DenseMatrix gshape;
-   DenseMatrix Jadj;
-   DenseMatrix elmat_comp;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *trial_maps, *test_maps; ///< Not owned
-   const GeometricFactors *geom;            ///< Not owned
-   int dim, ne, nq;
-   int trial_dofs1D, test_dofs1D, quad1D;
-
-public:
-   GradientIntegrator() :
-      Q{NULL}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
-   { }
-   GradientIntegrator(Coefficient *_q) :
-      Q{_q}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
-   { }
-   GradientIntegrator(Coefficient &q) :
-      Q{&q}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
-   { }
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
-};
-
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
    can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-   MatrixCoefficient *MQ;
-
 private:
   Vector vec, pointflux, shape;
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix dshape, dshapedxt, invdfdx, mq;
   DenseMatrix te_dshape, te_dshapedxt;
 #endif
-
+   Coefficient *Q;
+   MatrixCoefficient *MQ;
   // PA extension
-   const FiniteElementSpace *fespace;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
+   DofToQuad *maps;
+   GeometryExtension *geom;
   int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
-
-#ifdef MFEM_USE_CEED
-   // CEED extension
-   CeedData* ceedDataPtr;
-#endif
-
 public:
   /// Construct a diffusion integrator with coefficient Q = 1
-   DiffusionIntegrator()
-   {
-      Q = NULL;
-      MQ = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+   DiffusionIntegrator() { Q = NULL; MQ = NULL; maps = NULL; geom = NULL; }

   /// Construct a diffusion integrator with a scalar coefficient q
-   DiffusionIntegrator(Coefficient &q)
-      : Q(&q)
-   {
-      MQ = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+   DiffusionIntegrator (Coefficient &q) : Q(&q) { MQ = NULL; maps = NULL; geom = NULL; }

   /// Construct a diffusion integrator with a matrix coefficient q
-   DiffusionIntegrator(MatrixCoefficient &q)
-      : MQ(&q)
-   {
-      Q = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
-
-   virtual ~DiffusionIntegrator()
-   {
-#ifdef MFEM_USE_CEED
-      delete ceedDataPtr;
-#endif
-   }
+   DiffusionIntegrator (MatrixCoefficient &q) : MQ(&q) { Q = NULL; maps = NULL; geom = NULL; }

   /** Given a particular Finite Element
       computes the element stiffness matrix elmat. */
@@ -1813,18 +1685,11 @@ public:
                                    ElementTransformation &Trans,
                                    Vector &flux, Vector *d_energy = NULL);

-   using BilinearFormIntegrator::AssemblePA;
+   /// PA extension
+   virtual void Assemble(const FiniteElementSpace&);
+   virtual void MultAssembled(Vector&, Vector&);

-   virtual void AssemblePA(const FiniteElementSpace &fes);
-
-   virtual void AssembleDiagonalPA(Vector &diag);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe);
-
-   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
+   virtual ~DiffusionIntegrator();
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1836,46 +1701,17 @@ protected:
 #endif
   Coefficient *Q;
   // PA extension
-   const FiniteElementSpace *fespace;
-   Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
+   Vector vec;
+   DofToQuad *maps;
+   GeometryExtension *geom;
   int dim, ne, nq, dofs1D, quad1D;
-
-#ifdef MFEM_USE_CEED
-   // CEED extension
-   CeedData* ceedDataPtr;
-#endif
-
 public:
   MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir)
-   {
-      Q = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
-
+      : BilinearFormIntegrator(ir) { Q = NULL; maps = NULL; geom = NULL; }
   /// Construct a mass integrator with coefficient q
   MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q)
-   {
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : BilinearFormIntegrator(ir), Q(&q) { maps = NULL; geom = NULL; }

-   virtual ~MassIntegrator()
-   {
-#ifdef MFEM_USE_CEED
-      delete ceedDataPtr;
-#endif
-   }
   /** Given a particular Finite Element
       computes the element mass matrix elmat. */
   virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -1885,20 +1721,11 @@ public:
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
+   /// PA extension
+   virtual void Assemble(const FiniteElementSpace&);
+   virtual void MultAssembled(Vector&, Vector&);

-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-
-   virtual void AssembleDiagonalPA(Vector &diag);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
-
-   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
+   virtual ~MassIntegrator();
 };

 class BoundaryMassIntegrator : public MassIntegrator
@@ -1917,55 +1744,34 @@ public:
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   VectorCoefficient *Q;
-   double alpha;
-
 private:
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix dshape, adjJ, Q_ir;
   Vector shape, vec2, BdFidxT;
 #endif
-
-   // PA extension
-   Vector pa_data;
-   Vector coeff;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
+   VectorCoefficient &Q;
+   double alpha;

 public:
   ConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
-      : Q(&q) { alpha = a; }
+      : Q(q) { alpha = a; }
   virtual void AssembleElementMatrix(const FiniteElement &,
                                      ElementTransformation &,
                                      DenseMatrix &);
-
-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace&);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
 };

 /// alpha (q . grad u, v) using the "group" FE discretization
 class GroupConvectionIntegrator : public BilinearFormIntegrator
 {
-protected:
-   VectorCoefficient *Q;
-   double alpha;
-
 private:
   DenseMatrix dshape, adjJ, Q_nodal, grad;
   Vector shape;
+   VectorCoefficient &Q;
+   double alpha;

 public:
   GroupConvectionIntegrator(VectorCoefficient &q, double a = 1.0)
-      : Q(&q) { alpha = a; }
+      : Q(q) { alpha = a; }
   virtual void AssembleElementMatrix(const FiniteElement &,
                                      ElementTransformation &,
                                      DenseMatrix &);
@@ -1981,22 +1787,16 @@ private:
   Vector shape, te_shape, vec;
   DenseMatrix partelmat;
   DenseMatrix mcoeff;
-   int Q_order;
-
-protected:
   Coefficient *Q;
   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
+
+   int Q_order;

 public:
   /// Construct an integrator with coefficient 1.0
   VectorMassIntegrator()
-      : vdim(-1), Q_order(0), Q(NULL), VQ(NULL), MQ(NULL) { }
+      : vdim(-1), Q(NULL), VQ(NULL), MQ(NULL), Q_order(0) { }
   /** Construct an integrator with scalar coefficient q.
       If possible, save memory by using a scalar integrator since
       the resulting matrix is block diagonal with the same diagonal
@@ -2023,9 +1823,6 @@ public:
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
 };


@@ -2038,14 +1835,11 @@ public:
    does NOT depend on the ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
   Vector divshape, shape;
 #endif
-
 public:
   VectorFEDivergenceIntegrator() { Q = NULL; }
   VectorFEDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -2063,17 +1857,14 @@ public:
    This is equivalent to a weak divergence of the Nedelec basis functions. */
 class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix dshape;
   DenseMatrix dshapedxt;
   DenseMatrix vshape;
   DenseMatrix invdfdx;
 #endif
-
 public:
   VectorFEWeakDivergenceIntegrator() { Q = NULL; }
   VectorFEWeakDivergenceIntegrator(Coefficient &q) { Q = &q; }
@@ -2090,16 +1881,13 @@ public:
    test spaces are switched, assembles the form (u, curl v). */
 class VectorFECurlIntegrator: public BilinearFormIntegrator
 {
-protected:
-   Coefficient *Q;
-
 private:
+   Coefficient *Q;
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix curlshapeTrial;
   DenseMatrix vshapeTest;
   DenseMatrix curlshapeTrial_dFT;
 #endif
-
 public:
   VectorFECurlIntegrator() { Q = NULL; }
   VectorFECurlIntegrator(Coefficient &q) { Q = &q; }
@@ -2112,19 +1900,17 @@ public:
                                       DenseMatrix &elmat);
 };

+
 /// Class for integrating (Q D_i(u), v); u and v are scalars
 class DerivativeIntegrator : public BilinearFormIntegrator
 {
-protected:
-   Coefficient* Q;
-
 private:
+   Coefficient & Q;
   int xi;
   DenseMatrix dshape, dshapedxt, invdfdx;
   Vector shape, dshapedxi;
-
 public:
-   DerivativeIntegrator(Coefficient &q, int i) : Q(&q), xi(i) { }
+   DerivativeIntegrator(Coefficient &q, int i) : Q(q), xi(i) { }
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat)
@@ -2144,8 +1930,6 @@ private:
   DenseMatrix curlshape, curlshape_dFt, M;
   DenseMatrix vshape, projcurl;
 #endif
-
-protected:
   Coefficient *Q;
   MatrixCoefficient *MQ;

@@ -2179,8 +1963,6 @@ private:
 #ifndef MFEM_THREAD_SAFE
   DenseMatrix dshape_hat, dshape, curlshape, Jadj, grad_hat, grad;
 #endif
-
-protected:
   Coefficient *Q;

 public:
@@ -2198,13 +1980,13 @@ public:
                                   const Vector &elfun);
 };

-/** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
-    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
-    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
-    vi are in H1. */
+/// Integrator for (Q u, v) for VectorFiniteElements
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
+   Coefficient *Q;
+   VectorCoefficient *VQ;
+   MatrixCoefficient *MQ;
   void Init(Coefficient *q, VectorCoefficient *vq, MatrixCoefficient *mq)
   { Q = q; VQ = vq; MQ = mq; }

@@ -2212,16 +1994,10 @@ private:
   Vector shape;
   Vector D;
   DenseMatrix K;
-   DenseMatrix partelmat;
   DenseMatrix test_vshape;
   DenseMatrix trial_vshape;
 #endif

-protected:
-   Coefficient *Q;
-   VectorCoefficient *VQ;
-   MatrixCoefficient *MQ;
-
 public:
   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
   VectorFEMassIntegrator(Coefficient *_q) { Init(_q, NULL, NULL); }
@@ -2244,57 +2020,32 @@ public:
    scalar FE space; p is also in a (different) scalar FE space.  */
 class VectorDivergenceIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
   Coefficient *Q;

-private:
   Vector shape;
   Vector divshape;
   DenseMatrix dshape;
   DenseMatrix gshape;
   DenseMatrix Jadj;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *trial_maps, *test_maps; ///< Not owned
-   const GeometricFactors *geom;            ///< Not owned
-   int dim, ne, nq;
-   int trial_dofs1D, test_dofs1D, quad1D;

 public:
-   VectorDivergenceIntegrator() :
-      Q(NULL), trial_maps(NULL), test_maps(NULL), geom(NULL)
-   {  }
-   VectorDivergenceIntegrator(Coefficient *_q) :
-      Q(_q), trial_maps(NULL), test_maps(NULL), geom(NULL)
-   { }
-   VectorDivergenceIntegrator(Coefficient &q) :
-      Q(&q), trial_maps(NULL), test_maps(NULL), geom(NULL)
-   { }
+   VectorDivergenceIntegrator() { Q = NULL; }
+   VectorDivergenceIntegrator(Coefficient *_q) { Q = _q; }
+   VectorDivergenceIntegrator(Coefficient &q) { Q = &q; }

   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
 };

 /// (Q div u, div v) for RT elements
 class DivDivIntegrator: public BilinearFormIntegrator
 {
-protected:
+private:
   Coefficient *Q;

-private:
 #ifndef MFEM_THREAD_SAFE
   Vector divshape;
 #endif
@@ -2316,16 +2067,9 @@ public:
    diffusion matrix in each diagonal block. */
 class VectorDiffusionIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
   Coefficient *Q;

-   // PA extension
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
-
-private:
   DenseMatrix Jinv;
   DenseMatrix dshape;
   DenseMatrix gshape;
@@ -2341,9 +2085,6 @@ public:
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &Tr,
                                      const Vector &elfun, Vector &elvect);
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
 };

 /** Integrator for the linear elasticity form:
@@ -2353,11 +2094,10 @@ public:
    using multiple copies of a scalar FE space. */
 class ElasticityIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
   double q_lambda, q_mu;
   Coefficient *lambda, *mu;

-private:
 #ifndef MFEM_THREAD_SAFE
   Vector shape;
   DenseMatrix dshape, gshape, pelmat;
@@ -2414,12 +2154,11 @@ public:
    points. */
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
-protected:
+private:
   Coefficient *rho;
   VectorCoefficient *u;
   double alpha, beta;

-private:
   Vector shape1, shape2;

 public:
@@ -2706,7 +2445,7 @@ public:
 class ScalarProductInterpolator : public DiscreteInterpolator
 {
 public:
-   ScalarProductInterpolator(Coefficient & sc) : Q(&sc) { }
+   ScalarProductInterpolator(Coefficient & sc) : Q(sc) { }

   virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                       const FiniteElement &ran_fe,
@@ -2714,7 +2453,7 @@ public:
                                       DenseMatrix &elmat);

 protected:
-   Coefficient *Q;
+   Coefficient &Q;
 };

 /** Interpolator of a scalar coefficient multiplied by a vector field onto
@@ -2724,14 +2463,14 @@ class ScalarVectorProductInterpolator : public DiscreteInterpolator
 {
 public:
   ScalarVectorProductInterpolator(Coefficient & sc)
-      : Q(&sc) { }
+      : Q(sc) { }

   virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                       const FiniteElement &ran_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
 protected:
-   Coefficient *Q;
+   Coefficient &Q;
 };

 /** Interpolator of a vector coefficient multiplied by a scalar field onto
@@ -2741,14 +2480,14 @@ class VectorScalarProductInterpolator : public DiscreteInterpolator
 {
 public:
   VectorScalarProductInterpolator(VectorCoefficient & vc)
-      : VQ(&vc) { }
+      : VQ(vc) { }

   virtual void AssembleElementMatrix2(const FiniteElement &dom_fe,
                                       const FiniteElement &ran_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };

 /** Interpolator of the cross product between a vector coefficient and an
@@ -2758,14 +2497,14 @@ class VectorCrossProductInterpolator : public DiscreteInterpolator
 {
 public:
   VectorCrossProductInterpolator(VectorCoefficient & vc)
-      : VQ(&vc) { }
+      : VQ(vc) { }

   virtual void AssembleElementMatrix2(const FiniteElement &nd_fe,
                                       const FiniteElement &rt_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };

 /** Interpolator of the inner product between a vector coefficient and an
@@ -2774,14 +2513,14 @@ protected:
 class VectorInnerProductInterpolator : public DiscreteInterpolator
 {
 public:
-   VectorInnerProductInterpolator(VectorCoefficient & vc) : VQ(&vc) { }
+   VectorInnerProductInterpolator(VectorCoefficient & vc) : VQ(vc) { }

   virtual void AssembleElementMatrix2(const FiniteElement &rt_fe,
                                       const FiniteElement &l2_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
 protected:
-   VectorCoefficient *VQ;
+   VectorCoefficient &VQ;
 };

 }
@@ -1,500 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Convection Integrator
-
-// PA Convection Assemble kernel
-void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   // Assuming the same element type
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
-   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(dim*ne*nq, Device::GetMemoryType());
-
-   const int NE = ne;
-   const int NQ = nq;
-   Vector e_coeff(dim);
-   coeff.SetSize(dim*ne*nq);
-   auto h_C = Reshape(coeff.HostWrite(),dim,nq, ne);
-
-   if ( Q == nullptr)
-   {
-      for (int e=0; e<NE; ++e)
-      {
-         for (int q=0; q<nq; ++q)
-         {
-            for (int idim=0; idim < dim; ++idim)
-            {
-               h_C(idim,q,e) = alpha;
-            }
-         }
-      }
-   }
-   else
-   {
-      for (int e=0; e<NE; ++e)
-      {
-         ElementTransformation& Te = *fes.GetElementTransformation(e);
-         for (int q=0; q<nq; ++q)
-         {
-            for (int idim=0; idim < dim; ++idim)
-            {
-               Q->Eval(e_coeff, Te, ir->IntPoint(q));
-               h_C(idim,q,e) = alpha*e_coeff(idim);
-            }
-         }
-      }
-   }
-
-   auto C = Reshape(coeff.Read(),dim,nq, ne);
-
-   if (dim==2)
-   {
-      auto w = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
-      auto v = Reshape(pa_data.Write(), 2, NQ, NE);
-
-      MFEM_FORALL(e, NE,
-      {
-         for (int q=0; q<NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e);
-            const double J21 = J(q,1,0,e);
-            const double J12 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-
-            const double cx = C(0,q,e);
-            const double cy = C(1,q,e);
-            const double w_coeff = w[q];
-            v(0,q,e) =    w_coeff*(cx * J22 - cy * J12);
-            v(1,q,e) =  - w_coeff*(cx * J21 - cy * J11);
-         }
-      });
-   }//dim = 2
-
-   if (dim==3)
-   {
-      auto w = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
-      auto v = Reshape(pa_data.Write(), 3, NQ, NE);
-
-      MFEM_FORALL(e, NE,
-      {
-         for (int q=0; q<NQ; ++q)
-         {
-
-            const double J00 = J(q,0,0,e);
-            const double J01 = J(q,0,1,e);
-            const double J02 = J(q,0,2,e);
-
-            const double J10 = J(q,1,0,e);
-            const double J11 = J(q,1,1,e);
-            const double J12 = J(q,1,2,e);
-
-            const double J20 = J(q,2,0,e);
-            const double J21 = J(q,2,1,e);
-            const double J22 = J(q,2,2,e);
-
-            const double A00 = (J11 * J22) - (J12 * J21);
-            const double A01 = (J02 * J21) - (J01 * J22);
-            const double A02 = (J01 * J12) - (J02 * J11);
-
-            const double A10 = (J12 * J20) - (J10 * J22);
-            const double A11 = (J00 * J22) - (J02 * J20);
-            const double A12 = (J02 * J10) - (J00 * J12);
-
-            const double A20 = (J10 * J21) - (J11 * J20);
-            const double A21 = (J01 * J20) - (J00 * J21);
-            const double A22 = (J00 * J11) - (J01 * J10);
-
-            const double w_coeff = w[q];
-
-            double cx = C(0,q,e);
-            double cy = C(1,q,e);
-            double cz = C(2,q,e);
-
-            v(0,q,e) = w_coeff*(cx*A00 + cy*A01 + cz*A02);
-            v(1,q,e) = w_coeff*(cx*A10 + cy*A11 + cz*A12);
-            v(2,q,e) = w_coeff*(cx*A20 + cy*A21 + cz*A22);
-         }
-
-      });
-   }//dim = 3
-}
-
-// PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
-void PAConvectionApply2D(const int NE,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Array<double> &gt,
-                         const Vector &_op,
-                         const Vector &_x,
-                         Vector &_y,
-                         const int d1d = 0,
-                         const int q1d = 0)
-{
-   const int DIM = 2;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto D = Reshape(_op.Read(), DIM, Q1D, Q1D, NE);
-   auto xloc = Reshape(_x.Read(), D1D, D1D, NE);
-   auto yloc = Reshape(_y.ReadWrite(), D1D, D1D, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      // the following variables are evaluated at compile time
-      constexpr int iDIM     = 2;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      double U[iDIM][max_D1D][max_Q1D];
-      for (int j1=0; j1<Q1D; ++j1)
-      {
-         for (int i2=0; i2<D1D; ++i2)
-         {
-
-            double dot0=0.0; double dot1=0.0;
-            for (int i1=0; i1<D1D; ++i1)
-            {
-               dot0 += G(j1,i1)*xloc(i1, i2, e);
-               dot1 += B(j1,i1)*xloc(i1, i2, e);
-            }
-            U[0][i2][j1] = dot0;
-            U[1][i2][j1] = dot1;
-         }
-      }
-
-      double W[iDIM][max_Q1D][max_Q1D];
-      for (int j1=0; j1<Q1D; ++j1)
-      {
-         for (int i2=0; i2<Q1D; ++i2)
-         {
-
-            double dot0=0.0; double dot1=0.0;
-            for (int i1=0; i1<D1D; ++i1)
-            {
-               dot0 += B(j1,i1)*U[0][i1][i2];
-               dot1 += G(j1,i1)*U[1][i1][i2];
-            }
-            W[0][i2][j1] = dot0;
-            W[1][i2][j1] = dot1;
-         }
-      }
-
-      double Z[max_Q1D][max_Q1D];
-      for (int k2=0; k2<Q1D; ++k2)
-      {
-         for (int k1=0; k1<Q1D; ++k1)
-         {
-
-            double dot(0.0);
-            for (int c=0; c<2; ++c)
-            {
-               dot += D(c, k1, k2, e) * W[c][k1][k2];
-            }
-            Z[k1][k2] = dot;
-         }
-      }
-
-      double Q[max_Q1D][max_D1D];
-      for (int j1=0; j1<D1D; ++j1)
-      {
-         for (int i2=0; i2<Q1D; ++i2)
-         {
-
-            double dot(0.0);
-            for (int i1=0; i1<Q1D; ++i1)
-            {
-               dot += Bt(j1, i1)*Z[i1][i2];
-            }
-            Q[i2][j1] = dot;
-         }
-      }
-
-      for (int j1=0; j1<D1D; ++j1)
-      {
-         for (int i2=0; i2<D1D; ++i2)
-         {
-
-            double dot(0.0);
-            for (int i1=0; i1<Q1D; ++i1)
-            {
-               dot += Bt(j1, i1)*Q[i1][i2];
-            }
-            yloc(i2,j1,e) += dot;
-         }
-      }
-   });
-
-}
-
-
-// PA Convection Apply 3D kernel
-template<const int T_D1D = 0,
-         const int T_Q1D = 0> static
-void PAConvectionApply3D(const int NE,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Array<double> &gt,
-                         const Vector &_op,
-                         const Vector &_x,
-                         Vector &_y,
-                         int d1d = 0, int q1d = 0)
-{
-   const int DIM = 3;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto D = Reshape(_op.Read(), DIM, Q1D,Q1D, Q1D, NE);
-   auto xloc = Reshape(_x.Read(), D1D, D1D, D1D, NE);
-   auto yloc = Reshape(_y.ReadWrite(), D1D, D1D, D1D, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      //qpt x dof x dof
-      double BX[max_Q1D][max_Q1D][max_Q1D];
-      double GX[max_Q1D][max_Q1D][max_Q1D];
-      for (int j1=0; j1<Q1D; ++j1)
-      {
-         for (int i3=0; i3<D1D; ++i3)
-         {
-            for (int i2=0; i2<D1D; ++i2)
-            {
-
-               BX[i2][i3][j1] = 0.0;
-               GX[i2][i3][j1] = 0.0;
-               for (int i1=0; i1<D1D; ++i1)
-               {
-                  BX[i2][i3][j1] += B(j1, i1) * xloc(i1,i2,i3,e);
-                  GX[i2][i3][j1] += G(j1, i1) * xloc(i1,i2,i3,e);
-               }
-            }
-         }
-      }
-
-      double BBX[max_D1D][max_Q1D][max_Q1D];
-      double GBX[max_D1D][max_Q1D][max_Q1D];
-      double BGX[max_D1D][max_Q1D][max_Q1D];
-
-      for (int j1=0; j1<Q1D; ++j1)
-      {
-         for (int i3=0; i3<Q1D; ++i3)
-         {
-            for (int i2=0; i2<D1D; ++i2)
-            {
-
-               BBX[i2][i3][j1] = 0.0;
-               GBX[i2][i3][j1] = 0.0;
-               BGX[i2][i3][j1] = 0.0;
-               for (int i1=0; i1<D1D; ++i1)
-               {
-                  BBX[i2][i3][j1] += B(j1, i1) * BX[i1][i2][i3];
-                  GBX[i2][i3][j1] += G(j1, i1) * BX[i1][i2][i3];
-                  BGX[i2][i3][j1] += B(j1, i1) * GX[i1][i2][i3];
-               }
-
-            }
-         }
-      }
-
-      double GBBX[max_Q1D][max_Q1D][max_Q1D];
-      double BGBX[max_Q1D][max_Q1D][max_Q1D];
-      double BBGX[max_Q1D][max_Q1D][max_Q1D];
-
-      for (int j1=0; j1<Q1D; ++j1)
-      {
-         for (int i3=0; i3<Q1D; ++i3)
-         {
-            for (int i2=0; i2<Q1D; ++i2)
-            {
-
-               GBBX[i2][i3][j1] = 0.0;
-               BGBX[i2][i3][j1] = 0.0;
-               BBGX[i2][i3][j1] = 0.0;
-               for (int i1=0; i1<D1D; ++i1)
-               {
-                  GBBX[i2][i3][j1] += G(j1, i1) * BBX[i1][i2][i3];
-                  BGBX[i2][i3][j1] += B(j1, i1) * GBX[i1][i2][i3];
-                  BBGX[i2][i3][j1] += B(j1, i1) * BGX[i1][i2][i3];
-               }
-
-            }
-         }
-      }
-
-      double Z[max_Q1D][max_Q1D][max_Q1D];
-      for (int k3=0; k3<Q1D; ++k3)
-      {
-         for (int k2=0; k2<Q1D; ++k2)
-         {
-            for (int k1=0; k1<Q1D; ++k1)
-            {
-
-               double dot(0.0);
-               {
-                  dot += D(0, k1, k2, k3, e) * BBGX[k1][k2][k3];
-                  dot += D(1, k1, k2, k3, e) * BGBX[k1][k2][k3];
-                  dot += D(2, k1, k2, k3, e) * GBBX[k1][k2][k3];
-               }
-               Z[k1][k2][k3] = dot;
-            }
-         }
-      }
-
-      //Apply (B1d)^T 3 more times
-      double BZ[max_Q1D][max_Q1D][max_Q1D];
-      for (int j1=0; j1<D1D; ++j1)
-      {
-         for (int i3=0; i3<Q1D; ++i3)
-         {
-            for (int i2=0; i2<Q1D; ++i2)
-            {
-
-               BZ[i2][i3][j1]=0.0;
-               for (int i1=0; i1<Q1D; ++i1)
-               {
-                  BZ[i2][i3][j1] += Bt(j1,i1)*Z[i1][i2][i3];
-               }
-
-            }
-         }
-      }
-
-      double BBZ[max_Q1D][max_Q1D][max_Q1D];
-      for (int j1=0; j1<D1D; ++j1)
-      {
-         for (int i3=0; i3<D1D; ++i3)
-         {
-            for (int i2=0; i2<Q1D; ++i2)
-            {
-
-               BBZ[i2][i3][j1]=0.0;
-               for (int i1=0; i1<Q1D; ++i1)
-               {
-                  BBZ[i2][i3][j1] += Bt(j1,i1)*BZ[i1][i2][i3];
-               }
-
-            }
-         }
-      }
-
-      for (int j1=0; j1<D1D; ++j1)
-      {
-         for (int i3=0; i3<D1D; ++i3)
-         {
-            for (int i2=0; i2<D1D; ++i2)
-            {
-
-               double dot(0.0);
-               for (int i1=0; i1<Q1D; ++i1)
-               {
-                  dot += Bt(j1,i1)*BBZ[i1][i2][i3];
-               }
-               yloc(i2,i3,j1,e) += dot;
-            }
-         }
-      }
-
-   });
-
-}
-
-
-static void PAConvectionApply(const int dim,
-                              const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &G,
-                              const Array<double> &Bt,
-                              const Array<double> &Gt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-
-   if (dim==2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         //case 0x22: PAConvectionApply2D<2,2>(NE, B, G, Bt, Gt, op, x, y); break;
-         //case 0x33: PAConvectionApply2D<3,3>(NE, B, G, Bt, Gt, op, x, y); break;
-         //case 0x44: PAConvectionApply2D<4,4>(NE, B, G, Bt, Gt, op, x, y); break;
-         //case 0x55: PAConvectionApply2D<5,5>(NE, B, G, Bt, Gt, op, x, y); break;
-         default: PAConvectionApply2D(NE, B, G, Bt, Gt, op, x, y,D1D,Q1D); break;
-      }
-      return;
-   }
-   if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         //case 0x23: PAConvectionApply3D<2,3>(NE, B, G, Bt, G, op, x, y); break;
-         //case 0x34: PAConvectionApply3D<3,4>(NE, B, G, Bt, G, op, x, y); break;
-         //case 0x45: PAConvectionApply3D<4,5>(NE, B, G, Bt, G, op, x, y); break;
-         //case 0x56: PAConvectionApply3D<5,6>(NE, B, G, Bt, G, op, x, y); break;
-         default: PAConvectionApply3D(NE, B, G, Bt, G, op, x, y,D1D,Q1D); break;
-      }
-      return;
-   }
-   MFEM_ABORT("Unknown kernel.");
-
-}
-
-void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAConvectionApply(dim, dofs1D, quad1D, ne,maps->B, maps->G, maps->Bt, maps->Gt,
-                     pa_data, x, y);
-}
-
-} // namespace mfem
@@ -0,0 +1,80 @@
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability see http://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.
+
+#ifndef MFEM_BILININTEG_EXT
+#define MFEM_BILININTEG_EXT
+
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+/// GeometryExtension
+class GeometryExtension
+{
+public:
+   Array<int> eMap;
+   Array<double> nodes;
+   Array<double> X, J, invJ, detJ;
+   static GeometryExtension* Get(const FiniteElementSpace&,
+                                 const IntegrationRule&);
+   static GeometryExtension* Get(const FiniteElementSpace&,
+                                 const IntegrationRule&,
+                                 const Vector&);
+   static void ReorderByVDim(const GridFunction*);
+   static void ReorderByNodes(const GridFunction*);
+};
+
+/// DofToQuad
+class DofToQuad
+{
+private:
+   std::string hash;
+public:
+   ~DofToQuad();
+   void operator=(DofToQuad&);
+   void operator=(DofToQuad const&);
+public:
+   Array<double> W, B, G, Bt, Gt;
+public:
+   static DofToQuad* Get(const FiniteElementSpace&,
+                         const IntegrationRule&,
+                         const bool = false);
+   static DofToQuad* Get(const FiniteElementSpace&,
+                         const FiniteElementSpace&,
+                         const IntegrationRule&,
+                         const bool = false);
+   static DofToQuad* Get(const FiniteElement&,
+                         const FiniteElement&,
+                         const IntegrationRule&,
+                         const bool = false);
+   static DofToQuad* GetTensorMaps(const FiniteElement&,
+                                   const FiniteElement&,
+                                   const IntegrationRule&,
+                                   const bool = false);
+   static DofToQuad* GetD2QTensorMaps(const FiniteElement&,
+                                      const IntegrationRule&,
+                                      const bool = false);
+   static DofToQuad* GetSimplexMaps(const FiniteElement&,
+                                    const IntegrationRule&,
+                                    const bool = false);
+   static DofToQuad* GetSimplexMaps(const FiniteElement&,
+                                    const FiniteElement&,
+                                    const IntegrationRule&,
+                                    const bool = false);
+   static DofToQuad* GetD2QSimplexMaps(const FiniteElement&,
+                                       const IntegrationRule&,
+                                       const bool = false);
+};
+
+}
+
+#endif
@@ -1,823 +0,0 @@
-// Copyright (c) 2019, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Gradient Integrator
-
-/* Description of the *SetupND functions
-   Inputs are as follows
-   \b Q1D number of quadrature points in one dimension.
-   \b w quadrature weights.
-   \b j element Jacobians.
-   \b COEFF coefficient at quadrature points.
-
-   The function is used precompute data needed at quadrature points during
-   the action. */
-
-/* Description of the *ApplyND functions
-   The template parameters are
-   \b T_D1D number of degrees of freedom in one dimension,
-   \b T_Q1D number of quadrature points in one dimension,
-   and are necessary to allow for compiler optimizations inside the kernel.
-
-   Inputs are as follows
-   \b NE number of elements.
-   \b B matrix of basis functions.
-   \b G matrix of derivatives of the basis functions.
-   \b Bt transpose of matrix of basis functions.
-   \b Gt transpose matrix of derivatives of the basis functions.
-   \b op data used during action of the element matrix in the tensor
-   product application.
-
-   \b x input vector of degrees of freedom on the element.
-   \b y output vector of degrees of freedom on the element.
-
-   The function computes the kernel for one dimension that is suitable for
-   tensor product action to form ND operators.
-   Most of the ND inputs are reshaped as NQ*(ND*ND)*NE data structure, i.e
-   to allow indexing such as op(qpt,i,j,el).
-
-   The output data structure is dependent on the kernel and layout of the
-   dimension ND and element number, but in general resembles the action of the
-   element matrix in the tensor product application. */
-
-/* Description of the Smem*ApplyND functions
-   The shared memory (Smem) versions of the kernels differ from the regular
-   versions in the following properties.
-
-   \b MFEM_FORALL is using only one level of parallelism.
-   \b MFEM_FORALL_ND uses an additional level of parallelism
-   \b MFEM_FOREACH_THREAD
-
-   These macros allow automatic mapping of manually defined blocks to
-   underlying hardware threads. These threads can share memory by using
-   the \b MFEM_SHARED keyword for local arrays. */
-
-// PA Gradient Assemble 2D kernel
-static void PAGradientSetup2D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              const double COEFF,
-                              Vector &op)
-{
-   const int NQ = Q1D*Q1D;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto y = Reshape(op.Write(), NQ, 2, 2, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J21 = J(q,1,0,e);
-         const double J22 = J(q,1,1,e);
-         // Store wq * Q * adj(J)
-         y(q,0,0,e) = W[q] * COEFF *  J22; // 1,1
-         y(q,0,1,e) = W[q] * COEFF * -J12; // 1,2
-         y(q,1,0,e) = W[q] * COEFF * -J21; // 2,1
-         y(q,1,1,e) = W[q] * COEFF *  J11; // 2,2
-      }
-   });
-}
-
-// PA Gradient Assemble 3D kernel
-static void PAGradientSetup3D(const int Q1D,
-                              const int NE,
-                              const Array<double> &w,
-                              const Vector &j,
-                              const double COEFF,
-                              Vector &op)
-{
-   const int NQ = Q1D*Q1D*Q1D;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto y = Reshape(op.Write(), NQ, 3, 3, NE);
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double cw  = W[q] * COEFF;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-         // Store wq * Q * adj(J)
-         y(q,0,0,e) = cw * A11; // 1,1
-         y(q,0,1,e) = cw * A12; // 1,2
-         y(q,0,2,e) = cw * A13; // 1,3
-         y(q,1,0,e) = cw * A21; // 2,1
-         y(q,1,1,e) = cw * A22; // 2,2
-         y(q,1,2,e) = cw * A23; // 2,3
-         y(q,2,0,e) = cw * A31; // 3,1
-         y(q,2,1,e) = cw * A32; // 3,2
-         y(q,2,2,e) = cw * A33; // 3,3
-      }
-   });
-}
-
-static void PAGradientSetup(const int dim,
-                            const int TR_D1D,
-                            const int TE_D1D,
-                            const int Q1D,
-                            const int NE,
-                            const Array<double> &W,
-                            const Vector &J,
-                            const double COEFF,
-                            Vector &op)
-{
-   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAGradientSetup"); }
-   if (dim == 2)
-   {
-      PAGradientSetup2D(Q1D, NE, W, J, COEFF, op);
-   }
-   if (dim == 3)
-   {
-      PAGradientSetup3D(Q1D, NE, W, J, COEFF, op);
-   }
-}
-
-void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
-                                    const FiniteElementSpace &test_fes)
-{
-   // Assumes tensor-product elements ordered by nodes
-   MFEM_ASSERT(trial_fes.GetOrdering() == Ordering::byNODES,
-               "PA Only supports Ordering::byNODES!");
-   Mesh *mesh = trial_fes.GetMesh();
-   const FiniteElement &trial_fe = *trial_fes.GetFE(0);
-   const FiniteElement &test_fe = *test_fes.GetFE(0);
-   ElementTransformation *trans = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
-                                                            *trans);
-   const int dims = trial_fe.GetDim();
-   const int dimsToStore = dims * dims;
-   const int nq = ir->GetNPoints();
-   dim = mesh->Dimension();
-   ne = trial_fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   trial_maps = &trial_fe.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   trial_dofs1D = trial_maps->ndof;
-   quad1D = trial_maps->nqpt;
-   test_maps  = &test_fe.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   test_dofs1D = test_maps->ndof;
-   MFEM_ASSERT(quad1D == test_maps->nqpt,
-               "PA requires test and trial space to have same number of quadrature points!");
-   pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());
-   double coeff = 1.0;
-   if (Q)
-   {
-      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
-      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
-      coeff = cQ->constant;
-   }
-   PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
-                   ne, ir->GetWeights(), geom->J, coeff, pa_data);
-}
-
-// PA Gradient Apply 2D kernel
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-static void PAGradientApply2D(const int NE,
-                              const Array<double> &b,
-                              const Array<double> &g,
-                              const Array<double> &bt,
-                              const Vector &_op,
-                              const Vector &_x,
-                              Vector &_y,
-                              const int tr_d1d = 0,
-                              const int te_d1d = 0,
-                              const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, TR_D1D);
-   auto G = Reshape(g.Read(), Q1D, TR_D1D);
-   auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D*Q1D, 2,2, NE);
-   auto x = Reshape(_x.Read(), TR_D1D, TR_D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), TE_D1D, TE_D1D, 2, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-      const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = 2;
-      // the following variables are evaluated at compile time
-      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      double grad[max_Q1D][max_Q1D][VDIM];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            grad[qy][qx][0] = 0.0;
-            grad[qy][qx][1] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < TR_D1D; ++dy)
-      {
-         double gradX[max_Q1D][VDIM];
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            gradX[qx][0] = 0.0;
-            gradX[qx][1] = 0.0;
-         }
-         for (int dx = 0; dx < TR_D1D; ++dx)
-         {
-            const double s = x(dx,dy,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               gradX[qx][0] += s * G(qx,dx);
-               gradX[qx][1] += s * B(qx,dx);
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double wy  = B(qy,dy);
-            const double wDy = G(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               grad[qy][qx][0] += gradX[qx][0] * wy;
-               grad[qy][qx][1] += gradX[qx][1] * wDy;
-            }
-         }
-      }
-      // We've now calculated grad(p) = [Dxy, xDy] in plane
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const int q = qx + qy * Q1D;
-            const double gradX = grad[qy][qx][0];
-            const double gradY = grad[qy][qx][1];
-
-            grad[qy][qx][0] = gradX*op(q,0,0,e) + gradY*op(q,1,0,e);
-            grad[qy][qx][1] = gradX*op(q,0,1,e) + gradY*op(q,1,1,e);
-         }
-      }
-      // We've now calculated grad = grad p * op
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double opX[max_TE_D1D][VDIM];
-         for (int dx = 0; dx < TE_D1D; ++dx)
-         {
-            opX[dx][0] = 0.0;
-            opX[dx][1] = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               opX[dx][0] += Bt(dx,qx)*grad[qy][qx][0];
-               opX[dx][1] += Bt(dx,qx)*grad[qy][qx][1];
-            }
-         }
-         for (int dy = 0; dy < TE_D1D; ++dy)
-         {
-            for (int dx = 0; dx < TE_D1D; ++dx)
-            {
-               y(dx,dy,0,e) += Bt(dy,qy)*opX[dx][0];
-               y(dx,dy,1,e) += Bt(dy,qy)*opX[dx][1];
-            }
-         }
-      }
-      // We've now calculated y = u * grad
-   });
-
-}
-
-// PA Gradient Apply 2D kernel transpose
-template<int T_TR_D1D = 0, int T_TE_D1D = 0, int T_Q1D = 0>
-static void PAGradientApplyTranspose2D(const int NE,
-                                       const Array<double> &bt,
-                                       const Array<double> &gt,
-                                       const Array<double> &b,
-                                       const Vector &_op,
-                                       const Vector &_x,
-                                       Vector &_y,
-                                       const int tr_d1d = 0,
-                                       const int te_d1d = 0,
-                                       const int q1d = 0)
-{
-   // TODO
-   MFEM_ASSERT(false, "GradientPAApplyTranspose 3D not implemented.");
-}
-
-// PA Gradient Apply 3D kernel
-template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PAGradientApply3D(const int NE,
-                              const Array<double> &b,
-                              const Array<double> &g,
-                              const Array<double> &bt,
-                              const Vector &_op,
-                              const Vector &_x,
-                              Vector &_y,
-                              int tr_d1d = 0,
-                              int te_d1d = 0,
-                              int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, TR_D1D);
-   auto G = Reshape(g.Read(), Q1D, TR_D1D);
-   auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D*Q1D*Q1D, 3,3, NE);
-   auto x = Reshape(_x.Read(), TR_D1D, TR_D1D, TR_D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 3, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-      const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = 3;
-      // the following variables are evaluated at compile time
-      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      double grad[max_Q1D][max_Q1D][max_Q1D][VDIM];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               grad[qz][qy][qx][0] = 0.0;
-               grad[qz][qy][qx][1] = 0.0;
-               grad[qz][qy][qx][2] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < TR_D1D; ++dz)
-      {
-         double gradXY[max_Q1D][max_Q1D][3];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               gradXY[qy][qx][0] = 0.0;
-               gradXY[qy][qx][1] = 0.0;
-               gradXY[qy][qx][2] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < TR_D1D; ++dy)
-         {
-            double gradX[max_Q1D][2];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               gradX[qx][0] = 0.0;
-               gradX[qx][1] = 0.0;
-            }
-            for (int dx = 0; dx < TR_D1D; ++dx)
-            {
-               const double s = x(dx,dy,dz,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] += s * B(qx,dx);
-                  gradX[qx][1] += s * G(qx,dx);
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy  = B(qy,dy);
-               const double wDy = G(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double wx  = gradX[qx][0];
-                  const double wDx = gradX[qx][1];
-                  gradXY[qy][qx][0] += wDx * wy;
-                  gradXY[qy][qx][1] += wx  * wDy;
-                  gradXY[qy][qx][2] += wx  * wy;
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double wz  = B(qz,dz);
-            const double wDz = G(qz,dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
-                  grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
-                  grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
-               }
-            }
-         }
-      }
-      // We've now calculated grad(p) = [Dxyz, xDyz, xyDz] in plane
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int q = qx + (qy + qz * Q1D) * Q1D;
-               const double gradX = grad[qz][qy][qx][0];
-               const double gradY = grad[qz][qy][qx][1];
-               const double gradZ = grad[qz][qy][qx][2];
-
-               grad[qz][qy][qx][0] = gradX*op(q,0,0,e) + gradY*op(q,1,0,e) + gradZ*op(q,2,0,e);
-               grad[qz][qy][qx][1] = gradX*op(q,0,1,e) + gradY*op(q,1,1,e) + gradZ*op(q,2,1,e);
-               grad[qz][qy][qx][2] = gradX*op(q,0,2,e) + gradY*op(q,1,2,e) + gradZ*op(q,2,2,e);
-            }
-         }
-      }
-      // We've now calculated grad = grad p * op
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double opXY[max_TE_D1D][max_TE_D1D][VDIM];
-         for (int dy = 0; dy < TE_D1D; ++dy)
-         {
-            for (int dx = 0; dx < TE_D1D; ++dx)
-            {
-               opXY[dy][dx][0] = 0.0;
-               opXY[dy][dx][1] = 0.0;
-               opXY[dy][dx][2] = 0.0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double opX[max_TE_D1D][VDIM];
-            for (int dx = 0; dx < TE_D1D; ++dx)
-            {
-               opX[dx][0] = 0.0;
-               opX[dx][1] = 0.0;
-               opX[dx][2] = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  opX[dx][0] += Bt(dx,qx)*grad[qz][qy][qx][0];
-                  opX[dx][1] += Bt(dx,qx)*grad[qz][qy][qx][1];
-                  opX[dx][2] += Bt(dx,qx)*grad[qz][qy][qx][2];
-               }
-            }
-            for (int dy = 0; dy < TE_D1D; ++dy)
-            {
-               for (int dx = 0; dx < TE_D1D; ++dx)
-               {
-                  opXY[dy][dx][0] += Bt(dy,qy)*opX[dx][0];
-                  opXY[dy][dx][1] += Bt(dy,qy)*opX[dx][1];
-                  opXY[dy][dx][2] += Bt(dy,qy)*opX[dx][2];
-               }
-            }
-         }
-         for (int dz = 0; dz < TE_D1D; ++dz)
-         {
-            for (int dy = 0; dy < TE_D1D; ++dy)
-            {
-               for (int dx = 0; dx < TE_D1D; ++dx)
-               {
-                  y(dx,dy,dz,0,e) += Bt(dz,qz)*opXY[dy][dx][0];
-                  y(dx,dy,dz,1,e) += Bt(dz,qz)*opXY[dy][dx][1];
-                  y(dx,dy,dz,2,e) += Bt(dz,qz)*opXY[dy][dx][2];
-               }
-            }
-         }
-      }
-      // We've now calculated y = u * grad
-   });
-}
-
-// PA Gradient Apply 3D kernel
-template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void PAGradientApplyTranspose3D(const int NE,
-                                       const Array<double> &bt,
-                                       const Array<double> &gt,
-                                       const Array<double> &b,
-                                       const Vector &_op,
-                                       const Vector &_x,
-                                       Vector &_y,
-                                       int tr_d1d = 0,
-                                       int te_d1d = 0,
-                                       int q1d = 0)
-{
-   MFEM_ASSERT(false, "Gradient PA Apply Transpose 3D not implemented.");
-}
-
-// Shared memory PA Gradient Apply 3D kernel
-template<const int T_TR_D1D = 0, const int T_TE_D1D = 0, const int T_Q1D = 0>
-static void SmemPAGradientApply3D(const int NE,
-                                  const Array<double> &b_,
-                                  const Array<double> &g_,
-                                  const Array<double> &bt_,
-                                  const Vector &d_,
-                                  const Vector &x_,
-                                  Vector &y_,
-                                  const int tr_d1d = 0,
-                                  const int te_d1d = 0,
-                                  const int q1d = 0)
-{
-   const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
-   const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
-   MFEM_VERIFY(TR_D1D <= Q1D, "");
-   MFEM_VERIFY(TE_D1D <= Q1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-
-   auto b = Reshape(b_.Read(), Q1D, TR_D1D);
-   auto g = Reshape(g_.Read(), Q1D, TR_D1D);
-   auto bt = Reshape(bt_.Read(), TE_D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, 3, 3, NE);
-   auto x = Reshape(x_.Read(), TR_D1D, TR_D1D, TR_D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), TE_D1D, TE_D1D, TE_D1D, 3, NE);
-
-   MFEM_FORALL_3D(e, NE, (Q1D>8)?8:Q1D, (Q1D>8)?8:Q1D, (Q1D>8)?8:Q1D,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1DR = T_TR_D1D ? T_TR_D1D : tr_d1d;
-      const int D1DE = T_TE_D1D ? T_TE_D1D : te_d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1R = T_TR_D1D ? T_TR_D1D : MAX_D1D;
-      constexpr int MD1E = T_TE_D1D ? T_TE_D1D : MAX_D1D;
-      constexpr int MD1 = MD1E > MD1R ? MD1E : MD1R;
-      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
-      MFEM_SHARED double sBG[2][MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) (sBG+0);
-      double (*G)[MD1] = (double (*)[MD1]) (sBG+1);
-      double (*Bt)[MQ1] = (double (*)[MQ1]) (sBG+0);
-      MFEM_SHARED double sm0[3][MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[3][MDQ*MDQ*MDQ];
-      double (*X)[MD1][MD1]    = (double (*)[MD1][MD1]) (sm0+2);
-      double (*DDQ0)[MD1][MQ1] = (double (*)[MD1][MQ1]) (sm0+0);
-      double (*DDQ1)[MD1][MQ1] = (double (*)[MD1][MQ1]) (sm0+1);
-
-      double (*DQQ0)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) (sm1+0);
-      double (*DQQ1)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) (sm1+1);
-      double (*DQQ2)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) (sm1+2);
-
-      double (*QQQ0)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) (sm0+0);
-      double (*QQQ1)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) (sm0+1);
-      double (*QQQ2)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) (sm0+2);
-
-      double (*QQD0)[MQ1][MD1] = (double (*)[MQ1][MD1]) (sm1+0);
-      double (*QQD1)[MQ1][MD1] = (double (*)[MQ1][MD1]) (sm1+1);
-      double (*QQD2)[MQ1][MD1] = (double (*)[MQ1][MD1]) (sm1+2);
-
-      double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
-      double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
-      double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dz,z,D1DR)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1DR)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1DR)
-            {
-               X[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1DR)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-               G[q][d] = g(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1DR)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1DR)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               for (int dx = 0; dx < D1DR; ++dx)
-               {
-                  const double coord = X[dz][dy][dx];
-                  u += coord * B[qx][dx];
-                  v += coord * G[qx][dx];
-               }
-               DDQ0[dz][dy][qx] = u;
-               DDQ1[dz][dy][qx] = v;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1DR)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int dy = 0; dy < D1DR; ++dy)
-               {
-                  u += DDQ1[dz][dy][qx] * B[qy][dy];
-                  v += DDQ0[dz][dy][qx] * G[qy][dy];
-                  w += DDQ0[dz][dy][qx] * B[qy][dy];
-               }
-               DQQ0[dz][qy][qx] = u;
-               DQQ1[dz][qy][qx] = v;
-               DQQ2[dz][qy][qx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int dz = 0; dz < D1DR; ++dz)
-               {
-                  u += DQQ0[dz][qy][qx] * B[qz][dz];
-                  v += DQQ1[dz][qy][qx] * B[qz][dz];
-                  w += DQQ2[dz][qy][qx] * G[qz][dz];
-               }
-               QQQ0[qz][qy][qx] = u;
-               QQQ1[qz][qy][qx] = v;
-               QQQ2[qz][qy][qx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const int q = qx + (qy + qz * Q1D) * Q1D;
-               const double gX = QQQ0[qz][qy][qx];
-               const double gY = QQQ1[qz][qy][qx];
-               const double gZ = QQQ2[qz][qy][qx];
-               QQQ0[qz][qy][qx] = (D(q,0,0,e)*gX) + (D(q,1,0,e)*gY) + (D(q,2,0,e)*gZ);
-               QQQ1[qz][qy][qx] = (D(q,0,1,e)*gX) + (D(q,1,1,e)*gY) + (D(q,2,1,e)*gZ);
-               QQQ2[qz][qy][qx] = (D(q,0,2,e)*gX) + (D(q,1,2,e)*gY) + (D(q,2,2,e)*gZ);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1DE)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[d][q] = bt(d,q);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1DE)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  u += QQQ0[qz][qy][qx] * Bt[dx][qx];
-                  v += QQQ1[qz][qy][qx] * Bt[dx][qx];
-                  w += QQQ2[qz][qy][qx] * Bt[dx][qx];
-               }
-               QQD0[qz][qy][dx] = u;
-               QQD1[qz][qy][dx] = v;
-               QQD2[qz][qy][dx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1DE)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1DE)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  u += QQD0[qz][qy][dx] * Bt[dy][qy];
-                  v += QQD1[qz][qy][dx] * Bt[dy][qy];
-                  w += QQD2[qz][qy][dx] * Bt[dy][qy];
-               }
-               QDD0[qz][dy][dx] = u;
-               QDD1[qz][dy][dx] = v;
-               QDD2[qz][dy][dx] = w;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1DE)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1DE)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1DE)
-            {
-               double u = 0.0;
-               double v = 0.0;
-               double w = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u += QDD0[qz][dy][dx] * Bt[dz][qz];
-                  v += QDD1[qz][dy][dx] * Bt[dz][qz];
-                  w += QDD2[qz][dy][dx] * Bt[dz][qz];
-               }
-               y(dx,dy,dz,0,e) += u;
-               y(dx,dy,dz,1,e) += v;
-               y(dx,dy,dz,2,e) += w;
-            }
-         }
-      }
-   });
-}
-
-static void PAGradientApply(const int dim,
-                            const int TR_D1D,
-                            const int TE_D1D,
-                            const int Q1D,
-                            const int NE,
-                            const Array<double> &B,
-                            const Array<double> &G,
-                            const Array<double> &Bt,
-                            const Vector &op,
-                            const Vector &x,
-                            Vector &y,
-                            bool transpose=false)
-{
-
-   if (dim == 2)
-   {
-      return PAGradientApply2D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
-   }
-   if (dim == 3)
-   {
-      return PAGradientApply3D(NE,B,G,Bt,op,x,y,TR_D1D,TE_D1D,Q1D);
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-// PA Gradient Apply kernel
-void GradientIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAGradientApply(dim, trial_dofs1D, test_dofs1D, quad1D, ne,
-                   trial_maps->B, trial_maps->G, test_maps->Bt, pa_data, x, y,
-                   false);
-}
-
-// PA Gradient Apply kernel
-void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
-{
-   MFEM_ABORT("PA Gradient AddMultTransposePA not implemented.");
-}
-
-} // namespace mfem
-
@@ -1,503 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Vector Diffusion Integrator
-
-// PA Diffusion Assemble 2D kernel
-static void PAVectorDiffusionSetup2D(const int Q1D,
-                                     const int NE,
-                                     const Array<double> &w,
-                                     const Vector &j,
-                                     const double COEFF,
-                                     Vector &op)
-{
-   const int NQ = Q1D*Q1D;
-   auto W = w.Read();
-
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto y = Reshape(op.Write(), NQ, 3, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double c_detJ = W[q] * COEFF / ((J11*J22)-(J21*J12));
-         y(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
-         y(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
-         y(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
-      }
-   });
-}
-
-// PA Diffusion Assemble 3D kernel
-static void PAVectorDiffusionSetup3D(const int Q1D,
-                                     const int NE,
-                                     const Array<double> &w,
-                                     const Vector &j,
-                                     const double COEFF,
-                                     Vector &op)
-{
-   const int NQ = Q1D*Q1D*Q1D;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto y = Reshape(op.Write(), NQ, 6, NE);
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double detJ = J11 * (J22 * J33 - J32 * J23) -
-         /* */               J21 * (J12 * J33 - J32 * J13) +
-         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double c_detJ = W[q] * COEFF / detJ;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-         // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-         y(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-         y(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-         y(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-         y(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-         y(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-         y(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
-      }
-   });
-}
-
-static void PAVectorDiffusionSetup(const int dim,
-                                   const int D1D,
-                                   const int Q1D,
-                                   const int NE,
-                                   const Array<double> &W,
-                                   const Vector &J,
-                                   const double COEFF,
-                                   Vector &op)
-{
-   if (!(dim == 2 || dim == 3))
-   {
-      MFEM_ABORT("Dimension not supported.");
-   }
-   if (dim == 2)
-   {
-      PAVectorDiffusionSetup2D(Q1D, NE, W, J, COEFF, op);
-   }
-   if (dim == 3)
-   {
-      PAVectorDiffusionSetup3D(Q1D, NE, W, J, COEFF, op);
-   }
-}
-
-void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement &el = *fes.GetFE(0);
-   const IntegrationRule *ir
-      = IntRule ? IntRule : &DiffusionIntegrator::GetRule(el, el);
-   const int dims = el.GetDim();
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int nq = ir->GetNPoints();
-   dim = mesh->Dimension();
-   ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
-   double coeff = 1.0;
-   if (Q)
-   {
-      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
-      MFEM_VERIFY(cQ != NULL, "only ConstantCoefficient is supported!");
-      coeff = cQ->constant;
-   }
-   PAVectorDiffusionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
-                          coeff, pa_data);
-}
-
-// PA Diffusion Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
-void PAVectorDiffusionApply2D(const int NE,
-                              const Array<double> &b,
-                              const Array<double> &g,
-                              const Array<double> &bt,
-                              const Array<double> &gt,
-                              const Vector &_op,
-                              const Vector &_x,
-                              Vector &_y,
-                              const int d1d = 0,
-                              const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int VDIM = 2;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D*Q1D, 3, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, VDIM, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, VDIM, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      for (int c = 0; c < VDIM; ++ c)
-      {
-         double grad[max_Q1D][max_Q1D][2];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               grad[qy][qx][0] = 0.0;
-               grad[qy][qx][1] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double gradX[max_Q1D][2];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               gradX[qx][0] = 0.0;
-               gradX[qx][1] = 0.0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = x(dx,dy,c,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] += s * B(qx,dx);
-                  gradX[qx][1] += s * G(qx,dx);
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy  = B(qy,dy);
-               const double wDy = G(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qy][qx][0] += gradX[qx][1] * wy;
-                  grad[qy][qx][1] += gradX[qx][0] * wDy;
-               }
-            }
-         }
-         // Calculate Dxy, xDy in plane
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int q = qx + qy * Q1D;
-
-               const double O11 = op(q,0,e);
-               const double O12 = op(q,1,e);
-               const double O22 = op(q,2,e);
-
-               const double gradX = grad[qy][qx][0];
-               const double gradY = grad[qy][qx][1];
-
-               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double gradX[max_D1D][2];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               gradX[dx][0] = 0;
-               gradX[dx][1] = 0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double gX = grad[qy][qx][0];
-               const double gY = grad[qy][qx][1];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double wx  = Bt(dx,qx);
-                  const double wDx = Gt(dx,qx);
-                  gradX[dx][0] += gX * wDx;
-                  gradX[dx][1] += gY * wx;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double wy  = Bt(dy,qy);
-               const double wDy = Gt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  y(dx,dy,c,e) += ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
-               }
-            }
-         }
-      }
-   });
-}
-
-// PA Diffusion Apply 3D kernel
-template<const int T_D1D = 0,
-         const int T_Q1D = 0> static
-void PAVectorDiffusionApply3D(const int NE,
-                              const Array<double> &b,
-                              const Array<double> &g,
-                              const Array<double> &bt,
-                              const Array<double> &gt,
-                              const Vector &_op,
-                              const Vector &_x,
-                              Vector &_y,
-                              int d1d = 0, int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D*Q1D*Q1D, 6, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      for (int c = 0; c < VDIM; ++ c)
-      {
-         double grad[max_Q1D][max_Q1D][max_Q1D][3];
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qz][qy][qx][0] = 0.0;
-                  grad[qz][qy][qx][1] = 0.0;
-                  grad[qz][qy][qx][2] = 0.0;
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            double gradXY[max_Q1D][max_Q1D][3];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradXY[qy][qx][0] = 0.0;
-                  gradXY[qy][qx][1] = 0.0;
-                  gradXY[qy][qx][2] = 0.0;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               double gradX[max_Q1D][2];
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] = 0.0;
-                  gradX[qx][1] = 0.0;
-               }
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double s = x(dx,dy,dz,c,e);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     gradX[qx][0] += s * B(qx,dx);
-                     gradX[qx][1] += s * G(qx,dx);
-                  }
-               }
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double wy  = B(qy,dy);
-                  const double wDy = G(qy,dy);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const double wx  = gradX[qx][0];
-                     const double wDx = gradX[qx][1];
-                     gradXY[qy][qx][0] += wDx * wy;
-                     gradXY[qy][qx][1] += wx  * wDy;
-                     gradXY[qy][qx][2] += wx  * wy;
-                  }
-               }
-            }
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double wz  = B(qz,dz);
-               const double wDz = G(qz,dz);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
-                     grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
-                     grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
-                  }
-               }
-            }
-         }
-         // Calculate Dxyz, xDyz, xyDz in plane
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const int q = qx + (qy + qz * Q1D) * Q1D;
-                  const double O11 = op(q,0,e);
-                  const double O12 = op(q,1,e);
-                  const double O13 = op(q,2,e);
-                  const double O22 = op(q,3,e);
-                  const double O23 = op(q,4,e);
-                  const double O33 = op(q,5,e);
-                  const double gradX = grad[qz][qy][qx][0];
-                  const double gradY = grad[qz][qy][qx][1];
-                  const double gradZ = grad[qz][qy][qx][2];
-                  grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
-                  grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
-                  grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            double gradXY[max_D1D][max_D1D][3];
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  gradXY[dy][dx][0] = 0;
-                  gradXY[dy][dx][1] = 0;
-                  gradXY[dy][dx][2] = 0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               double gradX[max_D1D][3];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  gradX[dx][0] = 0;
-                  gradX[dx][1] = 0;
-                  gradX[dx][2] = 0;
-               }
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double gX = grad[qz][qy][qx][0];
-                  const double gY = grad[qz][qy][qx][1];
-                  const double gZ = grad[qz][qy][qx][2];
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     const double wx  = Bt(dx,qx);
-                     const double wDx = Gt(dx,qx);
-                     gradX[dx][0] += gX * wDx;
-                     gradX[dx][1] += gY * wx;
-                     gradX[dx][2] += gZ * wx;
-                  }
-               }
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const double wy  = Bt(dy,qy);
-                  const double wDy = Gt(dy,qy);
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     gradXY[dy][dx][0] += gradX[dx][0] * wy;
-                     gradXY[dy][dx][1] += gradX[dx][1] * wDy;
-                     gradXY[dy][dx][2] += gradX[dx][2] * wy;
-                  }
-               }
-            }
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double wz  = Bt(dz,qz);
-               const double wDz = Gt(dz,qz);
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     y(dx,dy,dz,c,e) +=
-                        ((gradXY[dy][dx][0] * wz) +
-                         (gradXY[dy][dx][1] * wz) +
-                         (gradXY[dy][dx][2] * wDz));
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-static void PAVectorDiffusionApply(const int dim,
-                                   const int D1D,
-                                   const int Q1D,
-                                   const int NE,
-                                   const Array<double> &B,
-                                   const Array<double> &G,
-                                   const Array<double> &Bt,
-                                   const Array<double> &Gt,
-                                   const Vector &op,
-                                   const Vector &x,
-                                   Vector &y)
-{
-   if (dim == 2)
-   {
-      return PAVectorDiffusionApply2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
-   }
-   if (dim == 3)
-   {
-      return PAVectorDiffusionApply3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-// PA Diffusion Apply kernel
-void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAVectorDiffusionApply(dim, dofs1D, quad1D, ne,
-                          maps->B, maps->G, maps->Bt, maps->Gt,
-                          pa_data, x, y);
-}
-
-} // namespace mfem
@@ -1,367 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Mass Integrator
-
-// PA Mass Assemble kernel
-void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   // Assuming the same element type
-   Mesh *mesh = fes.GetMesh();
-   if (mesh->GetNE() == 0) { return; }
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation *T = mesh->GetElementTransformation(0);
-   const IntegrationRule *ir
-      = IntRule ? IntRule : &MassIntegrator::GetRule(el, el, *T);
-   dim = mesh->Dimension();
-   ne = fes.GetMesh()->GetNE();
-   nq = ir->GetNPoints();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::COORDINATES |
-                                    GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetMemoryType());
-   double coeff = 1.0;
-   if (Q)
-   {
-      ConstantCoefficient *cQ = dynamic_cast<ConstantCoefficient*>(Q);
-      MFEM_VERIFY(cQ != NULL, "Only ConstantCoefficient is supported.");
-      coeff = cQ->constant;
-   }
-   if (!(dim == 2 || dim == 3))
-   {
-      MFEM_ABORT("Dimension not supported.");
-   }
-   if (dim == 2)
-   {
-      const double constant = coeff;
-      const int NE = ne;
-      const int NQ = nq;
-      auto w = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
-      auto v = Reshape(pa_data.Write(), NQ, NE);
-      MFEM_FORALL(e, NE,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e);
-            const double J12 = J(q,1,0,e);
-            const double J21 = J(q,0,1,e);
-            const double J22 = J(q,1,1,e);
-            const double detJ = (J11*J22)-(J21*J12);
-            v(q,e) =  w[q] * constant * detJ;
-         }
-      });
-   }
-   if (dim == 3)
-   {
-      const double constant = coeff;
-      const int NE = ne;
-      const int NQ = nq;
-      auto W = ir->GetWeights().Read();
-      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
-      auto v = Reshape(pa_data.Write(), NQ,NE);
-      MFEM_FORALL(e, NE,
-      {
-         for (int q = 0; q < NQ; ++q)
-         {
-            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
-            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
-            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
-            const double detJ = J11 * (J22 * J33 - J32 * J23) -
-            /* */               J21 * (J12 * J33 - J32 * J13) +
-            /* */               J31 * (J12 * J23 - J22 * J13);
-            v(q,e) = W[q] * constant * detJ;
-         }
-      });
-   }
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void PAVectorMassApply2D(const int NE,
-                                const Array<double> &_B,
-                                const Array<double> &_Bt,
-                                const Vector &_op,
-                                const Vector &_x,
-                                Vector &_y,
-                                const int d1d = 0,
-                                const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int VDIM = 2;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(_B.Read(), Q1D, D1D);
-   auto Bt = Reshape(_Bt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, VDIM, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, VDIM, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int c = 0; c < VDIM; ++c)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double sol_x[max_Q1D];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               sol_x[qy] = 0.0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = x(dx,dy,c,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] += B(qx,dx)* s;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double d2q = B(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] += d2q * sol_x[qx];
-               }
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] *= op(qx,qy,e);
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double sol_x[max_D1D];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] = 0.0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double s = sol_xy[qy][qx];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] += Bt(dx,qx) * s;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double q2d = Bt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  y(dx,dy,c,e) += q2d * sol_x[dx];
-               }
-            }
-         }
-      }
-   });
-}
-
-template<const int T_D1D = 0,
-         const int T_Q1D = 0>
-static void PAVectorMassApply3D(const int NE,
-                                const Array<double> &_B,
-                                const Array<double> &_Bt,
-                                const Vector &_op,
-                                const Vector &_x,
-                                Vector &_y,
-                                const int d1d = 0,
-                                const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(_B.Read(), Q1D, D1D);
-   auto Bt = Reshape(_Bt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-      for (int c = 0; c < VDIM; ++ c)
-      {
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] = 0.0;
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            double sol_xy[max_Q1D][max_Q1D];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] = 0.0;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               double sol_x[max_Q1D];
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] = 0;
-               }
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double s = x(dx,dy,dz,c,e);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     sol_x[qx] += B(qx,dx) * s;
-                  }
-               }
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double wy = B(qy,dy);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     sol_xy[qy][qx] += wy * sol_x[qx];
-                  }
-               }
-            }
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double wz = B(qz,dz);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-                  }
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] *= op(qx,qy,qz,e);
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            double sol_xy[max_D1D][max_D1D];
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_xy[dy][dx] = 0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               double sol_x[max_D1D];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] = 0;
-               }
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double s = sol_xyz[qz][qy][qx];
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     sol_x[dx] += Bt(dx,qx) * s;
-                  }
-               }
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const double wy = Bt(dy,qy);
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     sol_xy[dy][dx] += wy * sol_x[dx];
-                  }
-               }
-            }
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const double wz = Bt(dz,qz);
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     y(dx,dy,dz,c,e) += wz * sol_xy[dy][dx];
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-
-static void PAVectorMassApply(const int dim,
-                              const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &Bt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-   if (dim == 2)
-   {
-      return PAVectorMassApply2D(NE, B, Bt, op, x, y, D1D, Q1D);
-   }
-   if (dim == 3)
-   {
-      return PAVectorMassApply3D(NE, B, Bt, op, x, y, D1D, Q1D);
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-void VectorMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAVectorMassApply(dim, dofs1D, quad1D, ne, maps->B, maps->Bt, pa_data, x, y);
-}
-
-} // namespace mfem
@@ -28,6 +28,11 @@ double PWConstCoefficient::Eval(ElementTransformation & T,
   return (constants(att-1));
 }

+DeviceFunctionCoefficientPtr FunctionCoefficient::GetDeviceFunction()
+{
+   return DeviceFunction;
+}
+
 double FunctionCoefficient::Eval(ElementTransformation & T,
                                 const IntegrationPoint & ip)
 {
@@ -40,6 +45,10 @@ double FunctionCoefficient::Eval(ElementTransformation & T,
   {
      return ((*Function)(transip));
   }
+   else if (DeviceFunction)
+   {
+      return ((*DeviceFunction)(Vector3(x)));
+   }
   else
   {
      return (*TDFunction)(transip, GetTime());
@@ -125,27 +134,19 @@ void VectorFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
 }

 VectorArrayCoefficient::VectorArrayCoefficient (int dim)
-   : VectorCoefficient(dim), Coeff(dim), ownCoeff(dim)
+   : VectorCoefficient(dim), Coeff(dim)
 {
   for (int i = 0; i < dim; i++)
   {
      Coeff[i] = NULL;
-      ownCoeff[i] = true;
   }
 }

-void VectorArrayCoefficient::Set(int i, Coefficient *c, bool own)
-{
-   if (ownCoeff[i]) { delete Coeff[i]; }
-   Coeff[i] = c;
-   ownCoeff[i] = own;
-}
-
 VectorArrayCoefficient::~VectorArrayCoefficient()
 {
   for (int i = 0; i < vdim; i++)
   {
-      if (ownCoeff[i]) { delete Coeff[i]; }
+      delete Coeff[i];
   }
 }

@@ -317,26 +318,17 @@ MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
   : MatrixCoefficient (dim)
 {
   Coeff.SetSize(height*width);
-   ownCoeff.SetSize(height*width);
   for (int i = 0; i < (height*width); i++)
   {
      Coeff[i] = NULL;
-      ownCoeff[i] = true;
   }
 }

-void MatrixArrayCoefficient::Set(int i, int j, Coefficient * c, bool own)
-{
-   if (ownCoeff[i*width+j]) { delete Coeff[i*width+j]; }
-   Coeff[i*width+j] = c;
-   ownCoeff[i*width+j] = own;
-}
-
 MatrixArrayCoefficient::~MatrixArrayCoefficient ()
 {
   for (int i=0; i < height*width; i++)
   {
-      if (ownCoeff[i]) { delete Coeff[i]; }
+      delete Coeff[i];
   }
 }

@@ -112,6 +112,7 @@ public:
                       const IntegrationPoint &ip);
 };

+typedef double (*DeviceFunctionCoefficientPtr)(const Vector3&);

 /// class for C-function coefficient
 class FunctionCoefficient : public Coefficient
@@ -119,6 +120,7 @@ class FunctionCoefficient : public Coefficient
 protected:
   double (*Function)(const Vector &);
   double (*TDFunction)(const Vector &, double);
+   double (*DeviceFunction)(const Vector3&);

 public:
   /// Define a time-independent coefficient from a C-function
@@ -126,6 +128,7 @@ public:
   {
      Function = f;
      TDFunction = NULL;
+      DeviceFunction = NULL;
   }

   /// Define a time-dependent coefficient from a C-function
@@ -133,6 +136,16 @@ public:
   {
      Function = NULL;
      TDFunction = tdf;
+      DeviceFunction = NULL;
+   }
+
+   /// Define a time-independent coefficient from a C-function using
+   /// Vector3 instead of a Vector.
+   FunctionCoefficient(double (*df)(const Vector3 &))
+   {
+      Function = NULL;
+      TDFunction = NULL;
+      DeviceFunction = df;
   }

   /// (DEPRECATED) Define a time-independent coefficient from a C-function
@@ -142,6 +155,7 @@ public:
   {
      Function = reinterpret_cast<double(*)(const Vector&)>(f);
      TDFunction = NULL;
+      DeviceFunction = NULL;
   }

   /// (DEPRECATED) Define a time-dependent coefficient from a C-function
@@ -151,11 +165,17 @@ public:
   {
      Function = NULL;
      TDFunction = reinterpret_cast<double(*)(const Vector&,double)>(tdf);
+      DeviceFunction = NULL;
   }

   /// Evaluate coefficient
   virtual double Eval(ElementTransformation &T,
                       const IntegrationPoint &ip);
+
+   /// Return the coefficient's C-function that uses Vector3.
+   /// Warning: for now, the returned function can only be used on the
+   /// host inside a MFEM_FORALL.
+   DeviceFunctionCoefficientPtr GetDeviceFunction();
 };

 class GridFunction;
@@ -369,7 +389,6 @@ class VectorArrayCoefficient : public VectorCoefficient
 {
 private:
   Array<Coefficient*> Coeff;
-   Array<bool> ownCoeff;

 public:
   /// Construct vector of dim coefficients.
@@ -381,7 +400,7 @@ public:
   Coefficient **GetCoeffs() { return Coeff; }

   /// Sets coefficient in the vector.
-   void Set(int i, Coefficient *c, bool own=true);
+   void Set(int i, Coefficient *c) { delete Coeff[i]; Coeff[i] = c; }

   /// Evaluates i'th component of the vector.
   double Eval(int i, ElementTransformation &T, const IntegrationPoint &ip)
@@ -501,13 +520,9 @@ public:
   void SetDeltaCoefficient(const DeltaCoefficient& _d) { d = _d; }
   /// Return the associated scalar DeltaCoefficient.
   DeltaCoefficient& GetDeltaCoefficient() { return d; }
-
-   void SetScale(double s) { d.SetScale(s); }
   void SetDirection(const Vector& _d);

-   void SetDeltaCenter(const Vector& center) { d.SetDeltaCenter(center); }
   void GetDeltaCenter(Vector& center) { d.GetDeltaCenter(center); }
-
   /** @brief Return the specified direction vector multiplied by the value
       returned by DeltaCoefficient::EvalDelta() of the associated scalar
       DeltaCoefficient. */
@@ -633,7 +648,6 @@ class MatrixArrayCoefficient : public MatrixCoefficient
 {
 private:
   Array<Coefficient *> Coeff;
-   Array<bool> ownCoeff;

 public:

@@ -641,7 +655,7 @@ public:

   Coefficient* GetCoeff (int i, int j) { return Coeff[i*width+j]; }

-   void Set(int i, int j, Coefficient * c, bool own=true);
+   void Set(int i, int j, Coefficient * c) { delete Coeff[i*width+j]; Coeff[i*width+j] = c; }

   double Eval(int i, int j, ElementTransformation &T, const IntegrationPoint &ip)
   { return Coeff[i*width+j] ? Coeff[i*width+j] -> Eval(T, ip, GetTime()) : 0.0; }
@@ -1,604 +0,0 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
-
-#ifndef MFEM_COMPLEX_FEM
-#define MFEM_COMPLEX_FEM
-
-#include "../linalg/complex_operator.hpp"
-#include "gridfunc.hpp"
-#include "linearform.hpp"
-#include "bilinearform.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#include "plinearform.hpp"
-#include "pbilinearform.hpp"
-#endif
-#include <complex>
-
-namespace mfem
-{
-
-/// Class for complex-valued grid function - real + imaginary part Vector with
-/// associated FE space.
-class ComplexGridFunction : public Vector
-{
-private:
-   GridFunction * gfr;
-   GridFunction * gfi;
-
-protected:
-   void Destroy() { delete gfr; delete gfi; }
-
-public:
-   /* @brief Construct a ComplexGridFunction associated with the
-      FiniteElementSpace @a *f. */
-   ComplexGridFunction(FiniteElementSpace *f);
-
-   void Update();
-
-   /// Assign constant values to the ComplexGridFunction data.
-   ComplexGridFunction &operator=(const std::complex<double> & value)
-   { *gfr = value.real(); *gfi = value.imag(); return *this; }
-
-   virtual void ProjectCoefficient(Coefficient &real_coeff,
-                                   Coefficient &imag_coeff);
-   virtual void ProjectCoefficient(VectorCoefficient &real_vcoeff,
-                                   VectorCoefficient &imag_vcoeff);
-
-   virtual void ProjectBdrCoefficient(Coefficient &real_coeff,
-                                      Coefficient &imag_coeff,
-                                      Array<int> &attr);
-   virtual void ProjectBdrCoefficientNormal(VectorCoefficient &real_coeff,
-                                            VectorCoefficient &imag_coeff,
-                                            Array<int> &attr);
-   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &real_coeff,
-                                             VectorCoefficient &imag_coeff,
-                                             Array<int> &attr);
-
-   FiniteElementSpace *FESpace() { return gfr->FESpace(); }
-   const FiniteElementSpace *FESpace() const { return gfr->FESpace(); }
-
-   GridFunction & real() { return *gfr; }
-   GridFunction & imag() { return *gfi; }
-   const GridFunction & real() const { return *gfr; }
-   const GridFunction & imag() const { return *gfi; }
-
-   /// Destroys the grid function.
-   virtual ~ComplexGridFunction() { Destroy(); }
-
-};
-
-/** Class for a complex-valued linear form
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ComplexLinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued field is equal to zero.
- */
-class ComplexLinearForm : public Vector
-{
-private:
-   ComplexOperator::Convention conv;
-
-protected:
-   LinearForm * lfr;
-   LinearForm * lfi;
-
-public:
-   ComplexLinearForm(FiniteElementSpace *fes,
-                     ComplexOperator::Convention
-                     convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
-       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .
-
-       The pointer @a fes is not owned by the newly constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ComplexLinearForm. */
-   ComplexLinearForm(FiniteElementSpace *fes, LinearForm *lf_r, LinearForm *lf_i,
-                     ComplexOperator::Convention
-                     convention = ComplexOperator::HERMITIAN);
-
-   virtual ~ComplexLinearForm();
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention &
-                      convention) { conv = convention; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
-                            LinearFormIntegrator *lfi_imag);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
-                              LinearFormIntegrator *lfi_imag);
-
-   /** @brief Add new Boundary Integrator, restricted to the given boundary
-       attributes.
-
-       Assumes ownership of @a lfi_real and @a lfi_imag.
-
-       The array @a bdr_attr_marker is stored internally as a pointer to the
-       given Array<int> object. */
-   void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
-                              LinearFormIntegrator *lfi_imag,
-                              Array<int> &bdr_attr_marker);
-
-   /// Adds new Boundary Face Integrator. Assumes ownership of @a lfi.
-   void AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
-                             LinearFormIntegrator *lfi_imag);
-
-   /** @brief Add new Boundary Face Integrator, restricted to the given boundary
-       attributes.
-
-       Assumes ownership of @a lfi_real and @a lfi_imag.
-
-       The array @a bdr_attr_marker is stored internally as a pointer to the
-       given Array<int> object. */
-   void AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
-                             LinearFormIntegrator *lfi_imag,
-                             Array<int> &bdr_attr_marker);
-
-   FiniteElementSpace *FESpace() const { return lfr->FESpace(); }
-
-   LinearForm & real() { return *lfr; }
-   LinearForm & imag() { return *lfi; }
-   const LinearForm & real() const { return *lfr; }
-   const LinearForm & imag() const { return *lfi; }
-
-   void Update();
-   void Update(FiniteElementSpace *f);
-
-   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
-   void Assemble();
-
-   std::complex<double> operator()(const ComplexGridFunction &gf) const;
-};
-
-
-/** Class for sesquilinear form
-
-    A sesquilinear form is a generalization of a bilinear form to complex-valued
-    fields. Sesquilinear forms are linear in the second argument but the first
-    argument involves a complex conjugate in the sense that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the SesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class SesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   /** This data member allows one to specify what should be done to the
-       diagonal matrix entries and corresponding RHS values upon elimination of
-       the constrained DoFs. */
-   mfem::Matrix::DiagonalPolicy diag_policy = mfem::Matrix::DIAG_ONE;
-
-   BilinearForm *blfr;
-   BilinearForm *blfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   SesquilinearForm(FiniteElementSpace *fes,
-                    ComplexOperator::Convention
-                    convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
-       the same integrators as the BilinearForms @a bfr and @a bfi .
-
-       The pointer @a fes is not owned by the newly constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed SesquilinearForm. */
-   SesquilinearForm(FiniteElementSpace *fes, BilinearForm *bfr, BilinearForm *bfi,
-                    ComplexOperator::Convention
-                    convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention &
-                      convention) { conv = convention; }
-
-   BilinearForm & real() { return *blfr; }
-   BilinearForm & imag() { return *blfi; }
-   const BilinearForm & real() const { return *blfr; }
-   const BilinearForm & imag() const { return *blfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
-                            BilinearFormIntegrator *bfi_imag);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
-                              BilinearFormIntegrator *bfi_imag);
-
-   /// Adds new Boundary Integrator, restricted to specific boundary attributes.
-   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
-                              BilinearFormIntegrator *bfi_imag,
-                              Array<int> &bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator *bfi_real,
-                                  BilinearFormIntegrator *bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi_real,
-                             BilinearFormIntegrator *bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The array @a bdr_marker is stored internally as a pointer to the given
-       Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi_real,
-                             BilinearFormIntegrator *bfi_imag,
-                             Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexSparseMatrix *AssembleComplexSparseMatrix();
-
-   /// Return the parallel FE space associated with the ParBilinearForm.
-   FiniteElementSpace *FESpace() const { return blfr->FESpace(); }
-
-   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
-
-   void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                         OperatorHandle &A);
-
-   /** Call this method after solving a linear system constructed using the
-       FormLinearSystem method to recover the solution as a ParGridFunction-size
-       vector in x. Use the same arguments as in the FormLinearSystem call. */
-   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
-
-   virtual void Update(FiniteElementSpace *nfes = NULL);
-
-   /// Sets diagonal policy used upon construction of the linear system
-   void SetDiagonalPolicy(mfem::Matrix::DiagonalPolicy dpolicy);
-
-   /// Returns the diagonal policy of the sesquilinear form
-   Matrix::DiagonalPolicy GetDiagonalPolicy() const {return diag_policy;}
-
-   virtual ~SesquilinearForm();
-};
-
-#ifdef MFEM_USE_MPI
-
-/// Class for parallel complex-valued grid function - real + imaginary part
-/// Vector with associated parallel FE space.
-class ParComplexGridFunction : public Vector
-{
-private:
-
-   ParGridFunction * pgfr;
-   ParGridFunction * pgfi;
-
-protected:
-   void Destroy() { delete pgfr; delete pgfi; }
-
-public:
-
-   /* @brief Construct a ParComplexGridFunction associated with the
-      ParFiniteElementSpace @a *f. */
-   ParComplexGridFunction(ParFiniteElementSpace *pf);
-
-   void Update();
-
-   /// Assign constant values to the ParComplexGridFunction data.
-   ParComplexGridFunction &operator=(const std::complex<double> & value)
-   { *pgfr = value.real(); *pgfi = value.imag(); return *this; }
-
-   virtual void ProjectCoefficient(Coefficient &real_coeff,
-                                   Coefficient &imag_coeff);
-   virtual void ProjectCoefficient(VectorCoefficient &real_vcoeff,
-                                   VectorCoefficient &imag_vcoeff);
-
-   virtual void ProjectBdrCoefficient(Coefficient &real_coeff,
-                                      Coefficient &imag_coeff,
-                                      Array<int> &attr);
-   virtual void ProjectBdrCoefficientNormal(VectorCoefficient &real_coeff,
-                                            VectorCoefficient &imag_coeff,
-                                            Array<int> &attr);
-   virtual void ProjectBdrCoefficientTangent(VectorCoefficient &real_coeff,
-                                             VectorCoefficient &imag_coeff,
-                                             Array<int> &attr);
-
-   void Distribute(const Vector *tv);
-   void Distribute(const Vector &tv) { Distribute(&tv); }
-
-   /// Returns the vector restricted to the true dofs.
-   void ParallelProject(Vector &tv) const;
-
-   FiniteElementSpace *FESpace() { return pgfr->FESpace(); }
-   const FiniteElementSpace *FESpace() const { return pgfr->FESpace(); }
-
-   ParFiniteElementSpace *ParFESpace() { return pgfr->ParFESpace(); }
-   const ParFiniteElementSpace *ParFESpace() const { return pgfr->ParFESpace(); }
-
-   ParGridFunction & real() { return *pgfr; }
-   ParGridFunction & imag() { return *pgfi; }
-   const ParGridFunction & real() const { return *pgfr; }
-   const ParGridFunction & imag() const { return *pgfi; }
-
-   virtual double ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
-                                 const IntegrationRule *irs[] = NULL) const
-   {
-      double err_r = pgfr->ComputeL2Error(exsolr, irs);
-      double err_i = pgfi->ComputeL2Error(exsoli, irs);
-      return sqrt(err_r * err_r + err_i * err_i);
-   }
-
-   virtual double ComputeL2Error(VectorCoefficient &exsolr,
-                                 VectorCoefficient &exsoli,
-                                 const IntegrationRule *irs[] = NULL,
-                                 Array<int> *elems = NULL) const
-   {
-      double err_r = pgfr->ComputeL2Error(exsolr, irs, elems);
-      double err_i = pgfi->ComputeL2Error(exsoli, irs, elems);
-      return sqrt(err_r * err_r + err_i * err_i);
-   }
-
-
-   /// Destroys grid function.
-   virtual ~ParComplexGridFunction() { Destroy(); }
-
-};
-
-/** Class for a complex-valued, parallel linear form
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParComplexLinearForm either the real or
-    imaginary integrator can be NULL.  This indicates that the corresponding
-    portion of the complex-valued field is equal to zero.
- */
-class ParComplexLinearForm : public Vector
-{
-private:
-   ComplexOperator::Convention conv;
-
-protected:
-   ParLinearForm * plfr;
-   ParLinearForm * plfi;
-
-   HYPRE_Int * tdof_offsets;
-
-public:
-
-   ParComplexLinearForm(ParFiniteElementSpace *pf,
-                        ComplexOperator::Convention
-                        convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plfr (real) and @a plfi
-       (imag) .
-
-      The pointer @a fes is not owned by the newly constructed object.
-
-      The integrators are copied as pointers and they are not owned by the newly
-      constructed ParComplexLinearForm. */
-   ParComplexLinearForm(ParFiniteElementSpace *pf, ParLinearForm *plf_r,
-                        ParLinearForm *plf_i,
-                        ComplexOperator::Convention
-                        convention = ComplexOperator::HERMITIAN);
-
-   virtual ~ParComplexLinearForm();
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention &
-                      convention) { conv = convention; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(LinearFormIntegrator *lfi_real,
-                            LinearFormIntegrator *lfi_imag);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
-                              LinearFormIntegrator *lfi_imag);
-
-   /** @brief Add new Boundary Integrator, restricted to the given boundary
-       attributes.
-
-       Assumes ownership of @a lfi_real and @a lfi_imag.
-
-       The array @a bdr_attr_marker is stored internally as a pointer to the
-       given Array<int> object. */
-   void AddBoundaryIntegrator(LinearFormIntegrator *lfi_real,
-                              LinearFormIntegrator *lfi_imag,
-                              Array<int> &bdr_attr_marker);
-
-   /// Adds new Boundary Face Integrator. Assumes ownership of @a lfi.
-   void AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
-                             LinearFormIntegrator *lfi_imag);
-
-   /** @brief Add new Boundary Face Integrator, restricted to the given boundary
-       attributes.
-
-       Assumes ownership of @a lfi_real and @a lfi_imag.
-
-       The array @a bdr_attr_marker is stored internally as a pointer to the
-       given Array<int> object. */
-   void AddBdrFaceIntegrator(LinearFormIntegrator *lfi_real,
-                             LinearFormIntegrator *lfi_imag,
-                             Array<int> &bdr_attr_marker);
-
-   ParFiniteElementSpace *ParFESpace() const { return plfr->ParFESpace(); }
-
-   ParLinearForm & real() { return *plfr; }
-   ParLinearForm & imag() { return *plfi; }
-   const ParLinearForm & real() const { return *plfr; }
-   const ParLinearForm & imag() const { return *plfi; }
-
-   void Update(ParFiniteElementSpace *pf = NULL);
-
-   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
-   void Assemble();
-
-   /// Assemble the vector on the true dofs, i.e. P^t v.
-   void ParallelAssemble(Vector &tv);
-
-   /// Returns the vector assembled on the true dofs, i.e. P^t v.
-   HypreParVector *ParallelAssemble();
-
-   std::complex<double> operator()(const ParComplexGridFunction &gf) const;
-
-};
-
-/** Class for a parallel sesquilinear form
-
-    A sesquilinear form is a generalization of a bilinear form to complex-valued
-    fields. Sesquilinear forms are linear in the second argument but but the
-    first argument involves a complex conjugate in the sense that:
-
-                a(alpha u, beta v) = conj(alpha) beta a(u, v)
-
-    The @a convention argument in the class's constructor is documented in the
-    mfem::ComplexOperator class found in linalg/complex_operator.hpp.
-
-    When supplying integrators to the ParSesquilinearForm either the real or
-    imaginary integrator can be NULL. This indicates that the corresponding
-    portion of the complex-valued material coefficient is equal to zero.
-*/
-class ParSesquilinearForm
-{
-private:
-   ComplexOperator::Convention conv;
-
-   ParBilinearForm *pblfr;
-   ParBilinearForm *pblfi;
-
-   /* These methods check if the real/imag parts of the sesqulinear form are not
-      empty */
-   bool RealInteg();
-   bool ImagInteg();
-
-public:
-   ParSesquilinearForm(ParFiniteElementSpace *pf,
-                       ComplexOperator::Convention
-                       convention = ComplexOperator::HERMITIAN);
-
-   /** @brief Create a ParSesquilinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the ParBilinearForms @a pbfr and @a pbfi .
-
-       The pointer @a pf is not owned by the newly constructed object.
-
-       The integrators are copied as pointers and they are not owned by the
-       newly constructed ParSesquilinearForm. */
-   ParSesquilinearForm(ParFiniteElementSpace *pf, ParBilinearForm *pbfr,
-                       ParBilinearForm *pbfi,
-                       ComplexOperator::Convention
-                       convention = ComplexOperator::HERMITIAN);
-
-   ComplexOperator::Convention GetConvention() const { return conv; }
-   void SetConvention(const ComplexOperator::Convention &
-                      convention) { conv = convention; }
-
-   ParBilinearForm & real() { return *pblfr; }
-   ParBilinearForm & imag() { return *pblfi; }
-   const ParBilinearForm & real() const { return *pblfr; }
-   const ParBilinearForm & imag() const { return *pblfi; }
-
-   /// Adds new Domain Integrator.
-   void AddDomainIntegrator(BilinearFormIntegrator *bfi_real,
-                            BilinearFormIntegrator *bfi_imag);
-
-   /// Adds new Boundary Integrator.
-   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
-                              BilinearFormIntegrator *bfi_imag);
-
-   /** @brief Adds new boundary Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The array @a bdr_marker is stored internally as a pointer to the given
-       Array<int> object. */
-   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi_real,
-                              BilinearFormIntegrator *bfi_imag,
-                              Array<int> &bdr_marker);
-
-   /// Adds new interior Face Integrator. Assumes ownership of @a bfi.
-   void AddInteriorFaceIntegrator(BilinearFormIntegrator *bfi_real,
-                                  BilinearFormIntegrator *bfi_imag);
-
-   /// Adds new boundary Face Integrator. Assumes ownership of @a bfi.
-   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi_real,
-                             BilinearFormIntegrator *bfi_imag);
-
-   /** @brief Adds new boundary Face Integrator, restricted to specific boundary
-       attributes.
-
-       Assumes ownership of @a bfi.
-
-       The array @a bdr_marker is stored internally as a pointer to the given
-       Array<int> object. */
-   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi_real,
-                             BilinearFormIntegrator *bfi_imag,
-                             Array<int> &bdr_marker);
-
-   /// Assemble the local matrix
-   void Assemble(int skip_zeros = 1);
-
-   /// Finalizes the matrix initialization.
-   void Finalize(int skip_zeros = 1);
-
-   /// Returns the matrix assembled on the true dofs, i.e. P^t A P.
-   /** The returned matrix has to be deleted by the caller. */
-   ComplexHypreParMatrix *ParallelAssemble();
-
-   /// Return the parallel FE space associated with the ParBilinearForm.
-   ParFiniteElementSpace *ParFESpace() const { return pblfr->ParFESpace(); }
-
-   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
-
-   void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                         OperatorHandle &A);
-
-   /** Call this method after solving a linear system constructed using the
-       FormLinearSystem method to recover the solution as a ParGridFunction-size
-       vector in x. Use the same arguments as in the FormLinearSystem call. */
-   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
-
-   virtual void Update(FiniteElementSpace *nfes = NULL);
-
-   virtual ~ParSesquilinearForm();
-};
-
-#endif // MFEM_USE_MPI
-
-}
-
-#endif // MFEM_COMPLEX_FEM
@@ -14,6 +14,7 @@
 #include "../general/text.hpp"
 #include "picojson.h"

+#include <fstream>
 #include <cerrno>      // errno
 #include <sstream>

@@ -107,7 +108,6 @@ DataCollection::DataCollection(const std::string& collection_name, Mesh *mesh_)
   precision = precision_default;
   pad_digits_cycle = pad_digits_rank = pad_digits_default;
   format = SERIAL_FORMAT; // use serial mesh format
-   compression = false;
   error = NO_ERROR;
 }

@@ -161,14 +161,6 @@ void DataCollection::SetFormat(int fmt)
   format = fmt;
 }

-void DataCollection::SetCompression(bool comp)
-{
-   compression = comp;
-#ifdef MFEM_USE_GZSTREAM
-   MFEM_ASSERT(!compression, "GZStream not enabled in MFEM build.");
-#endif
-}
-
 void DataCollection::SetPrefixPath(const std::string& prefix)
 {
   if (!prefix.empty())
@@ -227,8 +219,7 @@ void DataCollection::SaveMesh()
   }

   std::string mesh_name = GetMeshFileName();
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream mesh_file(mesh_name.c_str(), mode);
+   std::ofstream mesh_file(mesh_name.c_str());
   mesh_file.precision(precision);
 #ifdef MFEM_USE_MPI
   const ParMesh *pmesh = dynamic_cast<const ParMesh*>(mesh);
@@ -276,9 +267,7 @@ const

 void DataCollection::SaveOneField(const FieldMapIterator &it)
 {
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream field_file(GetFieldFileName(it->first).c_str(), mode);
-
+   std::ofstream field_file(GetFieldFileName(it->first).c_str());
   field_file.precision(precision);
   (it->second)->Save(field_file);
   if (!field_file)
@@ -290,8 +279,7 @@ void DataCollection::SaveOneField(const FieldMapIterator &it)

 void DataCollection::SaveOneQField(const QFieldMapIterator &it)
 {
-   const char *mode = (compression) ? "zwb6" : "w";
-   ofgzstream q_field_file(GetFieldFileName(it->first).c_str(), mode);
+   std::ofstream q_field_file(GetFieldFileName(it->first).c_str());
   q_field_file.precision(precision);
   (it->second)->Save(q_field_file);
   if (!q_field_file)
@@ -588,7 +576,7 @@ void VisItDataCollection::LoadFields()
        it != field_info_map.end(); ++it)
   {
      std::string fname = path_left + it->first + path_right;
-      ifgzstream file(fname.c_str());
+      std::ifstream file(fname.c_str());
      // TODO: in parallel, check for errors on all processors
      if (!file)
      {
@@ -725,373 +713,4 @@ void VisItDataCollection::ParseVisItRootString(const std::string& json)
   }
 }

-
-ParaViewDataCollection::~ParaViewDataCollection()
-{
-   if (myrank==0)
-   {
-      // Close the data collection
-      pvd_stream << "</Collection>" << std::endl;
-      pvd_stream << "</VTKFile>" << std::endl;
-      pvd_stream.close();
-   }
-}
-
-ParaViewDataCollection::ParaViewDataCollection(const std::string&
-                                               collection_name,
-                                               mfem::Mesh *mesh_)
-   :DataCollection(collection_name, mesh_)
-{
-   myrank = 0;
-   nprocs = 1;
-   levels_of_detail = 1;
-
-#ifdef MFEM_USE_MPI
-   lcomm = MPI_COMM_SELF;
-#endif
-
-   std::string dpath=GenerateCollectionPath();
-   std::string pvdname=dpath+"/"+GeneratePVDFileName();
-   create_directory(dpath); // this one is a serial
-   pvd_stream.open(pvdname.c_str(),std::ios::out);
-   // initialize the file
-   pvd_stream << "<?xml version=\"1.0\"?>" << std::endl;
-   pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"" << std::endl;
-   pvd_stream << "     byte_order=\"LittleEndian\"" << std::endl;
-   pvd_stream << "     compressor=\"vtkZLibDataCompressor\">" << std::endl;
-   pvd_stream << "<Collection>" << std::endl;
-}
-
-void ParaViewDataCollection::SetMesh(mfem::Mesh * new_mesh)
-{
-   DataCollection::SetMesh(new_mesh);
-}
-
-void ParaViewDataCollection::RegisterField(const std::string& field_name,
-                                           mfem::GridFunction *gf)
-{
-   DataCollection::RegisterField(field_name,gf);
-}
-
-void ParaViewDataCollection::SetLevelsOfDetail(int levels_of_detail_)
-{
-   levels_of_detail = levels_of_detail_;
-}
-
-void ParaViewDataCollection::Load(int )
-{
-   MFEM_WARNING("ParaViewDataCollection::Load() is not implemented!");
-}
-
-std::string  ParaViewDataCollection::GenerateCollectionPath()
-{
-   std::string out = "";
-   out=DataCollection::GetPrefixPath() + DataCollection::GetCollectionName();
-   return out;
-}
-
-std::string ParaViewDataCollection::GeneratePVTUPath()
-{
-   std::string out = "Cycle" + to_padded_string(cycle,pad_digits_cycle);
-   return out;
-}
-
-std::string ParaViewDataCollection::GenerateVTUPath()
-{
-   std::string out = GeneratePVTUPath();
-   return out;
-}
-
-std::string ParaViewDataCollection::GeneratePVDFileName()
-{
-   std::string out = GetCollectionName()+".pvd";
-   return out;
-}
-
-std::string ParaViewDataCollection::GeneratePVTUFileName()
-{
-   std::string out = "data.pvtu";
-   return out;
-}
-
-std::string ParaViewDataCollection::GenerateVTUFileName()
-{
-   std::string out = "proc" + to_padded_string(myrank,pad_digits_rank)+".vtu";
-   return out;
-}
-std::string ParaViewDataCollection::GenerateVTUFileName(int crank)
-{
-   std::string out = "proc" + to_padded_string(crank,pad_digits_rank)+".vtu";
-   return out;
-}
-
-void ParaViewDataCollection::Save()
-{
-   // add a new collection to the PDV file
-
-   // check if the directories are created
-   {
-      std::string path = GenerateCollectionPath()+"/"+GenerateVTUPath();
-#ifndef MFEM_USE_MPI
-      int err = create_directory(path);
-#else
-      int err;
-      if (nprocs==1)
-      {
-         err = create_directory(path);
-      }
-      else
-      {
-         err = create_directory(path,myrank,lcomm);
-      }
-#endif
-      if (err)
-      {
-         error = WRITE_ERROR;
-         MFEM_WARNING("Error creating directory: " << path);
-         return; // do not even try to write the mesh
-      }
-   }
-   // the directory is created
-
-   // define the vtu file
-   {
-      std::string fname = GenerateCollectionPath()+"/"+GenerateVTUPath()+"/"
-                          +GenerateVTUFileName();
-      std::fstream out; out.open(fname.c_str(), std::ios::out);
-      SaveDataVTU(out,levels_of_detail);
-      out.close();
-   }
-
-   // define the pvtu file only on process 0
-   if (myrank==0)
-   {
-      std::string fname = GenerateCollectionPath()+"/"+GeneratePVTUPath()+"/"
-                          +GeneratePVTUFileName();
-      std::fstream out; out.open(fname.c_str(), std::ios::out);
-
-      out << "<?xml version=\"1.0\"?>" << std::endl;
-      out << "<VTKFile type=\"PUnstructuredGrid\"";
-      out << " version =\"0.1\" byte_order=\"LittleEndian\"> " << std::endl;
-      out << "<PUnstructuredGrid GhostLevel=\"0\">" << std::endl ;
-
-      out << "<PPoints>" << std::endl;
-      out << "\t<PDataArray type=\"Float64\" ";
-      out << " Name=\"Points\" NumberOfComponents=\"3\"/>"  << std::endl;
-      out << "</PPoints>" << std::endl;
-
-      out << "<PCells>" << std::endl ;
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"connectivity\" NumberOfComponents=\"1\"/>"  << std::endl ;
-      out << "\t<PDataArray type=\"Int32\" ";
-      out << " Name=\"offsets\"      NumberOfComponents=\"1\"/>"  << std::endl ;
-      out << "\t<PDataArray type=\"UInt8\" ";
-      out << " Name=\"types\"        NumberOfComponents=\"1\"/>"  << std::endl ;
-      out << "</PCells>" << std::endl ;
-
-      out << "<PPointData>" << std::endl ;
-      for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
-      {
-         out << "<PDataArray type=\"Float64\" Name=\"" << it->first;
-         int vec_dim=it->second->VectorDim();
-         out<<"\" NumberOfComponents=\""<< vec_dim <<"\" format=\"ascii\" />" <<
-            std::endl;
-      }
-      out << "</PPointData>" << std::endl ;
-
-      // CELL DATA
-      out << "<PCellData>" << std::endl ;
-      out << "\t<PDataArray type=\"Int32\" Name=\"" << "material"
-          <<"\" NumberOfComponents=\"1\"/> " << std::endl ;
-      out << "</PCellData>" << std::endl ;
-
-      for (int ii=0; ii<nprocs; ii++)
-      {
-         // this one is generated without the path
-         std::string nfname=GenerateVTUFileName(ii);
-         out << "<Piece Source=\"" << nfname << "\"/>" << std::endl;
-      }
-      out << "</PUnstructuredGrid>" << std::endl;
-      out << "</VTKFile>" << std::endl;
-      out.close();
-
-      fname = GeneratePVTUPath()+"/"+GeneratePVTUFileName();
-      // add the pvtu file to the pvd_stream
-      pvd_stream << "<DataSet timestep=\"" << GetTime();  // GetCycle();
-      pvd_stream << "\" group=\"\" part=\"" << 0 << "\" file=\"";
-      pvd_stream << fname << "\"/>" << std::endl;
-   }
-}
-
-void ParaViewDataCollection::SaveDataVTU(std::ostream &out, int ref)
-{
-   out << "<VTKFile type=\"UnstructuredGrid\" ";
-   out << " version=\"0.1\" byte_order=\"LittleEndian\">" << std::endl;
-   out << "<UnstructuredGrid>" << std::endl;
-   mesh->PrintVTU(out,ref);
-
-   // dump out the grid functions as point data
-   out << "<PointData >" << std::endl;
-   // save the grid functions
-   // iterate over all grid functions
-   for (FieldMapIterator it=field_map.begin(); it!=field_map.end(); ++it)
-   {
-      SaveGFieldVTU(out,ref,it);
-   }
-   // iterate over all quadrature functions
-   // if the Quadrature functions are dumped as cell data
-   // the cycle should be moved before the grid functions
-   // and the PrintVTU CellData section should be open in the mesh dump
-   for (QFieldMapIterator it=q_field_map.begin(); it!=q_field_map.end(); ++it)
-   {
-      // save the quadrature functions
-      // this one is not implemented yet
-      SaveQFieldVTU(out,ref,it);
-   }
-   out << "</PointData>" << std::endl;
-   // close the mesh
-   out << "</Piece>" << std::endl; // close the piece open in the PrintVTU method
-   out << "</UnstructuredGrid>" << std::endl;
-   out << "</VTKFile>" << std::endl;
-}
-
-void ParaViewDataCollection::SaveQFieldVTU(std::ostream &out, int ref,
-                                           const QFieldMapIterator& it )
-{
-   MFEM_WARNING("SaveQFieldVTU is wotk in progress - field name:"<<it->second);
-}
-
-void ParaViewDataCollection::SaveGFieldVTU(std::ostream &out, int ref_,
-                                           const FieldMapIterator& it)
-{
-   RefinedGeometry *RefG;
-   Vector val;
-   DenseMatrix vval, pmat;
-   int vec_dim = it->second->VectorDim();
-   if (vec_dim == 1)
-   {
-      // scalar data
-      out << "<DataArray type=\"Float64\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"1\" format=\"ascii\" >" << std::endl;
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         RefG = GlobGeometryRefiner.Refine(
-                   mesh->GetElementBaseGeometry(i), ref_, 1);
-         it->second->GetValues(i, RefG->RefPts, val, pmat);
-         for (int j = 0; j < val.Size(); j++)
-         {
-            out << val(j) << '\n';
-         }
-      }
-
-   }
-   else
-   {
-      // vector data
-      out << "<DataArray type=\"Float64\" Name=\"" << it->first;
-      out << "\" NumberOfComponents=\"" << vec_dim << "\" format=\"ascii\" >" <<
-          std::endl;
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         RefG = GlobGeometryRefiner.Refine(
-                   mesh->GetElementBaseGeometry(i), ref_, 1);
-
-         it->second->GetVectorValues(i, RefG->RefPts, vval, pmat);
-
-         for (int jj = 0; jj < vval.Width(); jj++)
-         {
-            for (int ii = 0; ii < vval.Height(); ii++)
-            {
-               out << vval(ii, jj) << ' ';
-            }
-            out << std::endl;
-         }
-      }
-   }
-   out << "</DataArray>" << std::endl;
-   out.flush();
-}
-
-int ParaViewDataCollection::create_directory(const std::string &dir_name)
-{
-   // create directories recursively
-   const char path_delim = '/';
-   std::string::size_type pos = 0;
-   int err;
-
-   do
-   {
-      pos = dir_name.find(path_delim, pos+1);
-      std::string subdir = dir_name.substr(0, pos);
-      err = mkdir(subdir.c_str(), 0777);
-      err = (err && (errno != EEXIST)) ? 1 : 0;
-   }
-   while ( pos != std::string::npos );
-
-   return err;
-}
-
-#ifdef MFEM_USE_MPI
-ParaViewDataCollection::ParaViewDataCollection(const std::string&
-                                               collection_name,
-                                               mfem::ParMesh *mesh_)
-   :DataCollection(collection_name,mesh_)
-{
-   lcomm = mesh_->GetComm();
-   MPI_Comm_rank(lcomm, &myrank);
-   MPI_Comm_size(lcomm, &nprocs);
-   levels_of_detail = 1;
-
-   std::string dpath = GenerateCollectionPath();
-   std::string pvdname = dpath+"/"+GeneratePVDFileName();
-   int err = create_directory(dpath,myrank,lcomm);
-   if (err) { MFEM_ABORT("Cannot create the directory:"<<dpath);}
-   if (myrank==0)
-   {
-      pvd_stream.open(pvdname.c_str(),std::ios::out);
-      pvd_stream << "<?xml version=\"1.0\"?>" << std::endl;
-      pvd_stream << "<VTKFile type=\"Collection\" version=\"0.1\"" << std::endl;
-      pvd_stream << "     byte_order=\"LittleEndian\"" << std::endl;
-      pvd_stream << "     compressor=\"vtkZLibDataCompressor\">" << std::endl;
-      pvd_stream << "<Collection>" << std::endl;
-   }
-}
-
-int ParaViewDataCollection::create_directory(const std::string &dir_name,
-                                             int myid,
-                                             MPI_Comm lcomm_)
-{
-   // create directories recursively
-   const char path_delim = '/';
-   std::string::size_type pos = 0;
-   int err;
-
-   // create the directories only on process 0
-   if (myid==0)
-   {
-      do
-      {
-         pos = dir_name.find(path_delim, pos+1);
-         std::string subdir = dir_name.substr(0, pos);
-         err = mkdir(subdir.c_str(), 0777);
-         err = (err && (errno != EEXIST)) ? 1 : 0;
-      }
-      while ( pos != std::string::npos );
-   }
-   // broadcast the error
-   MPI_Bcast(&err, 1, MPI_INT, 0, lcomm_);
-
-   return err;
-}
-
-void ParaViewDataCollection::SetMesh(MPI_Comm comm, mfem::Mesh *new_mesh)
-{
-   DataCollection::SetMesh(new_mesh);
-   lcomm = comm;
-   MPI_Comm_rank(comm, &myrank);
-   MPI_Comm_size(comm, &nprocs);
-}
-
-#endif
-
 }  // end namespace MFEM
@@ -19,7 +19,6 @@
 #endif
 #include <string>
 #include <map>
-#include <fstream>

 namespace mfem
 {
@@ -206,7 +205,6 @@ protected:

   /// Output mesh format: see the #Format enumeration
   int format;
-   bool compression;

   /// Should the collection delete its mesh and fields
   bool own_data;
@@ -348,9 +346,6 @@ public:
       validation. */
   virtual void SetFormat(int fmt);

-   /// Set the flag for use of gz compressed files
-   void SetCompression(bool comp);
-
   /// Set the path where the DataCollection will be saved.
   void SetPrefixPath(const std::string &prefix);

@@ -468,78 +463,6 @@ public:
   virtual ~VisItDataCollection() {}
 };

-
-/// Helper class for ParaView visualization data
-class ParaViewDataCollection : public DataCollection
-{
-private:
-#ifdef MFEM_USE_MPI
-   MPI_Comm lcomm;
-#endif
-   int myrank;
-   int nprocs;
-   int levels_of_detail;
-   std::fstream pvd_stream;
-
-protected:
-   void SaveDataVTU(std::ostream &out, int ref);
-   void SaveGFieldVTU(std::ostream& out, int ref_, const FieldMapIterator& it);
-   void SaveQFieldVTU(std::ostream &out, int ref, const QFieldMapIterator& it);
-
-   std::string  GenerateCollectionPath();
-   std::string  GenerateVTUFileName();
-   std::string  GenerateVTUFileName(int rank);
-   std::string  GenerateVTUPath();
-   std::string  GeneratePVDFileName();
-   std::string  GeneratePVTUFileName();
-   std::string  GeneratePVTUPath();
-
-public:
-   /// Constructor. The collection name is used when saving the data.
-   /** If @a mesh_ is NULL, then the mesh can be set later by calling SetMesh().
-       The constructor works only in serial. */
-   ParaViewDataCollection(const std::string& collection_name,
-                          mfem::Mesh *mesh_ = NULL);
-
-#ifdef MFEM_USE_MPI
-   /// Construct a parallel ParaViewDataCollection.
-   /** Before saving the data collection, some parameters in the collection can
-       be adjusted, e.g. SetPadDigits(), SetPrefixPath(), etc. */
-   ParaViewDataCollection(const std::string& collection_name,
-                          mfem::ParMesh *mesh_ = NULL);
-#endif
-
-   virtual ~ParaViewDataCollection() override;
-
-   virtual void SetMesh(mfem::Mesh * new_mesh) override;
-
-#ifdef MFEM_USE_MPI
-   /// Set/change the mesh associated with the collection.
-   virtual void SetMesh(MPI_Comm comm, mfem::Mesh *new_mesh) override;
-#endif
-
-   /// Add a grid function to the collection
-   virtual void RegisterField(const std::string& field_name,
-                              mfem::GridFunction *gf) override;
-
-   /// Set refinement levels - every element is uniformly split based on
-   /// levels_of_detail_
-   void SetLevelsOfDetail(int levels_of_detail_);
-
-   /// Save the collection - the directory name is constructed based on the
-   /// cycle value
-   virtual void Save() override;
-
-   /// Load the collection - not implemented in the ParaView writer
-   virtual void Load(int cycle_ = 0) override;
-
-   static int create_directory(const std::string &dir_name);
-
-#ifdef MFEM_USE_MPI
-   static int create_directory(const std::string &dir_name, int myid,
-                               MPI_Comm mycom);
-#endif
-};
-
 }
+
 #endif
@@ -413,26 +413,6 @@ const DenseMatrix &IsoparametricTransformation::EvalJacobian()
   return dFdx;
 }

-const DenseMatrix &IsoparametricTransformation::EvalHessian()
-{
-   MFEM_ASSERT(space_dim == PointMat.Height(),
-               "the IsoparametricTransformation has not been finalized;"
-               " call FinilizeTransformation() after setup");
-   MFEM_ASSERT((EvalState & HESSIAN_MASK) == 0, "");
-
-   int Dim = FElem->GetDim();
-   d2shape.SetSize(FElem->GetDof(), (Dim*(Dim+1))/2);
-   d2Fdx2.SetSize(PointMat.Height(), d2shape.Width());
-   if (d2shape.Width() > 0)
-   {
-      FElem->CalcHessian(*IntPoint, d2shape);
-      Mult(PointMat, d2shape, d2Fdx2);
-   }
-   EvalState |= HESSIAN_MASK;
-
-   return d2Fdx2;
-}
-
 int IsoparametricTransformation::OrderJ()
 {
   switch (FElem->Space())
@@ -25,7 +25,6 @@ class ElementTransformation
 protected:
   const IntegrationPoint *IntPoint;
   DenseMatrix dFdx, adjJ, invJ;
-   DenseMatrix d2Fdx2;
   double Wght;
   int EvalState;
   enum StateMasks
@@ -33,8 +32,7 @@ protected:
      JACOBIAN_MASK = 1,
      WEIGHT_MASK   = 2,
      ADJUGATE_MASK = 4,
-      INVERSE_MASK  = 8,
-      HESSIAN_MASK  = 16
+      INVERSE_MASK  = 8
   };
   Geometry::Type geom;
   int space_dim;
@@ -42,7 +40,6 @@ protected:
   // Evaluate the Jacobian of the transformation at the IntPoint and store it
   // in dFdx.
   virtual const DenseMatrix &EvalJacobian() = 0;
-   virtual const DenseMatrix &EvalHessian() = 0;

   double EvalWeight();
   const DenseMatrix &EvalAdjugateJ();
@@ -71,9 +68,6 @@ public:
   const DenseMatrix &Jacobian()
   { return (EvalState & JACOBIAN_MASK) ? dFdx : EvalJacobian(); }

-   const DenseMatrix &Hessian()
-   { return (EvalState & HESSIAN_MASK) ? d2Fdx2 : EvalHessian(); }
-
   double Weight() { return (EvalState & WEIGHT_MASK) ? Wght : EvalWeight(); }

   const DenseMatrix &AdjugateJacobian()
@@ -291,7 +285,7 @@ public:
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
-   DenseMatrix dshape,d2shape;
+   DenseMatrix dshape;
   Vector shape;

   const FiniteElement *FElem;
@@ -300,9 +294,7 @@ private:
   // Evaluate the Jacobian of the transformation at the IntPoint and store it
   // in dFdx.
   virtual const DenseMatrix &EvalJacobian();
-   // Evaluate the Hessian of the transformation at the IntPoint and store it
-   // in d2Fdx2.
-   virtual const DenseMatrix &EvalHessian();
+
 public:
   void SetFE(const FiniteElement *FE) { FElem = FE; geom = FE->GetGeomType(); }
   const FiniteElement* GetFE() const { return FElem; }
@@ -203,186 +203,6 @@ void FiniteElement::CalcPhysDShape(ElementTransformation &Trans,
   Mult(vshape, Trans.InverseJacobian(), dshape);
 }

-void FiniteElement::CalcPhysLaplacian(ElementTransformation &Trans,
-                                      Vector &Laplacian) const
-{
-   MFEM_ASSERT(MapType == VALUE, "");
-
-   // Simpler routine if mapping is affine
-   if (Trans.Hessian().FNorm2() < 1e-20)
-   {
-      CalcPhysLinLaplacian(Trans, Laplacian);
-      return;
-   }
-
-   // Compute full Hessian first if non-affine
-   int size = (Dim*(Dim+1))/2;
-   DenseMatrix hess(Dof, size);
-   CalcPhysHessian(Trans,hess);
-
-   if (Dim == 3)
-   {
-      for (int nd = 0; nd < Dof; nd++)
-      {
-         Laplacian[nd] = hess(nd,0) + hess(nd,4) + hess(nd,5);
-      }
-   }
-   else if (Dim == 2)
-   {
-      for (int nd = 0; nd < Dof; nd++)
-      {
-         Laplacian[nd] = hess(nd,0) + hess(nd,2);
-      }
-   }
-   else
-   {
-      for (int nd = 0; nd < Dof; nd++)
-      {
-         Laplacian[nd] = hess(nd,0);
-      }
-   }
-}
-
-
-// Assume a linear mapping
-void FiniteElement::CalcPhysLinLaplacian(ElementTransformation &Trans,
-                                         Vector &Laplacian) const
-{
-   MFEM_ASSERT(MapType == VALUE, "");
-   int size = (Dim*(Dim+1))/2;
-   DenseMatrix hess(Dof, size);
-   DenseMatrix Gij(Dim,Dim);
-   Vector scale(size);
-
-   CalcHessian (Trans.GetIntPoint(), hess);
-   MultAAt(Trans.InverseJacobian(), Gij);
-
-   if (Dim == 3)
-   {
-      scale[0] =   Gij(0,0);
-      scale[1] = 2*Gij(0,1);
-      scale[2] = 2*Gij(0,2);
-
-      scale[3] = 2*Gij(1,2);
-      scale[4] =   Gij(2,2);
-
-      scale[5] =   Gij(1,1);
-   }
-   else if (Dim == 2)
-   {
-      scale[0] =   Gij(0,0);
-      scale[1] = 2*Gij(0,1);
-      scale[2] =   Gij(1,1);
-   }
-   else
-   {
-      scale[0] =   Gij(0,0);
-   }
-
-   for (int nd = 0; nd < Dof; nd++)
-   {
-      Laplacian[nd] = 0.0;
-      for (int ii = 0; ii < size; ii++)
-      {
-         Laplacian[nd] += hess(nd,ii)*scale[ii];
-      }
-   }
-
-}
-
-void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,
-                                     DenseMatrix& Hessian) const
-{
-   MFEM_ASSERT(MapType == VALUE, "");
-
-   // Roll 2-Tensors in vectors and 4-Tensor in Matrix, exploiting symmetry
-   Array<int> map(Dim*Dim);
-   if (Dim == 3)
-   {
-      map[0] = 0;
-      map[1] = 1;
-      map[2] = 2;
-
-      map[3] = 1;
-      map[4] = 5;
-      map[5] = 3;
-
-      map[6] = 2;
-      map[7] = 3;
-      map[8] = 4;
-   }
-   else if (Dim == 2)
-   {
-      map[0] = 0;
-      map[1] = 1;
-
-      map[2] = 1;
-      map[3] = 2;
-   }
-   else
-   {
-      map[0] = 0;
-   }
-
-   // Hessian in ref coords
-   int size = (Dim*(Dim+1))/2;
-   DenseMatrix hess(Dof, size);
-   CalcHessian(Trans.GetIntPoint(), hess);
-
-   // Gradient in physical coords
-   if (Trans.Hessian().FNorm2() > 1e-10)
-   {
-      DenseMatrix grad(Dof, Dim);
-      CalcPhysDShape(Trans, grad);
-      DenseMatrix gmap(Dof, size);
-      Mult(grad,Trans.Hessian(),gmap);
-      hess -= gmap;
-   }
-
-   // LHM
-   DenseMatrix lhm(size,size);
-   DenseMatrix invJ = Trans.Jacobian();
-   lhm = 0.0;
-   for (int i = 0; i < Dim; i++)
-   {
-      for (int j = 0; j < Dim; j++)
-      {
-         for (int k = 0; k < Dim; k++)
-         {
-            for (int l = 0; l < Dim; l++)
-            {
-               lhm(map[i*Dim+j],map[k*Dim+l]) += invJ(i,k)*invJ(j,l);
-            }
-         }
-      }
-   }
-   // Correct multiplicity
-   Vector mult(size);
-   mult = 0.0;
-   for (int i = 0; i < Dim*Dim; i++) { mult[map[i]]++; }
-   lhm.InvRightScaling(mult);
-
-   // Hessian in physical coords
-   lhm.Invert();
-   Mult( hess, lhm, Hessian);
-}
-
-const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &,
-                                             DofToQuad::Mode) const
-{
-   mfem_error("FiniteElement::GetDofToQuad(...) is not implemented for "
-              "this element!");
-   return *dof2quad_array[0]; // suppress a warning
-}
-
-FiniteElement::~FiniteElement()
-{
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      delete dof2quad_array[i];
-   }
-}
-

 void ScalarFiniteElement::NodalLocalInterpolation (
   ElementTransformation &Trans, DenseMatrix &I,
@@ -458,95 +278,6 @@ void ScalarFiniteElement::ScalarLocalInterpolation(
   }
 }

-const DofToQuad &ScalarFiniteElement::GetDofToQuad(const IntegrationRule &ir,
-                                                   DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = Dof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*Dof);
-   d2q->Bt.SetSize(Dof*nqpt);
-   d2q->G.SetSize(nqpt*Dim*Dof);
-   d2q->Gt.SetSize(Dof*nqpt*Dim);
-#ifdef MFEM_THREAD_SAFE
-   Vector c_shape(Dof);
-   DenseMatrix vshape(Dof, Dim);
-#endif
-   for (int i = 0; i < nqpt; i++)
-   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
-      CalcShape(ip, c_shape);
-      for (int j = 0; j < Dof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+Dof*i] = c_shape(j);
-      }
-      CalcDShape(ip, vshape);
-      for (int d = 0; d < Dim; d++)
-      {
-         for (int j = 0; j < Dof; j++)
-         {
-            d2q->G[i+nqpt*(d+Dim*j)] = d2q->Gt[j+Dof*(i+nqpt*d)] = vshape(j,d);
-         }
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-
-// protected method
-const DofToQuad &ScalarFiniteElement::GetTensorDofToQuad(
-   const TensorBasisElement &tb,
-   const IntegrationRule &ir, DofToQuad::Mode mode) const
-{
-   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");
-
-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const Poly_1D::Basis &basis_1d = tb.GetBasis1D();
-   const int ndof = Order + 1;
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/Dim) + 0.5);
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis_1d.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
-      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
-      }
-   }
-   dof2quad_array.Append(d2q);
-   return *d2q;
-}
-

 void NodalFiniteElement::ProjectCurl_2D(
   const FiniteElement &fe, ElementTransformation &Trans,
@@ -1914,233 +1645,6 @@ void BiQuad2DFiniteElement::ProjectDelta(int vertex, Vector &dofs) const
 #endif
 }

-
-H1Ser_QuadrilateralElement::H1Ser_QuadrilateralElement(const int p)
-   : ScalarFiniteElement(2, Geometry::SQUARE, (p*p + 3*p +6) / 2, p,
-                         FunctionSpace::Qk)
-{
-   // Store the dof_map of the associated TensorBasisElement, which will be used
-   // to create the serendipity dof map.  Its size is larger than the size of
-   // the serendipity element.
-   TensorBasisElement tbeTemp =
-      TensorBasisElement(2, p, BasisType::GaussLobatto,
-                         TensorBasisElement::DofMapType::Sr_DOF_MAP);
-   const Array<int> tp_dof_map = tbeTemp.GetDofMap();
-
-   const double *cp = poly1d.ClosedPoints(p, BasisType::GaussLobatto);
-
-   // Fixing the Nodes is exactly the same as the H1_QuadrilateralElement
-   // constructor except we only use those values of the associated tensor
-   // product dof_map that are <= the number of serendipity Dofs e.g. only DoFs
-   // 0-7 out of the 9 tensor product dofs (at quadratic order)
-   int o = 0;
-
-   for (int j = 0; j <= p; j++)
-   {
-      for (int i = 0; i <= p; i++)
-      {
-         if (tp_dof_map[o] < Nodes.Size())
-         {
-            Nodes.IntPoint(tp_dof_map[o]).x = cp[i];
-            Nodes.IntPoint(tp_dof_map[o]).y = cp[j];
-         }
-         o++;
-      }
-   }
-}
-
-void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
-                                           Vector &shape) const
-{
-   int p = (this)->GetOrder();
-   double x = ip.x, y = ip.y;
-
-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
-   Vector nodalX(p+1);
-   Vector nodalY(p+1);
-
-   edgeNodalBasis.Eval(x, nodalX);
-   edgeNodalBasis.Eval(y, nodalY);
-
-   // First, fix edge-based shape functions. Use a nodal interpolant for edge
-   // points, weighted by the linear function that vanishes on opposite edge.
-   for (int i = 0; i < p-1; i++)
-   {
-      shape(4 + 0*(p-1) + i) = (nodalX(i+1))*(1.-y);         // south edge 0->1
-      shape(4 + 1*(p-1) + i) = (nodalY(i+1))*x;              // east edge  1->2
-      shape(4 + 3*(p-1) - i - 1) = (nodalX(i+1)) * y;        // north edge 3->2
-      shape(4 + 4*(p-1) - i - 1) = (nodalY(i+1)) * (1. - x); // west edge  0->3
-   }
-
-   BiLinear2DFiniteElement bilinear = BiLinear2DFiniteElement();
-   Vector bilinearsAtIP(4);
-   bilinear.CalcShape(ip, bilinearsAtIP);
-
-   const double *edgePts(poly1d.ClosedPoints(p, BasisType::GaussLobatto));
-
-   // Next, set the shape function associated with vertex V, evaluated at (x,y)
-   // to be: bilinear function associated to V, evaluated at (x,y) - sum (shape
-   // function at edge point P, weighted by bilinear function for V evaluated at
-   // P) where the sum is taken only for points P on edges incident to V.
-
-   double vtx0fix =0;
-   double vtx1fix =0;
-   double vtx2fix =0;
-   double vtx3fix =0;
-   for (int i = 0; i<p-1; i++)
-   {
-      vtx0fix += (1-edgePts[i+1])*(shape(4 + i) +
-                                   shape(4 + 4*(p-1) - i - 1)); // bot+left edge
-      vtx1fix += (1-edgePts[i+1])*(shape(4 + 1*(p-1) + i) +
-                                   shape(4 + (p-2)-i));        // right+bot edge
-      vtx2fix += (1-edgePts[i+1])*(shape(4 + 2*(p-1) + i) +
-                                   shape(1 + 2*p-i));          // top+right edge
-      vtx3fix += (1-edgePts[i+1])*(shape(4 + 3*(p-1) + i) +
-                                   shape(3*p - i));            // left+top edge
-   }
-   shape(0) = bilinearsAtIP(0) - vtx0fix;
-   shape(1) = bilinearsAtIP(1) - vtx1fix;
-   shape(2) = bilinearsAtIP(2) - vtx2fix;
-   shape(3) = bilinearsAtIP(3) - vtx3fix;
-
-   // Interior basis functions appear starting at order p=4. These are non-nodal
-   // bubble functions.
-   if (p > 3)
-   {
-      double *legX = new double[p-1];
-      double *legY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();
-
-      storeLegendre->CalcLegendre(p-2, x, legX);
-      storeLegendre->CalcLegendre(p-2, y, legY);
-
-      int interior_total = 0;
-      for (int j = 4; j < p + 1; j++)
-      {
-         for (int k = 0; k < j-3; k++)
-         {
-            shape(4 + 4*(p-1) + interior_total)
-               = legX[k] * legY[j-4-k] * x * (1. - x) * y * (1. - y);
-            interior_total++;
-         }
-      }
-
-      delete[] legX;
-      delete[] legY;
-      delete storeLegendre;
-   }
-}
-
-void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
-                                            DenseMatrix &dshape) const
-{
-   int p = (this)->GetOrder();
-   double x = ip.x, y = ip.y;
-
-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
-   Vector nodalX(p+1);
-   Vector DnodalX(p+1);
-   Vector nodalY(p+1);
-   Vector DnodalY(p+1);
-
-   edgeNodalBasis.Eval(x, nodalX, DnodalX);
-   edgeNodalBasis.Eval(y, nodalY, DnodalY);
-
-   for (int i = 0; i < p-1; i++)
-   {
-      dshape(4 + 0*(p-1) + i,0) =  DnodalX(i+1) * (1.-y);
-      dshape(4 + 0*(p-1) + i,1) = -nodalX(i+1);
-      dshape(4 + 1*(p-1) + i,0) =  nodalY(i+1);
-      dshape(4 + 1*(p-1) + i,1) =  DnodalY(i+1)*x;
-      dshape(4 + 3*(p-1) - i - 1,0) =  DnodalX(i+1)*y;
-      dshape(4 + 3*(p-1) - i - 1,1) =  nodalX(i+1);
-      dshape(4 + 4*(p-1) - i - 1,0) = -nodalY(i+1);
-      dshape(4 + 4*(p-1) - i - 1,1) =  DnodalY(i+1) * (1.-x);
-   }
-
-   BiLinear2DFiniteElement bilinear = BiLinear2DFiniteElement();
-   DenseMatrix DbilinearsAtIP(4);
-   bilinear.CalcDShape(ip, DbilinearsAtIP);
-
-   const double *edgePts(poly1d.ClosedPoints(p, BasisType::GaussLobatto));
-
-   dshape(0,0) = DbilinearsAtIP(0,0);
-   dshape(0,1) = DbilinearsAtIP(0,1);
-   dshape(1,0) = DbilinearsAtIP(1,0);
-   dshape(1,1) = DbilinearsAtIP(1,1);
-   dshape(2,0) = DbilinearsAtIP(2,0);
-   dshape(2,1) = DbilinearsAtIP(2,1);
-   dshape(3,0) = DbilinearsAtIP(3,0);
-   dshape(3,1) = DbilinearsAtIP(3,1);
-
-   for (int i = 0; i<p-1; i++)
-   {
-      dshape(0,0) -= (1-edgePts[i+1])*(dshape(4 + 0*(p-1) + i, 0) +
-                                       dshape(4 + 4*(p-1) - i - 1,0));
-      dshape(0,1) -= (1-edgePts[i+1])*(dshape(4 + 0*(p-1) + i, 1) +
-                                       dshape(4 + 4*(p-1) - i - 1,1));
-      dshape(1,0) -= (1-edgePts[i+1])*(dshape(4 + 1*(p-1) + i, 0) +
-                                       dshape(4 + (p-2)-i, 0));
-      dshape(1,1) -= (1-edgePts[i+1])*(dshape(4 + 1*(p-1) + i, 1) +
-                                       dshape(4 + (p-2)-i, 1));
-      dshape(2,0) -= (1-edgePts[i+1])*(dshape(4 + 2*(p-1) + i, 0) +
-                                       dshape(1 + 2*p-i, 0));
-      dshape(2,1) -= (1-edgePts[i+1])*(dshape(4 + 2*(p-1) + i, 1) +
-                                       dshape(1 + 2*p-i, 1));
-      dshape(3,0) -= (1-edgePts[i+1])*(dshape(4 + 3*(p-1) + i, 0) +
-                                       dshape(3*p - i, 0));
-      dshape(3,1) -= (1-edgePts[i+1])*(dshape(4 + 3*(p-1) + i, 1) +
-                                       dshape(3*p - i, 1));
-   }
-
-   if (p > 3)
-   {
-      double *legX = new double[p-1];
-      double *legY = new double[p-1];
-      double *DlegX = new double[p-1];
-      double *DlegY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();
-
-      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
-      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);
-
-      int interior_total = 0;
-      for (int j = 4; j < p + 1; j++)
-      {
-         for (int k = 0; k < j-3; k++)
-         {
-            dshape(4 + 4*(p-1) + interior_total, 0) =
-               legY[j-4-k]*y*(1-y) * (DlegX[k]*x*(1-x) + legX[k]*(1-2*x));
-            dshape(4 + 4*(p-1) + interior_total, 1) =
-               legX[k]*x*(1-x) * (DlegY[j-4-k]*y*(1-y) + legY[j-4-k]*(1-2*y));
-            interior_total++;
-         }
-      }
-      delete[] legX;
-      delete[] legY;
-      delete[] DlegX;
-      delete[] DlegY;
-      delete storeLegendre;
-   }
-}
-
-void H1Ser_QuadrilateralElement::GetLocalInterpolation(ElementTransformation
-                                                       &Trans,
-                                                       DenseMatrix &I) const
-{
-   // For p<=4, the basis is nodal; for p>4, the quad-interior functions are
-   // non-nodal.
-   if (Order <= 4)
-   {
-      NodalLocalInterpolation(Trans, I, *this);
-   }
-   else
-   {
-      ScalarLocalInterpolation(Trans, I, *this);
-   }
-}
-
-
 BiQuadPos2DFiniteElement::BiQuadPos2DFiniteElement()
   : PositiveFiniteElement(2, Geometry::SQUARE, 9, 2, FunctionSpace::Qk)
 {
@@ -7356,7 +6860,7 @@ TensorBasisElement::TensorBasisElement(const int dims, const int p,
   : b_type(btype),
     basis1d(poly1d.GetBasis(p, b_type))
 {
-   if (dmtype == H1_DOF_MAP || dmtype == Sr_DOF_MAP)
+   if (dmtype == H1_DOF_MAP)
   {
      switch (dims)
      {
@@ -10026,7 +9530,6 @@ void L2_TetrahedronElement::ProjectDelta(int vertex, Vector &dofs) const
            const IntegrationPoint &ip = Nodes.IntPoint(i);
            dofs[i] = pow(ip.y, Order);
         }
-         break;
      case 3:
         for (int i = 0; i < Dof; i++)
         {
@@ -12183,30 +11686,6 @@ void NURBS1DFiniteElement::CalcDShape(const IntegrationPoint &ip,
   add(sum, grad, -dsum*sum*sum, shape_x, grad);
 }

-void NURBS1DFiniteElement::CalcHessian (const IntegrationPoint &ip,
-                                        DenseMatrix &hessian) const
-{
-   Vector grad(Dof);
-   Vector hess(hessian.Data(), Dof);
-
-   kv[0]->CalcShape (shape_x,  ijk[0], ip.x);
-   kv[0]->CalcDShape(grad,     ijk[0], ip.x);
-   kv[0]->CalcD2Shape(hess,    ijk[0], ip.x);
-
-   double sum = 0.0, dsum = 0.0, d2sum = 0.0;
-   for (int i = 0; i <= Order; i++)
-   {
-      sum   += (shape_x(i) *= weights(i));
-      dsum  += (   grad(i) *= weights(i));
-      d2sum += (   hess(i) *= weights(i));
-   }
-
-   sum = 1.0/sum;
-   add(sum, hess, -2*dsum*sum*sum, grad, hess);
-   add(1.0, hess, (-d2sum + 2*dsum*dsum*sum)*sum*sum, shape_x, hess);
-}
-
-
 void NURBS2DFiniteElement::SetOrder() const
 {
   Orders[0] = kv[0]->GetOrder();
@@ -12215,13 +11694,10 @@ void NURBS2DFiniteElement::SetOrder() const
   shape_y.SetSize(Orders[1]+1);
   dshape_x.SetSize(Orders[0]+1);
   dshape_y.SetSize(Orders[1]+1);
-   d2shape_x.SetSize(Orders[0]+1);
-   d2shape_y.SetSize(Orders[1]+1);

   Order = max(Orders[0], Orders[1]);
   Dof = (Orders[0] + 1)*(Orders[1] + 1);
   u.SetSize(Dof);
-   du.SetSize(Dof);
   weights.SetSize(Dof);
 }

@@ -12279,65 +11755,7 @@ void NURBS2DFiniteElement::CalcDShape(const IntegrationPoint &ip,
   }
 }

-void NURBS2DFiniteElement::CalcHessian (const IntegrationPoint &ip,
-                                        DenseMatrix &hessian) const
-{
-   double sum, dsum[2], d2sum[3];
-
-   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
-   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
-
-   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
-   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
-
-   kv[0]->CalcD2Shape(d2shape_x, ijk[0], ip.x);
-   kv[1]->CalcD2Shape(d2shape_y, ijk[1], ip.y);
-
-   sum = dsum[0] = dsum[1] = 0.0;
-   d2sum[0] = d2sum[1] = d2sum[2] = 0.0;
-   for (int o = 0, j = 0; j <= Orders[1]; j++)
-   {
-      const double sy = shape_y(j), dsy = dshape_y(j), d2sy = d2shape_y(j);
-      for (int i = 0; i <= Orders[0]; i++, o++)
-      {
-         const double sx = shape_x(i), dsx = dshape_x(i), d2sx = d2shape_x(i);
-         sum += ( u(o) = sx*sy*weights(o) );
-
-         dsum[0] += ( du(o,0) = dsx*sy*weights(o) );
-         dsum[1] += ( du(o,1) = sx*dsy*weights(o) );
-
-         d2sum[0] += ( hessian(o,0) = d2sx*sy*weights(o) );
-         d2sum[1] += ( hessian(o,1) = dsx*dsy*weights(o) );
-         d2sum[2] += ( hessian(o,2) = sx*d2sy*weights(o) );
-      }
-   }
-
-   sum = 1.0/sum;
-   dsum[0] *= sum;
-   dsum[1] *= sum;
-
-   d2sum[0] *= sum;
-   d2sum[1] *= sum;
-   d2sum[2] *= sum;
-
-   for (int o = 0; o < Dof; o++)
-   {
-      hessian(o,0) = hessian(o,0)*sum
-                     - 2*du(o,0)*sum*dsum[0]
-                     + u[o]*sum*(2*dsum[0]*dsum[0] - d2sum[0]);
-
-      hessian(o,1) = hessian(o,1)*sum
-                     - du(o,0)*sum*dsum[1]
-                     - du(o,1)*sum*dsum[0]
-                     + u[o]*sum*(2*dsum[0]*dsum[1] - d2sum[1]);
-
-      hessian(o,2) = hessian(o,2)*sum
-                     - 2*du(o,1)*sum*dsum[1]
-                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[2]);
-   }
-}
-
-
+//---------------------------------------------------------------------
 void NURBS3DFiniteElement::SetOrder() const
 {
   Orders[0] = kv[0]->GetOrder();
@@ -12351,14 +11769,9 @@ void NURBS3DFiniteElement::SetOrder() const
   dshape_y.SetSize(Orders[1]+1);
   dshape_z.SetSize(Orders[2]+1);

-   d2shape_x.SetSize(Orders[0]+1);
-   d2shape_y.SetSize(Orders[1]+1);
-   d2shape_z.SetSize(Orders[2]+1);
-
   Order = max(max(Orders[0], Orders[1]), Orders[2]);
   Dof = (Orders[0] + 1)*(Orders[1] + 1)*(Orders[2] + 1);
   u.SetSize(Dof);
-   du.SetSize(Dof);
   weights.SetSize(Dof);
 }

@@ -12432,100 +11845,10 @@ void NURBS3DFiniteElement::CalcDShape(const IntegrationPoint &ip,
   }
 }

-void NURBS3DFiniteElement::CalcHessian (const IntegrationPoint &ip,
-                                        DenseMatrix &hessian) const
-{
-   double sum, dsum[3], d2sum[6];
-
-   kv[0]->CalcShape ( shape_x, ijk[0], ip.x);
-   kv[1]->CalcShape ( shape_y, ijk[1], ip.y);
-   kv[2]->CalcShape ( shape_z, ijk[2], ip.z);
-
-   kv[0]->CalcDShape(dshape_x, ijk[0], ip.x);
-   kv[1]->CalcDShape(dshape_y, ijk[1], ip.y);
-   kv[2]->CalcDShape(dshape_z, ijk[2], ip.z);
-
-   kv[0]->CalcD2Shape(d2shape_x, ijk[0], ip.x);
-   kv[1]->CalcD2Shape(d2shape_y, ijk[1], ip.y);
-   kv[2]->CalcD2Shape(d2shape_z, ijk[2], ip.z);
-
-   sum = dsum[0] = dsum[1] = dsum[2] = 0.0;
-   d2sum[0] = d2sum[1] = d2sum[2] = d2sum[3] = d2sum[4] = d2sum[5] = 0.0;
-
-   for (int o = 0, k = 0; k <= Orders[2]; k++)
-   {
-      const double sz = shape_z(k), dsz = dshape_z(k), d2sz = d2shape_z(k);
-      for (int j = 0; j <= Orders[1]; j++)
-      {
-         const double sy = shape_y(j), dsy = dshape_y(j), d2sy = d2shape_y(j);
-         for (int i = 0; i <= Orders[0]; i++, o++)
-         {
-            const double sx = shape_x(i), dsx = dshape_x(i), d2sx = d2shape_x(i);
-            sum += ( u(o) = sx*sy*sz*weights(o) );
-
-            dsum[0] += ( du(o,0) = dsx*sy*sz*weights(o) );
-            dsum[1] += ( du(o,1) = sx*dsy*sz*weights(o) );
-            dsum[2] += ( du(o,2) = sx*sy*dsz*weights(o) );
-
-            d2sum[0] += ( hessian(o,0) = d2sx*sy*sz*weights(o) );
-            d2sum[1] += ( hessian(o,1) = dsx*dsy*sz*weights(o) );
-            d2sum[2] += ( hessian(o,2) = dsx*sy*dsz*weights(o) );
-
-            d2sum[3] += ( hessian(o,3) = sx*dsy*dsz*weights(o) );
-
-            d2sum[4] += ( hessian(o,4) = sx*sy*d2sz*weights(o) );
-            d2sum[5] += ( hessian(o,5) = sx*d2sy*sz*weights(o) );
-         }
-      }
-   }
-
-   sum = 1.0/sum;
-   dsum[0] *= sum;
-   dsum[1] *= sum;
-   dsum[2] *= sum;
-
-   d2sum[0] *= sum;
-   d2sum[1] *= sum;
-   d2sum[2] *= sum;
-
-   d2sum[3] *= sum;
-   d2sum[4] *= sum;
-   d2sum[5] *= sum;
-
-   for (int o = 0; o < Dof; o++)
-   {
-      hessian(o,0) = hessian(o,0)*sum
-                     - 2*du(o,0)*sum*dsum[0]
-                     + u[o]*sum*(2*dsum[0]*dsum[0] - d2sum[0]);
-
-      hessian(o,1) = hessian(o,1)*sum
-                     - du(o,0)*sum*dsum[1]
-                     - du(o,1)*sum*dsum[0]
-                     + u[o]*sum*(2*dsum[0]*dsum[1] - d2sum[1]);
-
-      hessian(o,2) = hessian(o,2)*sum
-                     - du(o,0)*sum*dsum[2]
-                     - du(o,2)*sum*dsum[0]
-                     + u[o]*sum*(2*dsum[0]*dsum[2] - d2sum[2]);
-
-      hessian(o,3) = hessian(o,3)*sum
-                     - du(o,1)*sum*dsum[2]
-                     - du(o,2)*sum*dsum[1]
-                     + u[o]*sum*(2*dsum[1]*dsum[2] - d2sum[3]);
-
-      hessian(o,4) = hessian(o,4)*sum
-                     - 2*du(o,2)*sum*dsum[2]
-                     + u[o]*sum*(2*dsum[2]*dsum[2] - d2sum[4]);
-
-      hessian(o,5) = hessian(o,5)*sum
-                     - 2*du(o,1)*sum*dsum[1]
-                     + u[o]*sum*(2*dsum[1]*dsum[1] - d2sum[5]);
-
-   }
-}

 // Global object definitions

+
 // Object declared in mesh/triangle.hpp.
 // Defined here to ensure it is constructed before 'Geometries'.
 Linear2DFiniteElement TriangleFE;
@@ -12537,10 +11860,6 @@ Linear3DFiniteElement TetrahedronFE;
 // Object declared in mesh/wedge.hpp.
 // Defined here to ensure it is constructed after 'poly1d' and before
 // 'Geometries'.
-// TODO: define as thread_local to prevent race conditions in GLVis, because
-// there is no "LinearWedgeFiniteElement" and WedgeFE is in turn used from two
-// different threads for different things in GLVis. We also don't want to turn
-// MFEM_THREAD_SAFE on globally. (See PR #731)
 H1_WedgeElement WedgeFE(1);

 // Object declared in geom.hpp.
--- a/Show More
+++ b/Show More