fixed euclid issue

Merge branch 'master' into functional-dev
removed Functional alias to CalcEnergy, and removed comment about prolongate
2020-02-26 14:15:38 -05:00 · 2019-12-19 15:08:07 -05:00 · 2019-08-30 15:27:32 -04:00 · 2019-08-22 15:46:01 -04:00
441 changed files with 7485 additions and 94301 deletions
@@ -40,11 +40,11 @@ before_build:
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
- cmake --build build_parallel --config Release -j 4
- cmake --build build_serial --config Release -j 4
- cmake --build build_serial --target exec --config Release -j 4
+- cmake --build build_parallel
+- cmake --build build_serial
+- cmake --build build_serial --target exec

 after_build:
- cd build_serial
- ctest -C Release --output-on-failure
+# - cmake --build build_parallel --target check
+- cmake --build build_serial --target RUN_TESTS

@@ -9,7 +9,6 @@
 # Object and library files
 *.o
 /libmfem.*
-/miniapps/common/libmfem-common.*

 # CMake generated files
 CMakeCache.txt
@@ -54,7 +53,6 @@ examples/displaced.mesh
 examples/mesh.*
 examples/ex5.mesh
 examples/Example5*
-examples/ParaView
 examples/Example9*
 examples/Example15*
 examples/Example16*
@@ -62,8 +60,6 @@ examples/sphere_refined.*
 examples/sol.*
 examples/sol_u.*
 examples/sol_p.*
-examples/sol_r.*
-examples/sol_i.*
 examples/ex9.mesh
 examples/ex9-mesh.*
 examples/ex9-init.*
@@ -89,9 +85,6 @@ examples/gnuplot_ex20p.inp
 examples/ex21*.mesh
 examples/ex21*.sol
 examples/ex21p_*.*
-examples/ex23-*.gf
-examples/ex23*.mesh
-examples/Example23*

 examples/sundials/ex9
 examples/sundials/ex1[06]
@@ -120,7 +113,6 @@ examples/petsc/sol.*
 examples/petsc/sol_p.*
 examples/petsc/sol_u.*
 examples/petsc/Example5*
-examples/petsc/ex9.mesh
 examples/petsc/ex9-mesh.*
 examples/petsc/ex9-init.*
 examples/petsc/ex9-final.*
@@ -132,11 +124,6 @@ examples/petsc/elastic_energy.*
 examples/pumi/ex1
 examples/pumi/ex[126]p

-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
-
 examples/pumi/refined.mesh
 examples/pumi/sol.gf
 examples/pumi/mesh.*
@@ -183,52 +170,19 @@ miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
-miniapps/tools/get-values

-miniapps/toys/automata
-miniapps/toys/life
-miniapps/toys/mandel
-miniapps/toys/rubik
-miniapps/toys/snake
-miniapps/toys/lissajous
-miniapps/toys/mondrian
-
-miniapps/toys/snake-init.mesh
-miniapps/toys/snake-user.mesh
-miniapps/toys/snake-joined.mesh
-miniapps/toys/snake-c*.mesh
-miniapps/toys/automata.gf
-miniapps/toys/automata.mesh
-miniapps/toys/rubik-init.mesh
-miniapps/toys/mandel.mesh
-miniapps/toys/life.gf
-miniapps/toys/life.mesh
-miniapps/toys/lissajous.mesh
-miniapps/toys/lissajous.gf
-miniapps/toys/mondrian.mesh
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/ex1
+miniapps/nurbs/ex1p
+miniapps/nurbs/ex11p
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*

-miniapps/gslib/field-diff
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
-tests/unit/sedov_tests_*
-
-# Test script output
-tests/scripts/*.err
-tests/scripts/*.out
-tests/scripts/*.msg

 # VPATH builds
 build-*/*
@@ -1,114 +1,28 @@
-language: cpp
-
 sudo: false

-stages:
-  - checks
-  - tests
-  - optional
+language: cpp

-jobs:
+matrix:
  include:
-
-    # ========================
-    #         Checks
-    # ========================
-    # - code-style
-    # - documentation
-    # - gitignore
-
-    - stage: checks
-      os: linux
-      name: "code-style"
-      addons:
-        apt:
-          packages:
-            - astyle=2.05.1-0ubuntu1
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest code-style
-
-    - stage: checks
-      os: linux
-      name: "documentation"
-      addons:
-        apt:
-          packages:
-            - doxygen
-            - graphviz
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - cd tests/scripts
-        - ./runtest documentation
-
-    - stage: checks
-      os: linux
-      name: "gitignore"
-      addons:
-        apt:
-          packages:
-            - mpich
-            - libmpich-dev
-      env: MPI=YES
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
-        - make all -j3
-        - make test-noclean
-        - cd tests/scripts
-        - ./runtest gitignore
-
-    # ========================
-    #  Optional Checks/Tests
-    # ========================
-    # - branch-history
-
-    - stage: optional
-      name: "branch-history"
-      # need full git history for the binary/big files check
-      git:
-        depth: false
-      script:
-        - cd ${TRAVIS_BUILD_DIR}
-        # update master
-        - git fetch origin master:master
-        # checkout a branch (otherwise Travis works in detached head)
-        - git checkout -b travis_tests
-        - cd tests/scripts
-        - ./runtest branch-history
-
-    # ========================
-    #       Linux tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
-    - stage: tests
-      os: linux
+    #
+    # Linux
+    #
+    - os: linux
      compiler: gcc
-      name: "Linux: Serial + Debug"
      env: DEBUG=YES
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-
+    #
    - os: linux
      compiler: gcc
-      name: "Linux: Serial"
      env: DEBUG=NO
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-
+    #
    - os: linux
      compiler: gcc
-      name: "Linux: Parallel + Debug"
      addons:
        apt:
          # sources:
@@ -135,10 +49,9 @@ jobs:
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
-
+    #
    - os: linux
      compiler: gcc
-      name: "Linux: Parallel"
      addons:
        apt:
          # sources:
@@ -165,37 +78,28 @@ jobs:
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
-
-    # ========================
-    #      Mac OS X tests
-    # ========================
-    # - serial + debug
-    # - serial
-    # - parallel + debug
-    # - parallel
-
+    #
+    # Mac OS X
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Serial + Debug"
      env: DEBUG=YES
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Serial"
      env: DEBUG=NO
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Parallel + Debug"
      env: DEBUG=YES
           MPI=YES
           CODECOV=NO
@@ -211,11 +115,10 @@ jobs:
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
          mv libmetis.a ..; rm -rf *; mv ../libmetis.a .
-
+    #
    - os: osx
      # osx_image: xcode7.3
      compiler: clang
-      name: "Mac: Parallel"
      env: DEBUG=NO
           MPI=YES
           CODECOV=YES
@@ -11,231 +11,50 @@
 Version 4.0.1 (development)
 ===========================

-Starting with this version, the MFEM open source license is changed to BSD-3.
-
 Improved GPU support
 --------------------
 - Added initial support for AMD GPUs based on HIP: a C++ runtime API and kernel
-  language that can run on both AMD and NVIDIA hardware. With this change plus
-  the libCEED and debug devise additions below, the current list of available
-  backends is: "ceed-cuda", "occa-cuda", "raja-cuda", "cuda", "hip", "debug",
-  "occa-omp", "raja-omp", "omp", "ceed-cpu", "occa-cpu", "raja-cpu", and "cpu".
-
- In addition to Examples 1 and 6, GPU/device acceleration is now available in
-  three additional example codes: 3/3p, 9/9p and 24/24p.
-
- Added support for Umpire, a resource management library that allows the
-  discovery, provision, and management of memory on machines with multiple
-  memory devices like NUMA and GPUs, see https://github.com/LLNL/Umpire.
-
- Added a "debug" device designed specifically to aid in debugging GPU code by
-  following the "device" code path (using separate host/device memory spaces and
-  host <-> device transfers) without any GPU hardware.
-
- The MFEM memory manager now supports different memory types, associated with
-  the following memory backends:
-  * Default host memory, using standard C++ new and delete,
-  * CUDA pointers, using cudaMalloc and HIP pointers, using hipMalloc,
-  * Managed CUDA/HIP memory (UVM), using cudaMallocManaged/hipMallocManaged,
-  * Umpire-managed memory, including memory pools,
-  * 32- or 64-bit aligned memory, using posix_memalign (WIN32 also supported),
-  * Debug memory with mmap/mprotect protection used by the new "debug" device.
-
- Added support for matrix-free diagonal smoothers on GPUs.
+  language that can run on both AMD and NVIDIA hardware. With this change, the
+  list of backends is: "occa-cuda", "raja-cuda", "cuda", "hip", "occa-omp",
+  "raja-omp", "omp", "occa-cpu", "raja-cpu", and "cpu".

 - Improved RAJA backend and multi-GPU MPI communications.

-libCEED support
---------------
- Added support for libCEED, the portable library for high-order operator
-  evaluation developed by the Center for Efficient Exascale Discretizations in
-  the Exascale Computing Project, https://github.com/CEED/libCEED.
-
- This initial integration includes Mass and Diffusion integrators. libCEED GPU
-  backends can be used without specific MFEM configuration, however it is highly
-  recommended to use the "cuda" build option to minimize memory transfers.
-
- Both CPU and GPU modes are available as MFEM device backends (ceed-cpu and
-  ceed-cuda), using some of the best performing CPU and GPU backends from
-  libCEED, see the sample runs in examples 1 and 6.
+Discretization improvements
+---------------------------
+- Added support for non-conforming prism AMR, including coarsening and parallel
+  load balancing. Anisotropic prism refinement is only available in the serial
+  version at the moment.

 Meshing improvements
 --------------------
- Added support for non-conforming AMR on prisms and tetrahedra, including
-  coarsening and parallel load balancing. Anisotropic prism refinement is only
-  available in the serial version at the moment.
-
 - The TMOP mesh optimization algorithms were extended to support r-adaptivity.
  Target matrices can now be constructed either via a given analytical function
  (e.g. spatial dependence of size, aspect ratio, etc., for each element) or via
  a (Par)GridFunction specified on the original mesh.

- New method Mesh::GetHilbertElementOrdering for sorting mesh elements along the
-  Hilbert curve. The ordering can be used to improve caching and parallel
-  partitioning in non-conforming AMR.
+- The TMOP mesh optimization algorithms have been improved to support AMR meshes.

 - Added support for creating refined versions of periodic meshes, making use of
  the new L2ElementRestriction class. This class also allows for computing
  geometric factors on periodic meshes using partial assembly.

- The TMOP mesh optimization algorithms were improved to support AMR meshes.
-
 - Improved element numbering after uniform mesh refinement.

-Discretization improvements
---------------------------
- Added support for GSLIB-FindPoints, a general high-order interpolation utility
-  that can robustly evaluate a GridFunction in an arbitrary collection of points
-  in physical space. See INSTALL for details on building MFEM with GSLIB, and
-  miniapps/gslib for examples of how to use this feature.
-
- Extended the support for partial assembly to DG integrarors, H(curl) bilinear
-  forms, and the vector mass and vector diffusion bilinear integrators.
-
- Added support for serendipity elements of arbitrary order on affinely-mapped
-  square elements. Basis functions for these elements can be visualized using
-  an option in the display-basis miniapp.
-
- Added support for complex-valued finite element operators and fields using a
-  2x2 block structured linear system to mimic complex arithmetic. New classes
-  include: ComplexGridFunction, SesquilinearForm, ComplexLinearForm, and their
-  parallel counterparts.
-
- Two integrators related to Stokes problems, (Q grad u, v) and (Q div v, u),
-  where u and the components of v are in H1, were added/modified to support full
-  and partial assembly modes. See the new GradientIntegrator and the updated
-  VectorDivergenceIntegrator classes in fem/bilininteg.hpp, as well as the PA
-  kernels in fem/bilininteg_gradient.cpp and fem/bilininteg_divergence.cpp.
-
- Diagonals of partially assembled operators can now be computed efficiently.
-  See the new methods AssembleDiagonal in BilinearForm, AssembleDiagonalPA in
-  BilinearFormIntegrator and the implementations in fem/bilininteg_*.cpp.
-
- Added second order derivatives of NURBS shape functions.
-
- Added initial support for NonlinearForms to support the partial assembly mode.
-
- Added a nonlinear vector valued convection integrator (Q u \cdot grad u, v)
-  where u_i and v_i are in H1. This form occurs e.g. in the Navier-Stokes
-  equations. The integrator supports the partial assembly mode for its
-  action. In full assembly mode we also provide the GetGradient method that
-  computes the linearized version of the integrator.
-
- Added a new method, MixedBilinearForm::FormRectangularLinearSystem, that can
-  be used to impose boundary conditions on the non-square off-diagonal blocks of
-  a block operator (similar to FormLinearSystem in the square case).
-
-Linear and nonlinear solvers
----------------------------
- Added a general interface for specifying and solving nonlinear constrained
-  optimization problems through the new classes OptimizationProblem and
-  OptimizationSolver, see linalg/solver.hpp.
-
- Added support for HiOp, a lightweight HPC solver for nonlinear optimization
-  problems see class HiOpNLPOptimizer and the example codes in examples/hiop.
-
- Added support for Ginkgo, a high-performance linear algebra library for GPU
-  and manycore nodes, with a focus on sparse solution of linear systems. For
-  more details see linalg/ginkgo.hpp and the example code in examples/gingko.
-
- Added Adams-Bashforth and Adams-Moulton time integrators.
-
- Added several time integrators for 2nd order ODEs.
-
- Added a block ILU(0) preconditioner for DG-type discretizations. Example 9
-  (DG advection) now takes advantage of this for implicit time integration.
-
- Added a LinearSolve(A,X) convenience method to solve dense linear systems. In
-  the trivial cases, i.e., square matrices of size 1 or 2, the system is solved
-  directly, otherwise, LU factorization is employed.
-
 New and updated examples and miniapps
 -------------------------------------
- Added two new miniapps: Find Points (serial + parallel) and Field Diff in
-  miniapps/gslib that show how GSLIB-FindPoints can be used to interpolate a
-  (Par)GridFunction in an arbitrary number of physical space points in 2D and
-  3D. The GridFunction must be in H1 and in the same space as the mesh that is
-  used to find the points.
-
- Added a new example, Example 22/22p, to demonstrate the use of the new
-  complex-valued finite element operators. The example defines and solves
-  a family of time-harmonic PDEs related to damped harmonic oscillators.
-
- Added a new example, Example 24/24p, to demonstrate usage of mixed finite
-  element spaces in bilinear forms. Partial assembly is supported in this
-  example.
-
- Added device support in Example 3/3p and Example 9/9p.
-
- Updated Examples 1/1p and 3/3p to use diagonal preconditioning in partial
-  assembly mode.
-
 - The mesh-optimizer and pmesh-optimizer miniapps have been updated to
  demonstrate the new r-adaptivity capabilities of TMOP.

- New options to reorder and partition the mesh in the mesh-explorer miniapp.
-
- The mesh-explorer miniapp now supports visualization of boundary attributes of
-  3D meshes (key 'b').
-
 - The (p)mesh-optimizer miniapp has been updated to demonstrate mesh
  optimization for an AMR mesh.

- Added a modification of Example 1 in examples/ginkgo that demonstrates the use
-  of the Gingko interface to solve a linear system.
-
- Added a modification of ex9 in examples/hiop that demonstrates the nonlinear
-  constrained optimization interface and the use of the SLBQP and HiOp solvers.
-
- Added a collection of 7 playful miniapps in miniapps/toys that illustrate the
-  meshing and visualization features of the library in more relaxed settings.
-  The toys include simulations of cellular automata, Rubik's cube, Mandelbrot
-  set, a tool to convert any image to mfem mesh, and more.
-
- Example 9 and 9p now support implicit time integration, using the new block
-  ILU(0) solvers as preconditioners for the linear system.
-
-Improved testing
----------------
- Added a new directory, tests/scripts, with several shell scripts that perform
-  simple checks on the code including: code styling, documentation formatting,
-  proper use of .gitignore, and preventing the accidental commit of large files.
-
- It is recommended that developers run the above tests scripts (via the runtest
-  script) before pushing to GitHub. See the README file in tests/scripts.
-
- The Travis CI settings have been updated to include an initial Checks stage
-  which currently runs the code-style, documentation and gitignore test scripts,
-  as well as a final stage for optional checks/tests which currently runs the
-  branch-history script.
-
 Miscellaneous
 -------------
- Upgraded the SUNDIALS interface to utilize version 5.0. This necessitated a
-  complete rework of the interface and requires changes at the application
-  level. Example usage of the new interface can be found in examples/sundials.
-
- Switched from gzstream to zstr for the implementation of zlib-compressed C++
-  output stream. The build system definition now uses MFEM_USE_ZLIB instead of
-  MFEM_USE_GZSTREAM, but the code interface (e.g. ofgzstream) remains the same.
-
- Added support for output in the ParaView XML format. Both low-order and
-  high-order Lagrange elements are supported. Output can be in ASCII or binary
-  format. The binary output can be compressed if MFEM is compiled with zlib
-  support (MFEM_USE_ZLIB). See the new ParaViewDataCollection class and the
-  updated Examples 5/5p and 9/9p.
-
- Collected object files from the miniapps/common directory into a new library,
-  libmfem-common for the convenience of application developers. The new library
-  is now used in several miniapps in the electromagnetic and tools directories.
-
- Added unit tests for time integrators.
-
-API changes
-----------
- In the enum classes MemoryType and MemoryClass, 'CUDA' was renamed to 'DEVICE'
-  which now denotes either 'CUDA' or 'HIP' depending on the build configuration.
-  In the same enum classes, 'CUDA_UVM' was renamed to 'MANAGED'.
+- Upgraded the SUNDIALS interface to utilize SUNDIALS version 5.0.  This
+  necessitated a complete rework of the interface and requires changes at
+  the application level.  Example usage of this new interface can be found
+  in the examples/sundials directory.


 Version 4.0, released on May 24, 2019
@@ -489,9 +308,6 @@ New and updated examples and miniapps
  incompressible hyperelastic equations. The example demonstrates the use of
  block nonlinear forms as well as custom block preconditioners.

- Added a new serial example (ex23) to demonstrate the use of second order
-  time integration to solve the wave equation.
-
 - Added a new electromagnetics miniapp, Maxwell, for simulating time-domain
  electromagnetics phenomena as a coupled first order system of equations.

@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 cmake_minimum_required(VERSION 2.8.11)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
@@ -164,17 +164,8 @@ if (MFEM_USE_METIS)
  find_package(METIS REQUIRED)
 endif()

-if (MFEM_USE_GINKGO)
-  find_package(Ginkgo REQUIRED)
-  if (Ginkgo_FOUND)
-    get_target_property(Ginkgo_INCLUDE_DIRS
-      Ginkgo::ginkgo INTERFACE_INCLUDE_DIRECTORIES)
-    set(Ginkgo_LIBRARIES Ginkgo::ginkgo)
-  endif()
-endif()
-
-# zlib
-if (MFEM_USE_ZLIB)
+# GZSTREAM -> zlib
+if (MFEM_USE_GZSTREAM)
  find_package(ZLIB REQUIRED)
 endif()

@@ -257,10 +248,6 @@ if (MFEM_USE_MPFR)
  find_package(MPFR REQUIRED)
 endif()

-if (MFEM_USE_CEED)
-  find_package(libCEED REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -286,12 +273,6 @@ if (MFEM_USE_PUMI)
  endif()
 endif()

-# HiOp optimizer
-if (MFEM_USE_HIOP)
-  find_package(HIOP REQUIRED)
-  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
-endif()
-
 # CUDA
 if (MFEM_USE_CUDA)
   set(CMAKE_CUDA_STANDARD 11)
@@ -305,7 +286,6 @@ if (MFEM_USE_CUDA)
      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
   endif()
   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
-   set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CUDA_CCBIN_COMPILER})
 endif()

 # OCCA
@@ -317,11 +297,6 @@ endif()
 if (MFEM_USE_RAJA)
   find_package(RAJA REQUIRED)
 endif()
-   
-# UMPIRE
-if (MFEM_USE_UMPIRE)
-   find_package(UMPIRE REQUIRED)
-endif()

 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
@@ -347,8 +322,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO Ginkgo GNUTLS NETCDF MPFR
-    PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT GECKO GNUTLS NETCDF MPFR PUMI
+    POSIXCLOCKS MFEMBacktrace ZLIB OCCA RAJA)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -560,13 +535,6 @@ if (MFEM_USE_OCCA)
    FILES_MATCHING PATTERN "*.okl")
 endif()

-# Install the libCEED files
-if (MFEM_USE_CEED)
-  install(DIRECTORY ${MFEM_SOURCE_DIRS}
-    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/libceed/*.h")
-endif()
-
 # Install ${HEADERS}
 # ---
 # foreach (HDR ${HEADERS})
@@ -3,7 +3,7 @@
 </p>

 <p align="center">
-<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
+<a href="https://github.com/mfem/mfem/blob/master/COPYRIGHT"><img alt="License" src="https://img.shields.io/badge/License-LGPL--2.1-brightgreen.svg"></a>
 <a href="https://travis-ci.org/mfem/mfem"><img alt="Build Status" src="https://travis-ci.org/mfem/mfem.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="http://mfem.github.io/doxygen/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
@@ -12,28 +12,22 @@

 # How to Contribute

-The MFEM team welcomes contributions at all levels: bugfixes; code improvements;
-simplifications; new mesh, discretization or solver capabilities; improved
-documentation; new examples and miniapps; HPC performance improvements; etc.
+The MFEM team welcomes contributions at all levels: bugfixes; code
+improvements; simplifications; new mesh, discretization or solver
+capabilities; improved documentation; new examples and miniapps;
+HPC performance improvements; ...

-MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license.
-
-If you plan on contributing to MFEM, consider reviewing the
-[issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
-already exists for your desired feature or the bug you ran into. Use a pull
-request (PR) toward the `mfem:master` branch to propose your contribution. If
-you are planning significant code changes or have questions, you may want to
-open an [issue](https://github.com/mfem/mfem/issues) before issuing a PR. In
-addition to technical contributions, we are also interested in your results and
-[simulation images](http://mfem.org/gallery/), which you can share via a pull
-request in the [mfem/web](https://github.com/mfem/web) repo.
+Use a pull request (PR) toward the `mfem:master` branch to propose your
+contribution. If you are planning significant code changes, or have any
+questions, you can also open an [issue](https://github.com/mfem/mfem/issues)
+before issuing a PR.  We also welcome your [simulation
+images](http://mfem.org/gallery/), which you can submit via a pull request in
+[mfem/web](https://github.com/mfem/web).

 See the [Quick Summary](#quick-summary) section for the main highlights of our
 GitHub workflow. For more details, consult the following sections and refer
 back to them before issuing pull requests:

- [Quick Summary](#quick-summary)
 - [Code Overview](#code-overview)
 - [GitHub Workflow](#github-workflow)
  - [MFEM Organization](#mfem-organization)
@@ -65,12 +59,6 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
  with regards to documentation and code styling.
 - Pull requests  should be issued toward `mfem:master`. Make sure
  to check the items off the [Pull Request Checklist](#pull-request-checklist).
- When your contribution is fully working and ready to be reviewed, add
-  the `ready-for-review` label.
- PRs are treated similarly to journal submission with an "editor" assigning two
-  reviewers to evaluate the changes.
- The reviewers have 3 weeks to evaluate the PR and work with the author to
-  fix issues and implement improvements.
 - After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
 - After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
@@ -80,15 +68,12 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*

 ### Code Overview

-#### Source code structure:
+- The MFEM library uses object-orient design principles which reflect, in code,
+  the independent mathematical concepts of meshing, linear algebra and finite
+  element spaces and operators.

-The MFEM library uses object-oriented design principles which reflect, in code,
-the independent mathematical concepts of meshing, linear algebra and finite
-element spaces and operators.
-
-The MFEM source code has the following structure:
-
-```
+- The MFEM source code has the following structure:
+  ```
  .
  ├── config
  │   └── cmake
@@ -98,37 +83,30 @@ The MFEM source code has the following structure:
  │   └── web
  │       └── examples
  ├── examples
-  │   ├── ginkgo
-  │   ├── hiop
  │   ├── petsc
  │   ├── pumi
  │   └── sundials
  ├── fem
-  │   └── libceed
  ├── general
  ├── linalg
  ├── mesh
  ├── miniapps
  │   ├── common
  │   ├── electromagnetics
-  │   ├── gslib
  │   ├── meshing
  │   ├── nurbs
  │   ├── performance
-  │   ├── tools
-  │   └── toys
+  │   └── tools
  └── tests
-      ├── scripts
      ├── unit
      │   ├── ...
      └── ...
-```

-#### Main directories and classes
+  ```

-The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
-classes implementing the finite element, mesh and linear algebra concepts
-respectively.
+- The main directories are `fem/`, `mesh/` and `linalg/` containing the C++
+  classes implementing the finite element, mesh and linear algebra concepts
+  respectively.

 - The main mesh classes are:
  + [`Mesh`](http://mfem.github.io/doxygen/html/classmfem_1_1Mesh.html)
@@ -150,12 +128,10 @@ respectively.
  + [`DenseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1SparseMatrix.html)
  + Sparse [smoothers](http://mfem.github.io/doxygen/html/sparsesmoothers_8hpp.html) and linear [solvers](http://mfem.github.io/doxygen/html/solvers_8hpp.html)

-#### Parallel implementation
-
-Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
-mesh for example is just a serial mesh on each task plus the information on
-shared geometric entities between different tasks. The parallel source files
-have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
+- Parallel MPI objects in MFEM inherit their serial counterparts, so a parallel
+  mesh for example is just a serial mesh on each task plus the information on
+  shared geometric entities between different tasks. The parallel source files
+  have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.

 - The main parallel classes are
  + [`ParMesh`](http://mfem.github.io/doxygen/html/solvers_8hpp.html)
@@ -166,11 +142,9 @@ have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
  + [`HypreParMatrix`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParMatrix.html) and [`HypreParVector`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreParVector.html)
  + [`HypreSolver`](http://mfem.github.io/doxygen/html/classmfem_1_1HypreSolver.html) and other [hypre classes](http://mfem.github.io/doxygen/html/hypre_8hpp.html)

-#### GPU and general device support
-
-GPU and multi-core CPU support is based on device kernels supporting different
-backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
-device/host memory manager.
+- GPU and multi-core CPU support is based on device kernels supporting different
+  backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
+  device/host memory manager.

 - The main device-relevant classes and sources are:
  + [`Device`](http://mfem.github.io/doxygen/html/device_8hpp.html)
@@ -178,31 +152,32 @@ device/host memory manager.
  + the [`MFEM_FORALL`](http://mfem.github.io/doxygen/html/forall_8hpp.html) macro
  + the [`cuda.hpp`](http://mfem.github.io/doxygen/html/cuda_8hpp.html) and [`occa.hpp`](http://mfem.github.io/doxygen/html/occa_8hpp.html) files

-#### Utilities, building and documentation
 - The `general/` directory contains C++ classes that serve as utilities for
  communication, error handling, arrays, (Boolean) tables, timing, etc.
+
 - The `config/` directory contains build-related files, both for the plain
  Makefile and the CMake build options.
+
 - The `doc/` directory contains configuration for the Doxygen code documentation
-  that can either be built locally or browsed online at
+  that can either be build locally, or browsed online at
  http://mfem.github.io/doxygen/html/index.html.

-#### Examples and tests
- `examples` and `miniapps` respectively gather simple and more fully-featured
-  demonstrations of the usage on MFEM. They both rely on `data/` for the
-  collection of meshes.
+- The `data/` directory contains a collection of small mesh files, that are used
+  in the simple example codes and more fully-featured mini applications in the
+  `examples/` and `miniapps/` directories.
+
 - The `tests/` directory contains a unit test suite and will later contain more
  tests that run example codes.

-See also the [code overview](http://mfem.org/code-overview/) section on the MFEM
-website.
+- See also the [code overview](http://mfem.org/code-overview/) section on the
+  MFEM website.

 ## GitHub Workflow

-The MFEM GitHub organization: https://github.com/mfem, is the main developer hub
-for the MFEM project.
+The GitHub organization, https://github.com/mfem, is the main developer hub for
+the MFEM project.

-If you plan to make contributions or would like to stay up-to-date with changes
+If you plan to make contributions or will like to stay up-to-date with changes
 in the code, *we strongly encourage you to [join the MFEM organization](#mfem-organization)*.

 This will simplify the workflow (by providing you additional permissions), and
@@ -211,31 +186,34 @@ will allow us to reach you directly with project announcements.

 ### MFEM Organization

-#### Getting started (GitHub)
-Before you can start, you need a GitHub account, here are a few suggestions:
-  + Create the account at: [github.com/join](https://github.com/join).
-  + For easy identification, please add your full name and maybe a picture of you at:
-    https://github.com/settings/profile.
+- Before you can start, you need a GitHub account, here are a few suggestions:
+  + Create the account at: github.com/join.
+  + For easy identification, please add your name and maybe a picture of you at: https://github.com/settings/profile.
  + To receive notification, set a primary email at: https://github.com/settings/emails.
  + For password-less pull/push over SSH, add your SSH keys at: https://github.com/settings/keys.

-#### Joining
 - [Contact us](#contact-information) for an invitation to join the MFEM GitHub
-  organization. You will receive an invitation email, which you can directly accept.
+  organization.
+
+- You should receive an invitation email, which you can directly accept.
  Alternatively, *after logging into GitHub*, you can accept the invitation at
  the top of https://github.com/mfem.
+
 - Consider making your membership public by going to https://github.com/orgs/mfem/people
-  and clicking on the organization visibility drop box next to your name.
+  and clicking on the organization visibility dropbox next to your name.
+
 - Project discussions and announcements will be posted at
  https://github.com/orgs/mfem/teams/everyone.

-#### Structure
 - The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
  repository.
+
 - The website and corresponding documentation are in the
  [web](https://github.com/mfem/web) repository.
+
 - The [PyMFEM](https://github.com/mfem/PyMFEM) repository contains a Python
  wrapper for MFEM.
+
 - The [data](https://github.com/mfem/data) repository contains additional
  (large) datafiles for MFEM.

@@ -243,9 +221,9 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 ### New Feature Development

 - A new feature should be important enough that at least one person, the
-  author, is willing to work on it and be its champion.
+  proposer, is willing to work on it and be its champion.

- The author creates a branch for the new feature (with suffix `-dev`), off
+- The proposer creates a branch for the new feature (with suffix `-dev`), off
  the `master` branch, or another existing feature branch, for example:

  ```
@@ -288,13 +266,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
  - Lean code base is easier to understand by new collaborators.
  - New features should be added only if they are necessary or generally useful.
  - Introduction of language constructions not currently used in MFEM should be
-    justified and generally avoided (to maintain portability to various systems
-    and compilers, including early access hardware).
+    justified and generally avoided (so we can build on cutting-edge systems).
  - We prefer basic C++ and the C++03 standard, to keep the code readable by
    a large audience and to make sure it compiles anywhere.

 - *Keep the code general and reasonably efficient*
-  - The main goal is fast prototyping for research and application development.
+  - Main goal is fast prototyping for research.
  - When in doubt, generality wins over efficiency.
  - Respect the needs of different users (current and/or future).

@@ -337,38 +314,15 @@ Before you can start, you need a GitHub account, here are a few suggestions:

     `[DISCUSS] Hybridized DG [hdg-dev]`

- If the PR is still a work in progress, add the `WIP` label to it and
-  optionally the `[WIP]` prefix in the title.
-
 - Add a description, appropriate labels and assign yourself to the PR. The MFEM
  team will add reviewers as appropriate.

 - List outstanding TODO items in the description, see PR #222 for an example.

- When your contribution is fully working and ready to be reviewed, add
-  the `ready-for-review` label.
-
- PRs are treated similarly to journal submission with an "editor" assigning
-  two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
-  the PR and work with the author to implement improvements and fix issues.
-
- After approval, the PR is [tested](#masternext-workflow) for a week with
-  other approved PRs in the `mfem:next` branch.
-
- Consider manually running the tests in `tests/scripts` before merging in
-  `mfem:next`, see the [README](tests/scripts/README) file in that directory
-  for more details.
-
 - Track the Travis CI and Appveyor [continuous integration](#automated-testing)
-  builds at the end of the PR. These should generally run clean, so address any
-  errors as soon as possible. Please ask if you are unsure how to do that.
+  builds at the end of the PR. These should run clean, so address any errors as
+  soon as possible.

- Note that some tests, such as the `branch-history` check in Travis are
-  safeguards that are allowed to fail in certain cases.
-
- Other tests, such as the `code-style`, `documentation` and `gitignore`
-  checks in Travis enforce MFEM-specific rules which are explained in the
-  error messages and the `tests/scripts` directory.

 ### Pull Request Checklist

@@ -380,14 +334,14 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
    - [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
 - [ ] Update `INSTALL`:
-    - [ ] Had a new optional library been added? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
+    - [ ] Had a new optional library been added? (*Make sure the external library is licensed under LGPL, not GPL!*)
    - [ ] Does `make` or `cmake` have a new target?
    - [ ] Did the requirements or the installation process change? *(rare)*
 - [ ] Update `.gitignore`:
-    - [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
+    - [ ] Check if `make distclean; git status` shows any files that are generated from the source but we don't want to track in the repository.
    - [ ] Add new patterns (just for the new files above) and re-run the above test.
 - [ ] New examples:
-    - [ ] All sample runs at the top of the example source file work.
+    - [ ] All sample runs at the top of the example work.
    - [ ] Update `examples/makefile`:
      - [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
      - [ ] Add any files generated by it to the `clean` target.
@@ -396,14 +350,13 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add the example code to the `ALL_EXE_SRCS` variable.
      - [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
   - [ ] List the new example in `doc/CodeDocumentation.dox`.
-   - [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
      - [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
      - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
      - [ ] In `examples.md`, list the example under the appropriate categories, add new categories if necessary.
      - [ ] Add a short description of the example in the "Extensive Examples" section of `features.md`.
 - [ ] New miniapps:
-   - [ ] All sample runs at the top of the miniapp source file work.
+   - [ ] All sample runs at the top of the miniapp work.
   - [ ] Update top-level `makefile` and `makefile` in corresponding miniapp directory.
   - [ ] Add the miniapp binary and any files generated by it to the top-level `.gitignore` file.
   - [ ] Update CMake build system:
@@ -411,7 +364,6 @@ Before a PR can be merged, it should satisfy the following:
      - [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
      - [ ] Consider adding a new test for the new miniapp.
   - [ ] List the new miniapp in `doc/CodeDocumentation.dox`
-   - [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
   - [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
     - [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
     - [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -421,14 +373,18 @@ Before a PR can be merged, it should satisfy the following:
   - [ ] All significant new classes, methods and functions have Doxygen-style documentation in source comments.
   - [ ] Consider adding new sample runs in existing examples to highlight the new capability.
   - [ ] Consider saving cool simulation pictures with the new capability in the Confluence gallery (LLNL only) or submitting them, via pull request, to the gallery section of the `mfem/web` repo.
-   - [ ] If this is a major new feature, consider mentioning it in the short summary inside `README` *(rare)*.
+   - [ ] If this is a major new feature, consider mentioning in the short summary inside `README` *(rare)*.
   - [ ] List major new classes in `doc/CodeDocumentation.dox` *(rare)*.
 - [ ] Update this checklist, if the new pull request affects it.
- [ ] Run `make unittest` to make sure all unit tests pass.
- [ ] Run the tests in `tests/scripts`.
+- [ ] Run the unit tests and make sure they all pass `make unittest`.
+- [ ] (LLNL only) Clone the `tests` repository and run the following tests, see `mfem/tests/README.md`:
+   - [ ] `compilers`
+   - [ ] `memcheck`
+   - [ ] `documentation`
 - [ ] (LLNL only) After merging:
   - [ ] Regenerate `README.html` files from companion documentation pull requests.
-   - [ ] Update internal tests to include the new features.
+   - [ ] Update the `baseline` and `compiler` tests, add new tests if necessary.
+   - [ ] Consider updating the script `mfem/tests/sample-runs` (`sample-runs-serial` and `sample-runs-parallel`).

 ### Master/Next Workflow

@@ -487,7 +443,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
  (replace `3.3.2` with current release):
  - Rename the current `next` branch to `next-pre-v3.3.2`.
  - Create a new `next` branch starting from the `v3.3.2` release.
-  - Local copies of `next` can then be updated with `git fetch origin next && git checkout -B next origin/next`.
+  - Local copies of `next` can then be updated with `git checkout -B next origin/next`.

 ### Release Checklist

@@ -506,9 +462,9 @@ MFEM uses a `master`/`next`-branch workflow as described below:
 - [ ] Create the release tarball and push to `mfem/releases`.
 - [ ] Recreate the `next` branch as described in previous section.
 - [ ] Update and push documentation  to `mfem/doxygen`.
- [ ] Update URL shortlinks:
-    - [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. http://mfem.github.io/releases/mfem-3.1.tgz.
-    - [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
+- [ ] Update URL shorlinks:
+    - [ ] Create a shortlink at [https://goo.gl/](https://goo.gl/) for the release tarball, e.g. http://mfem.github.io/releases/mfem-3.1.tgz.
+    - [ ] (LLNL only) Add and commit the new shorlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
    - [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
 - [ ] Update website in `mfem/web` repo:
    - Update version and shortlinks in `src/index.md` and `src/download.md`.
@@ -517,8 +473,6 @@ MFEM uses a `master`/`next`-branch workflow as described below:

 ## LLNL Workflow

-### Mirroring on Bitbucket
-
 - The GitHub `master` and `next` branches are mirrored to the LLNL institutional
  Bitbucket repository as `gh-master` and `gh-next`.

@@ -562,12 +516,12 @@ CMake is used to generate the MSVC Project files and drive the build.  A release
 and debug build is performed with a simple run of `ex1` to verify the executable.

 ### Tests at LLNL
-
- We mirror the `master` and `next` branches internally (to `gh-master`
+At LLNL, we mirror the `master` and `next` branches internally (to `gh-master`
 and `gh-next`) and run longer nightly tests via cron. On the weekends, a more
 extensive test is run which extracts and executes all the different sample runs
 from each example.

+
 ## Contact Information

 - Contact the MFEM team by posting to the [GitHub issue tracker](https://github.com/mfem/mfem).
@@ -0,0 +1,64 @@
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability see http://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.
+
+COPYRIGHT
+
+The following copyright applies to each file in the MFEM distribution, unless
+otherwise stated in the file:
+
+Copyright (c) 2010, Lawrence Livermore National Security, LLC.  Produced at the
+Lawrence Livermore National Laboratory under Contract No. DE-AC52-07NA27344 with
+the DOE.
+
+Neither the United States Government nor Lawrence Livermore National Security,
+LLC nor any of their employees, makes any warranty, express or implied, or
+assumes any liability or responsibility for the accuracy, completeness, or
+usefulness of any information, apparatus, product, or process disclosed, or
+represents that its use would not infringe privately-owned rights.
+
+Also, reference herein to any specific commercial products, process, or services
+by trade name, trademark, manufacturer or otherwise does not necessarily
+constitute or imply its endorsement, recommendation, or favoring by the United
+States Government or Lawrence Livermore National Security, LLC. The views and
+opinions of authors expressed herein do not necessarily state or reflect those
+of the United States Government or Lawrence Livermore National Security, LLC,
+and shall not be used for advertising or product endorsement purposes.
+
+
+LICENSE
+
+MFEM is free software; you can redistribute it and/or modify it under the terms
+of the GNU Lesser General Public License (as published by the Free Software
+Foundation) version 2.1 dated February 1999, with the following EXCEPTIONS:
+
+   1. Subclasses of MFEM classes do not constitute a derivative work.
+
+   2. Static linking of applications to the MFEM library does not constitute a
+      derivative work and does not require the author to provide source code for
+      the application, use the shared MFEM library, or link their applications
+      against a user-supplied version of MFEM.
+
+MFEM is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the IMPLIED WARRANTY OF MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the terms and conditions of the GNU Lesser General
+Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this library (file LICENSE); if not, write to the Free Software Foundation,
+Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+
+CONTACT INFORMATION
+
+The software is released under LLNL-CODE-443211. Please see
+
+                   http://mfem.org
+
+for more information and source code availability.
@@ -349,12 +349,6 @@ MFEM_USE_STRUMPACK = YES/NO
   classes. When enabled, this option uses the STRUMPACK_* library options, see
   below.

-MFEM_USE_GINKGO = YES/NO
-   Enable MFEM functionality based on the Ginkgo library, which provides
-   iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
-   https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
-   Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.
-
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -402,10 +396,10 @@ MFEM_USE_CONDUIT = YES/NO
   an installation of Conduit. If Conduit was built with HDF5 support, it also
   requires an installation of HDF5 (see also MFEM_USE_NETCDF).

-MFEM_USE_ZLIB = YES/NO
+MFEM_USE_GZSTREAM = YES/NO
   Enables use of on-the-fly gzip compressed streams. With this feature enabled
   (YES), MFEM can compress its output files on-the-fly. In addition, it can
-   read back files compressed with zlib (or any compression utility capable
+   read back files compressed with gzstream (or any compression utility capable
   of creating a gzip-compatible output such as gzip).
   MFEM will write compressed files if the mode argument in the constructor
   includes a 'z' character. With this feature disabled (NO), MFEM will not be
@@ -421,15 +415,6 @@ MFEM_USE_PUMI = YES/NO
   models and effectively supports automated adaptive analysis. PUMI enables
   support for parallel unstructured mesh modifications in MFEM.

-MFEM_USE_UMPIRE = YES/NO
-   Enables support for Umpire, a resource management library that allows the
-   discovery, provision, and management of memory on machines with multiple
-   memory devices like NUMA and GPUs.
-
-MFEM_USE_HIOP = YES/NO
-  Enable the usage of HiOp (https://github.com/LLNL/hiop) in MFEM. HiOp is an
-  HPC solver for nonlinear optimization problems.
-
 MFEM_USE_CUDA = YES/NO
   Enables support for CUDA devices in MFEM. CUDA is a parallel computing
   platform and programming model for general computing on graphical processing
@@ -457,17 +442,6 @@ MFEM_USE_OCCA = YES/NO
   backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
   in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.

-MFEM_USE_GSLIB = YES/NO
-   Enables MFEM functionality based on the GSLIB library, and specifically its
-   FindPoints component, which provides a robust algorithms to evaluate finite
-   element functions in a collection of points in physical space. When enabled,
-   the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.
-
-MFEM_USE_CEED = YES/NO
-   Enables support for the libCEED library in MFEM. libCEED is a portable
-   library for performant high-order operator evaluation developed by the Center
-   for Efficient Exascale Discretizations in the Exascale Computing Project.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -544,12 +518,6 @@ The specific libraries and their options are:
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.

- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
-  C++ compiler that supports the C++-11 standard. For additional requirements
-  and dependencies of specific modules see the Ginkgo webpage below.
-  URL: https://ginkgo-project.github.io
-  Options: GINKGO_OPT (Not used), GINKGO_LIB.
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -581,56 +549,33 @@ The specific libraries and their options are:
       https://support.hdfgroup.org/HDF5 (HDF5)
  Options: SIDRE_OPT, SIDRE_LIB.

- Conduit (optional), used when MFEM_USE_CONDUIT = YES. Conduit Mesh Blueprint
+- Conduit, used when MFEM_USE_CONDUIT = YES. Direct Conduit Mesh Blueprint
  support requires Conduit >= v0.3.1 and VisIt >= v2.13.1 to read the output.
  URL: https://github.com/LLNL/conduit (Conduit)
       https://support.hdfgroup.org/HDF5 (HDF5)
  Options: CONDUIT_OPT, CONDUIT_LIB.

- PUMI (optional), used when MFEM_USE_PUMI = YES.
+- PUMI, used when MFEM_USE_PUMI = YES.
  URL: https://scorec.rpi.edu/pumi
  Options: PUMI_OPT, PUMI_LIB.

- HiOp (optional), used when MFEM_USE_HIOP = YES.
-  URL: https://github.com/LLNL/hiop
-  Options: HIOP_OPT, HIOP_LIB.
-
- GSLIB (optional), used when MFEM_USE_GSLIB = YES. The gslib library must be
-  built prior to the MFEM build, as follows: download gslib-1.0.5, untar it at
-  the same level as MFEM and create a symbolic link: "ln -s gslib-1.0.5 gslib".
-  Build gslib in parallel or in serial based on the desired MFEM build: "make
-  clean; make CC=mpicc" or "make clean; make CC=gcc MPI=0". Build MFEM with
-  MFEM_USE_GSLIB=YES.
-  URL: https://github.com/gslib/gslib/archive/v1.0.5.tar.gz
-  Options: GSLIB_OPT, GSLIB_LIB.
-
- CUDA (optional), used when MFEM_USE_CUDA = YES.
+- CUDA, used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.

- HIP (optional), used when MFEM_USE_HIP = YES.
+- HIP, used when MFEM_USE_HIP = YES.
  URL: https://rocm.github.io/ROCmInstall.html
  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.

- OCCA (optional), used when MFEM_USE_OCCA = YES.
+- OCCA, used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.

- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED's master
-  branch, specifically, git-hash c00ee0d or later.
-  URL: https://github.com/CEED/libCEED
-       https://ceed.exascaleproject.org/libceed
-  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-
- RAJA (optional), used when MFEM_USE_RAJA = YES.
+- RAJA, used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
  URL: https://github.com/LLNL/RAJA
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.

- Umpire, used when MFEM_USE_UMPIRE = YES.
-  URL: https://github.com/LLNL/Umpire
-  Options: UMPIRE_DIR, UMPIRE_OPT, UMPIRE_LIB.
-
 - MPFR (optional), used when MFEM_USE_MPFR = YES.
  URL: http://mpfr.org, it depends on the GMP library: https://gmplib.org
  Options: MPFR_OPT, MPFR_LIB.
@@ -641,11 +586,12 @@ The specific libraries and their options are:
  URL: http://www.nongnu.org/libunwind
  Options: LIBUNWIND_OPT, LIBUNWIND_LIB.

- ZLIB (optional), used when MFEM_USE_ZLIB = YES, or when MFEM_USE_NETCDF =
+- ZLIB (optional), used when MFEM_USE_GZSTREAM = YES, or when MFEM_USE_NETCDF =
  YES (in the default settings for NETCDF_OPT and NETCDF_LIB).
  URL: https://zlib.net
  Options: ZLIB_OPT, ZLIB_LIB.

+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -761,18 +707,14 @@ MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
-MFEM_USE_GINKGO
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
-MFEM_USE_ZLIB
+MFEM_USE_GZSTREAM
 MFEM_USE_PUMI
-MFEM_USE_HIOP
 MFEM_USE_CUDA
 MFEM_USE_OCCA
-MFEM_USE_CEED
 MFEM_USE_RAJA
-MFEM_USE_UMPIRE
 MFEM_USE_SIDRE

 The following options are CMake specific:
@@ -814,17 +756,14 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - SuiteSparse
 - SuperLUDist, STRUMPACK
 - ParMETIS
- - Ginkgo
 - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
 - NETCDF
 - MPFR
 - LIBUNWIND
 - POSIXCLOCKS
 - PUMI
- - HIOP
 - OCCA
 - RAJA
- - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled

 The following built-in CMake packages are also used:
@@ -1,38 +1,504 @@
-BSD 3-Clause License
+                  GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 2.1, February 1999

-Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC
-All rights reserved.
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.

-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]

-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
+                            Preamble

-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.

-* Neither the name of the copyright holder nor the names of its
-  contributors may be used to endorse or promote products derived from
-  this software without specific prior written permission.
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.

-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.

-Additionally this project distributes 3 libraries with their own respective
-licenses which can be found in their code and attached license files. These
-libraries and their licenses are as follows:
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.

-* Picojson (fem/picojson.h) -- Custom 2-clause license
-* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
-* Zstr (general/zstr.hpp) -- MIT license
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.

+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+                  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+                            NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
+    USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random
+  Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
@@ -1,20 +0,0 @@
-This work was produced under the auspices of the U.S. Department of Energy by
-Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344.
-
-This work was prepared as an account of work sponsored by an agency of
-the United States Government. Neither the United States Government nor
-Lawrence Livermore National Security, LLC, nor any of their employees
-makes any warranty, expressed or implied, or assumes any legal liability
-or responsibility for the accuracy, completeness, or usefulness of any
-information, apparatus, product, or process disclosed, or represents that
-its use would not infringe privately owned rights.
-
-Reference herein to any specific commercial product, process, or service
-by trade name, trademark, manufacturer, or otherwise does not necessarily
-constitute or imply its endorsement, recommendation, or favoring by the
-United States Government or Lawrence Livermore National Security, LLC.
-
-The views and opinions of authors expressed herein do not necessarily
-state or reflect those of the United States Government or Lawrence
-Livermore National Security, LLC, and shall not be used for advertising
-or product endorsement purposes.
@@ -14,7 +14,7 @@ supercomputers.

 * For building instructions, see the file INSTALL, or type "make help".

-* Copyright and licensing information can be found in files LICENSE and NOTICE.
+* Copyright and licensing information can be found in the file COPYRIGHT.

 * The best starting point for new users interested in MFEM's features is the
  interactive documentation in examples/README.html.
@@ -61,9 +61,8 @@ nonlinear solvers, preconditioners, time integrators, etc.
 For examples of using MFEM, see the examples/ and miniapps/ directories, as well
 as the OpenGL visualization tool GLVis which is available at http://glvis.org.

-MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See LICENSE and NOTICE for details.
+This project is released under the LGPL v2.1 license with static linking
+exception. See files COPYRIGHT and LICENSE file for full details.

-SPDX-License-Identifier: BSD-3-Clause
-LLNL Release Number: LLNL-CODE-806117
+LLNL Release Number: LLNL-CODE-443211
 DOI: 10.11578/dc.20171025.1248
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 set(HDRS
 config.hpp
@@ -196,8 +196,8 @@ IF (DEFINED TPL_ENABLE_METIS)
  SET(MFEM_USE_METIS ${TPL_ENABLE_METIS} CACHE BOOL "Enable METIS usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_ZLIB)
-  SET(MFEM_USE_ZLIB ${TPL_ENABLE_ZLIB} CACHE BOOL "Enable zlib for compressed data streams." FORCE)
+IF (DEFINED TPL_ENABLE_GZSTREAM)
+  SET(MFEM_USE_GZSTREAM ${TPL_ENABLE_GZSTREAM} CACHE BOOL "Enable gzstream for compressed data streams." FORCE)
 ENDIF()

 IF (DEFINED TPL_ENABLE_LIBUNWIND)
@@ -232,10 +232,6 @@ IF (DEFINED TPL_ENABLE_GECKO)
  SET(MFEM_USE_GECKO ${TPL_ENABLE_GECKO} CACHE BOOL "Enable GECKO usage" FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_GINKGO)
-  SET(MFEM_USE_GINKGO ${TPL_ENABLE_GINKGO} CACHE BOOL "Enable GINKGO usage" FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_GNUTLS)
  SET(MFEM_USE_GNUTLS ${TPL_ENABLE_GNUTLS} CACHE BOOL "Enable GNUTLS usage" FORCE)
 ENDIF()
@@ -275,7 +271,3 @@ ENDIF()
 IF (DEFINED TPL_ENABLE_RAJA)
  SET(MFEM_USE_RAJA ${TPL_ENABLE_RAJA} CACHE BOOL "Enable RAJA" FORCE)
 ENDIF()
-
-IF (DEFINED TPL_ENABLE_UMPIRE)
-  SET(MFEM_USE_UMPIRE ${TPL_ENABLE_UMPIRE} CACHE BOOL "Enable Umpire" FORCE)
-ENDIF()
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+# the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+# reserved. See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 include(${CMAKE_CURRENT_LIST_DIR}/MFEMConfigVersion.cmake)

@@ -20,7 +20,7 @@ set(MFEM_USE_METIS @MFEM_USE_METIS@)
 set(MFEM_USE_METIS_5 @MFEM_USE_METIS_5@)
 set(MFEM_DEBUG @MFEM_DEBUG@)
 set(MFEM_USE_EXCEPTIONS @MFEM_USE_EXCEPTIONS@)
-set(MFEM_USE_ZLIB @MFEM_USE_ZLIB@)
+set(MFEM_USE_GZSTREAM @MFEM_USE_GZSTREAM@)
 set(MFEM_USE_LIBUNWIND @MFEM_USE_LIBUNWIND@)
 set(MFEM_USE_LAPACK @MFEM_USE_LAPACK@)
 set(MFEM_THREAD_SAFE @MFEM_THREAD_SAFE@)
@@ -34,7 +34,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GECKO @MFEM_USE_GECKO@)
-set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
@@ -45,8 +44,6 @@ set(MFEM_USE_PUMI @MFEM_USE_PUMI@)
 set(MFEM_USE_CUDA @MFEM_USE_CUDA@)
 set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
 set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
-set(MFEM_USE_CEED @MFEM_USE_CEED@)
-set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #ifndef MFEM_CONFIG_HEADER
 #define MFEM_CONFIG_HEADER
@@ -49,8 +49,8 @@
 // Throw an exception on errors.
 #cmakedefine MFEM_USE_EXCEPTIONS

-// Enable zlib in MFEM.
-#cmakedefine MFEM_USE_ZLIB
+// Enable gzstream in MFEM.
+#cmakedefine MFEM_USE_GZSTREAM

 // Enable backtraces for mfem_error through libunwind.
 #cmakedefine MFEM_USE_LIBUNWIND
@@ -92,9 +92,6 @@
 // Enable functionality based on the Gecko library
 #cmakedefine MFEM_USE_GECKO

-// Enable functionality based on the Ginkgo library
-#cmakedefine MFEM_USE_GINKGO
-
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

@@ -113,9 +110,6 @@
 // Enable MFEM functionality based on the PUMI library
 #cmakedefine MFEM_USE_PUMI

-// Enable MFEM functionality based on the HiOp library
-#cmakedefine MFEM_USE_HIOP
-
 // Build the GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA
@@ -126,12 +120,6 @@
 // Enable MFEM functionality based on the OCCA library
 #cmakedefine MFEM_USE_OCCA

-// Enable MFEM functionality based on the libCEED library
-#cmakedefine MFEM_USE_CEED
-
-// Enable MFEM functionality based on the Umpire library
-#cmakedefine MFEM_USE_UMPIRE
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - AXOM_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Check for "abi::__cxa_demangle" in <cxxabi.h>.
 # Defines the variables:
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - CONDUIT_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - GECKO_FOUND
@@ -1,36 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - HIOP_FOUND
-#   - HIOP_INCLUDE_DIRS
-#   - HIOP_LIBRARIES
-
-include(MfemCmakeUtilities)
-mfem_find_package(HIOP HIOP HIOP_DIR
-  "include" "hiopInterface.hpp"
-  "lib" "hiop"
-  "Paths to headers required by HIOP."
-  "Libraries required by HIOP.")
-
-# this test fails with parallel MFEM since mpi.h is not available (cxx compiler is used for some reason)
-#  CHECK_BUILD HIOP_VERSION_OK TRUE
-#"
-##include <hiopInterface.hpp>
-#using namespace hiop;
-#int main(int argc, char *argv[])
-#{
-#   MPI_Init(&argc, &argv);
-#   MPI_Comm comm = MPI_COMM_WORLD;
-#
-#   return 0;
-#}
-#")
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - HYPRE_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Find libdl.
 # Defines:
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Find libunwind.
 # Defines the following variables:
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - METIS_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Find the libraries needed for backtrace in MFEM. This is a "meta-package",
 # i.e. it simply combines all packages from MFEMBacktrace_REQUIRED_PACKAGES.
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Find MPFR.
 # Defines the following variables:
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - MESQUITE_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - NETCDF_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - OCCA_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Find POSIX clocks.
 # Defines:
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Sets the following variables
 #   - ParMETIS_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - RAJA_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Sets the following variables:
 #   - STRUMPACK_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Defines the following variables:
 #   - SUNDIALS_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Sets the following variables:
 #   - Scotch_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Sets the following variables
 #   - SuiteSparse_FOUND
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Sets the following variables:
 #   - SuperLUDist_FOUND
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - UMPIRE_FOUND
-#   - UMPIRE_LIBRARIES
-#   - UMPIRE_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
-  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Find GnuTLS, searching ${GNUTLS_DIR} first. If not successful, tries the
 # standard FindGnuTLS.cmake. Defines:
@@ -1,19 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - CEED_FOUND
-#   - CEED_LIBRARIES
-#   - CEED_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(libCEED CEED CEED_DIR "include" ceed.h "lib" ceed
-  "Paths to headers required by libCEED." "Libraries required by libCEED.")
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Function that converts a version string of the form 'major[.minor[.patch]]' to
 # the integer ((major * 100) + minor) * 100 + patch.
@@ -70,11 +70,6 @@ function(add_mfem_examples EXE_SRCS)
    endif()
  endif()
  foreach(SRC_FILE IN LISTS ${EXE_SRCS})
-    # If CUDA is enabled, tag source files to be compiled with nvcc.
-    if (MFEM_USE_CUDA)
-      set_property(SOURCE ${SRC_FILE} PROPERTY LANGUAGE CUDA)
-    endif()
-
    get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)

    string(REPLACE ".cpp" "" EXE_NAME "${EXE_PREFIX}${SRC_FILENAME}")
@@ -124,13 +119,6 @@ function(add_mfem_miniapp MFEM_EXE_NAME)
    endif()
  endforeach()

-  # If CUDA is enabled, tag source files to be compiled with nvcc.
-  if (MFEM_USE_CUDA)
-    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
-      PROPERTY LANGUAGE CUDA)
-    list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
-  endif()
-
  # Actually add the executable
  add_executable(${MFEM_EXE_NAME} ${MAIN_LIST}
    ${EXTRA_SOURCES_LIST} ${EXTRA_HEADERS_LIST})
@@ -727,13 +715,12 @@ function(mfem_export_mk_files)

  # Convert Boolean vars to YES/NO without writting the values to cache
  set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
-      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
+      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_GZSTREAM MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GECKO MFEM_USE_GNUTLS
      MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
-      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
-      MFEM_USE_UMPIRE)
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -827,7 +814,7 @@ function(mfem_export_mk_files)
  foreach(lib ${TPL_LIBRARIES})
    get_filename_component(suffix ${lib} EXT)
    # handle interfaces (e.g., SCOREC::apf)
-    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
+    if ("${lib}" MATCHES "SCOREC::.*")
    elseif (NOT "${lib}" MATCHES "SCOREC::.*" AND "${lib}" MATCHES ".*::.*")
      message(FATAL_ERROR "***** interface lib found ... exiting *****")
      # handle static and shared libs
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.


 // Support out-of-source builds: if MFEM_CONFIG_FILE is defined, include it.
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #ifndef MFEM_CONFIG_HEADER
 #define MFEM_CONFIG_HEADER
@@ -49,8 +49,8 @@
 // Throw an exception on errors.
 // #define MFEM_USE_EXCEPTIONS

-// Enable zlib in MFEM.
-// #define MFEM_USE_ZLIB
+// Enable gzstream in MFEM.
+// #define MFEM_USE_GZSTREAM

 // Enable backtraces for mfem_error through libunwind.
 // #define MFEM_USE_LIBUNWIND
@@ -100,9 +100,6 @@
 // Enable functionality based on the Gecko library
 // #define MFEM_USE_GECKO

-// Enable MFEM features based on the Ginkgo library
-// #define MFEM_USE_GINKGO
-
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

@@ -124,12 +121,6 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI

-// Enable MFEM functionality based on the HIOP library.
-// #define MFEM_USE_HIOP
-
-// Enable MFEM functionality based on the GSLIB library
-// #define MFEM_USE_GSLIB
-
 // Build the NVIDIA GPU/CUDA-enabled version of the MFEM library.
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA
@@ -144,12 +135,6 @@
 // Enable functionality based on the OCCA library.
 // #define MFEM_USE_OCCA

-// Enable functionality based on the libCEED library.
-// #define MFEM_USE_CEED
-
-// Enable functionality based on the Umpire library.
-// #define MFEM_USE_UMPIRE
-
 // Version of HYPRE used for building MFEM.
 // #define MFEM_HYPRE_VERSION @MFEM_HYPRE_VERSION@

@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Variables corresponding to defines in config.hpp (YES, NO, or value)
 MFEM_VERSION           = @MFEM_VERSION@
@@ -20,7 +20,7 @@ MFEM_USE_METIS         = @MFEM_USE_METIS@
 MFEM_USE_METIS_5       = @MFEM_USE_METIS_5@
 MFEM_DEBUG             = @MFEM_DEBUG@
 MFEM_USE_EXCEPTIONS    = @MFEM_USE_EXCEPTIONS@
-MFEM_USE_ZLIB          = @MFEM_USE_ZLIB@
+MFEM_USE_GZSTREAM      = @MFEM_USE_GZSTREAM@
 MFEM_USE_LIBUNWIND     = @MFEM_USE_LIBUNWIND@
 MFEM_USE_LAPACK        = @MFEM_USE_LAPACK@
 MFEM_THREAD_SAFE       = @MFEM_THREAD_SAFE@
@@ -34,7 +34,6 @@ MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GECKO         = @MFEM_USE_GECKO@
-MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
@@ -42,14 +41,10 @@ MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
 MFEM_USE_PUMI          = @MFEM_USE_PUMI@
-MFEM_USE_HIOP          = @MFEM_USE_HIOP@
-MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
-MFEM_USE_CEED          = @MFEM_USE_CEED@
-MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # See the file INSTALL for description of the configuration options.

@@ -22,7 +22,7 @@ endif()
 option(MFEM_USE_MPI "Enable MPI parallel build" OFF)
 option(MFEM_USE_METIS "Enable METIS usage" ${MFEM_USE_MPI})
 option(MFEM_USE_EXCEPTIONS "Enable the use of exceptions" OFF)
-option(MFEM_USE_ZLIB "Enable zlib for compressed data streams." OFF)
+option(MFEM_USE_GZSTREAM "Enable gzstream for compressed data streams." OFF)
 option(MFEM_USE_LIBUNWIND "Enable backtrace for errors." OFF)
 option(MFEM_USE_LAPACK "Enable LAPACK usage" OFF)
 option(MFEM_THREAD_SAFE "Enable thread safety" OFF)
@@ -35,7 +35,6 @@ option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GECKO "Enable GECKO usage" OFF)
-option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
@@ -43,12 +42,9 @@ option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
 option(MFEM_USE_PUMI "Enable PUMI" OFF)
-option(MFEM_USE_HIOP "Enable HiOp" OFF)
 option(MFEM_USE_CUDA "Enable CUDA" OFF)
 option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
-option(MFEM_USE_CEED "Enable CEED" OFF)
-option(MFEM_USE_UMPIRE "Enable Umpire" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -140,8 +136,6 @@ set(ScaLAPACK_TARGET_NAMES scalapack)

 set(GECKO_DIR "${MFEM_DIR}/../gecko" CACHE PATH "Path to the Gecko library.")

-set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
@@ -166,15 +160,8 @@ set(Axom_REQUIRED_PACKAGES "Conduit/relay/blueprint" CACHE STRING
 set(PUMI_DIR "${MFEM_DIR}/../pumi-2.1.0" CACHE STRING
    "Directory where PUMI is installed")

-set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
-    "Directory where HiOp is installed")
-set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
-    "Packages that HiOp depends on.")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
-set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")

 set(BLAS_INCLUDE_DIRS "" CACHE STRING "Path to BLAS headers.")
 set(BLAS_LIBRARIES "" CACHE STRING "The BLAS library.")
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # This file describes the default MFEM build options.
 #
@@ -108,7 +108,7 @@ MFEM_USE_METIS         = $(MFEM_USE_MPI)
 MFEM_USE_METIS_5       = NO
 MFEM_DEBUG             = NO
 MFEM_USE_EXCEPTIONS    = NO
-MFEM_USE_ZLIB          = NO
+MFEM_USE_GZSTREAM      = NO
 MFEM_USE_LIBUNWIND     = NO
 MFEM_USE_LAPACK        = NO
 MFEM_THREAD_SAFE       = NO
@@ -122,7 +122,6 @@ MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GECKO         = NO
-MFEM_USE_GINKGO        = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
@@ -130,14 +129,10 @@ MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
 MFEM_USE_PUMI          = NO
-MFEM_USE_HIOP          = NO
-MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
 MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
-MFEM_USE_CEED          = NO
-MFEM_USE_UMPIRE        = NO

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -249,11 +244,6 @@ GECKO_DIR = @MFEM_DIR@/../gecko
 GECKO_OPT = -I$(GECKO_DIR)/inc
 GECKO_LIB = -L$(GECKO_DIR)/lib -lgecko

-# Ginkgo library configuration (currently not needed)
-GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
-GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference
-
 # GnuTLS library configuration
 GNUTLS_OPT =
 GNUTLS_LIB = -lgnutls
@@ -318,16 +308,6 @@ PUMI_OPT = -I$(PUMI_DIR)/include
 PUMI_LIB = -L$(PUMI_DIR)/lib -lpumi -lcrv -lma -lmds -lapf -lpcu -lgmi -lparma\
   -llion -lmth -lapf_zoltan -lspr

-# HIOP
-HIOP_DIR = @MFEM_DIR@/../hiop/install
-HIOP_OPT = -I$(HIOP_DIR)/include
-HIOP_LIB = -L$(HIOP_DIR)/lib -lhiop $(LAPACK_LIB)
-
-# GSLIB library
-GSLIB_DIR = @MFEM_DIR@/../gslib/build
-GSLIB_OPT = -I$(GSLIB_DIR)/include
-GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs
-
 # CUDA library configuration (currently not needed)
 CUDA_OPT =
 CUDA_LIB =
@@ -341,11 +321,6 @@ OCCA_DIR = @MFEM_DIR@/../occa
 OCCA_OPT = -I$(OCCA_DIR)/include
 OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca

-# libCEED library configuration
-CEED_DIR ?= @MFEM_DIR@/../libCEED
-CEED_OPT = -I$(CEED_DIR)/include
-CEED_LIB = $(XLINKER)-rpath,$(CEED_DIR)/lib -L$(CEED_DIR)/lib -lceed
-
 # RAJA library configuration
 RAJA_DIR = @MFEM_DIR@/../raja
 RAJA_OPT = -I$(RAJA_DIR)/include
@@ -354,11 +329,6 @@ ifdef CUB_DIR
 endif
 RAJA_LIB = $(XLINKER)-rpath,$(RAJA_DIR)/lib -L$(RAJA_DIR)/lib -lRAJA

-# UMPIRE library configuration
-UMPIRE_DIR = @MFEM_DIR@/../umpire
-UMPIRE_OPT = -I$(UMPIRE_DIR)/include
-UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire
-
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #include "HYPRE_config.h"
 #include <cstdio>
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 SRC = $(if $(MFEM_DIR),$(MFEM_DIR)/config/,)

@@ -1,15 +1,15 @@
 #!/bin/bash

-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 make="${MAKE:-make}"
 mpiexec="${MPIEXEC:-mpirun}"
@@ -152,15 +152,6 @@ function extract_sample_runs()
   fi
   if [ "$device_runs" == "yes" ]; then
      runs=`printf "%s" "$runs" | grep ".* -d .*"`
-      if [ "$have_occa" == "no" ]; then
-         runs=`printf "%s" "$runs" | grep -v ".* -d occa-.*"`
-      fi
-      if [ "$have_raja" == "no" ]; then
-         runs=`printf "%s" "$runs" | grep -v ".* -d raja-.*"`
-      fi
-      if [ "$have_ceed" == "no" ]; then
-         runs=`printf "%s" "$runs" | grep -v ".* -d ceed-.*"`
-      fi
   else
      runs=`printf "%s" "$runs" | grep -v ".* -d .*"`
   fi
@@ -285,8 +276,7 @@ case "$1" in
      ;;
   -dev)
       device_runs="yes"
-       mfem_config+=" MFEM_USE_CUDA=YES MFEM_USE_OPENMP=YES"
-       # OCCA, RAJA, libCEED are enabled below, if available
+       mfem_config+=" MFEM_USE_CUDA=YES MFEM_USE_OCCA=YES MFEM_USE_RAJA=YES MFEM_USE_OPENMP=YES"
      ;;
   -v)
      valgrind="yes"
@@ -467,30 +457,6 @@ fi

 TIMEFORMAT="${base_timeformat}"

-# Setup optional libraries when not using externally built MFEM:
-if [ "${built}" == "no" ]; then
-   have_occa="no"
-   have_raja="no"
-   have_ceed="no"
-   if [ "${device_runs}" == "yes" ]; then
-      if [ -n "${CUDA_ARCH}" ]; then
-         mfem_config+=" CUDA_ARCH=${CUDA_ARCH}"
-      fi
-      if [ -d "${mfem_dir}/../occa" ]; then
-         mfem_config+=" MFEM_USE_OCCA=YES"
-         have_occa="yes"
-      fi
-      if [ -d "${mfem_dir}/../raja" ]; then
-         mfem_config+=" MFEM_USE_RAJA=YES"
-         have_raja="yes"
-      fi
-      if [ -d "${mfem_dir}/../libCEED" ]; then
-         mfem_config+=" MFEM_USE_CEED=YES"
-         have_ceed="yes"
-      fi
-   fi
-fi
-
 function set_echo_log()
 {
   local dirname=`dirname "$1"`
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #ifndef MFEM_TEMPLATE_CONFIG
 #define MFEM_TEMPLATE_CONFIG
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Utilities for the "make test" and "make check" targets.

@@ -82,10 +82,8 @@ test-par-NO:  $(SEQ_$(MFEM_TESTS):=-test-seq)
 test-ser:     test-par-NO
 test-par:     test-par-YES
 test:         all test-par-$(MFEM_USE_MPI) clean-exec
-test-noclean: all test-par-$(MFEM_USE_MPI)
 test-clean: ; @rm -f *.stderr
-test-print: \
- mfem-test=printf "   $(3) [$(2) ./$(1) $(if $(5),,-no-vis )$(if $(4),$(4) )]\n"
+test-print: mfem-test=printf "   $(3) [$(2) ./$(1) -no-vis $(if $(4),$(4) )]\n"
 test-print: mfem-test-file=printf "   $(3) [$(2) ./$(1) -no-vis ]\n"
 test-print: test-par-$(MFEM_USE_MPI)
 ifeq ($(MAKECMDGOALS),test-print)
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 find_package(Doxygen)

@@ -768,16 +768,13 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
                         @MFEM_SOURCE_DIR@/miniapps/common \
-                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
-                         @MFEM_SOURCE_DIR@/miniapps/gslib \
                         @MFEM_SOURCE_DIR@/miniapps/meshing \
-                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
-                         @MFEM_SOURCE_DIR@/miniapps/performance \
                         @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/nurbs \
+                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
+                         @MFEM_SOURCE_DIR@/miniapps/performance

 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -81,9 +81,6 @@ namespace mfem {
 * - <a class="el" href="ex20p_8cpp_source.html">Example 20p</a>: parallel symplectic ODE integration
 * - <a class="el" href="ex21_8cpp_source.html">Example 21</a>: adaptive mesh refinement for linear elasticity
 * - <a class="el" href="ex21p_8cpp_source.html">Example 21p</a>: parallel adaptive mesh refinement for linear elasticity
- * - <a class="el" href="ex22_8cpp_source.html">Example 22</a>: complex-valued linear systems for damped harmonic oscillators
- * - <a class="el" href="ex22p_8cpp_source.html">Example 22p</a>: parallel complex-valued linear systems for damped harmonic oscillators
- * - <a class="el" href="ex23_8cpp_source.html">Example 23</a>: 2nd order in time the wave equation
 *
 * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
@@ -118,12 +115,6 @@ namespace mfem {
 *   <a class="el" href="pumi_2ex6p_8cpp_source.html">6p</a>
 *   demonstrating the use of MFEM's \link pumi.hpp PUMI classes\endlink
 *
- * <H4>HiOp Examples</H4>
- * - Variants of Examples
- *   <a class="el" href="hiop_2ex9_8cpp_source.html">9</a> and
- *   <a class="el" href="hiop_2ex9p_8cpp_source.html">9p</a>,
- *   demonstrating the use of MFEM's \link hiop.hpp HiOp classes\endlink
- *
 * <H3>Miniapps</H3>
 * - <a class="el" href="volta_8cpp_source.html">Volta</a>: simple electrostatics simulation code
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
@@ -137,12 +128,9 @@ namespace mfem {
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
 * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
- * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
 * - <a class="el" href="load-dc_8cpp_source.html">Load DC</a>: visualize fields saved via DataCollection classes
- * - <a class="el" href="convert-dc_8cpp_source.html">Convert DC</a>: convert between different DataCollection formats
+ * - <a class="el" href="convert-dc_8cpp_source.html">Convert DC</a>: convert between diffirent DataCollection formats
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
- * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 MFEM_DIR ?= ..
 DOXYGEN_CONF = CodeDocumentation.conf
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 list(APPEND ALL_EXE_SRCS
  ex1.cpp
@@ -28,9 +28,6 @@ list(APPEND ALL_EXE_SRCS
  ex19.cpp
  ex20.cpp
  ex21.cpp
-  ex22.cpp
-  ex23.cpp
-  ex24.cpp
  )

 if (MFEM_USE_MPI)
@@ -56,8 +53,6 @@ if (MFEM_USE_MPI)
    ex19p.cpp
    ex20p.cpp
    ex21p.cpp
-    ex22p.cpp
-    ex24p.cpp
    )
 endif()

@@ -105,11 +100,6 @@ if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
 endif()

-# Include the examples/sundials directory if SUNDIALS is enabled.
-if (MFEM_USE_GINKGO)
-  add_subdirectory(ginkgo)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -119,8 +109,3 @@ endif()
 if (MFEM_USE_PUMI)
  add_subdirectory(pumi)
 endif()
-
-if (MFEM_USE_HIOP)
-  add_subdirectory(hiop)
-endif()
-
@@ -31,10 +31,7 @@
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -197,17 +194,9 @@ int main(int argc, char *argv[])
      umf_solver.Mult(B, X);
 #endif
   }
-   else // Jacobi preconditioning in partial assembly mode
+   else // No preconditioning for now in partial assembly mode.
   {
-      if (UsesTensorBasis(*fespace))
-      {
-         OperatorJacobiSmoother M(*a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
+      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
   }

   // 12. Recover the solution as a finite element grid function.
@@ -179,9 +179,7 @@ int main(int argc, char *argv[])
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
                  "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4."
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            13 - RK3 SSP, 14 - RK4.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -193,9 +193,7 @@ int main(int argc, char *argv[])
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Backward Euler, 2 - SDIRK2, 3 - SDIRK3,\n\t"
                  "            11 - Forward Euler, 12 - RK2,\n\t"
-                  "            13 - RK3 SSP, 14 - RK4."
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            13 - RK3 SSP, 14 - RK4.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -8,7 +8,7 @@
 //    mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
 //    mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
-//    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3877 -o 2 -sys
+//    mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3876 -o 2 -sys
 //    mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
 //    mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh
 //    mpirun -np 4 ex12p -m ../data/beam-hex-nurbs.mesh
@@ -57,7 +57,7 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/beam-tri.mesh";
   int order = 1;
   int nev = 5;
-   int seed = 66;
+   int seed = 75;
   bool visualization = 1;
   bool amg_elast = 0;

@@ -3,7 +3,7 @@
 // Compile with: make ex13p
 //
 // Sample runs:  mpirun -np 4 ex13p -m ../data/star.mesh
-//               mpirun -np 4 ex13p -m ../data/square-disc.mesh -o 2 -n 4
+//               mpirun -np 4 ex13p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex13p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex13p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex13p -m ../data/escher.mesh
@@ -16,6 +16,9 @@
 //               ex15 -m ../data/ball-nurbs.mesh -tf 0.3
 //               ex15 -m ../data/mobius-strip.mesh
 //               ex15 -m ../data/amr-quad.mesh
+//
+//               Conforming meshes (no derefinement):
+//
 //               ex15 -m ../data/square-disc.mesh
 //               ex15 -m ../data/escher.mesh -r 2 -tf 0.3
 //
@@ -127,7 +130,7 @@ int main(int argc, char *argv[])
      if (ref_levels > 0) { ref_levels--; }
      mesh.SetCurvature(2);
   }
-   mesh.EnsureNCMesh(true);
+   mesh.EnsureNCMesh();
   for (int l = 0; l < ref_levels; l++)
   {
      mesh.UniformRefinement();
@@ -16,6 +16,9 @@
 //               mpirun -np 4 ex15p -m ../data/ball-nurbs.mesh -tf 0.5
 //               mpirun -np 4 ex15p -m ../data/mobius-strip.mesh
 //               mpirun -np 4 ex15p -m ../data/amr-quad.mesh
+//
+//               Conforming meshes (no load balancing and derefinement):
+//
 //               mpirun -np 4 ex15p -m ../data/square-disc.mesh
 //               mpirun -np 4 ex15p -m ../data/escher.mesh -r 2 -tf 0.3
 //
@@ -143,7 +146,7 @@ int main(int argc, char *argv[])
      if (ref_levels > 0) { ref_levels--; }
      mesh->SetCurvature(2);
   }
-   mesh->EnsureNCMesh(true);
+   mesh->EnsureNCMesh();
   for (int l = 0; l < ref_levels; l++)
   {
      mesh->UniformRefinement();
@@ -29,9 +29,6 @@
 //               mpirun -np 4 ex1p -pa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//               mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -211,19 +208,9 @@ int main(int argc, char *argv[])

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
+   //     * With partial assembly, use no preconditioner, for now.
   Solver *prec = NULL;
-   if (pa)
-   {
-      if (UsesTensorBasis(*fespace))
-      {
-         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
-      }
-   }
-   else
-   {
-      prec = new HypreBoomerAMG;
-   }
+   if (!pa) { prec = new HypreBoomerAMG; }
   CGSolver cg(MPI_COMM_WORLD);
   cg.SetRelTol(1e-12);
   cg.SetMaxIter(2000);
@@ -1,579 +0,0 @@
-//                               MFEM Example 22
-//
-// Compile with: make ex22
-//
-// Sample runs:  ex22 -m ../data/inline-segment.mesh -o 3
-//               ex22 -m ../data/inline-tri.mesh -o 3
-//               ex22 -m ../data/inline-quad.mesh -o 3
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
-//               ex22 -m ../data/inline-tet.mesh -o 2
-//               ex22 -m ../data/inline-hex.mesh -o 2
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
-//
-// Description:  This example code demonstrates the use of MFEM to define and
-//               solve simple complex-valued linear systems. It implements three
-//               variants of a damped harmonic oscillator:
-//
-//               1) A scalar H1 field
-//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
-//
-//               2) A vector H(Curl) field
-//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
-//
-//               3) A vector H(Div) field
-//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
-//
-//               In each case the field is driven by a forced oscillation, with
-//               angular frequency omega, imposed at the boundary or a portion
-//               of the boundary.
-//
-//               In electromagnetics, the coefficients are typically named the
-//               permeability, mu = 1/a, permittivity, epsilon = b, and
-//               conductivity, sigma = c. The user can specify these constants
-//               using either set of names.
-//
-//               The example also demonstrates how to display a time-varying
-//               solution as a sequence of fields sent to a single GLVis socket.
-//
-//               We recommend viewing examples 1, 3 and 4 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 20.0;
-static double omega_ = 10.0;
-
-double u0_real_exact(const Vector &);
-double u0_imag_exact(const Vector &);
-
-void u1_real_exact(const Vector &, Vector &);
-void u1_imag_exact(const Vector &, Vector &);
-
-void u2_real_exact(const Vector &, Vector &);
-void u2_imag_exact(const Vector &, Vector &);
-
-bool check_for_inline_mesh(const char * mesh_file);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/inline-quad.mesh";
-   int ref_levels = 0;
-   int order = 1;
-   int prob = 0;
-   double freq = -1.0;
-   double a_coef = 0.0;
-   bool visualization = 1;
-   bool herm_conv = true;
-   bool exact_sol = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H_1, 1: H(Curl), or 2: H(Div) "
-                  "damped harmonic oscillator.");
-   args.AddOption(&a_coef, "-a", "--stiffness-coef",
-                  "Stiffness coefficient (spring constant or 1/mu).");
-   args.AddOption(&epsilon_, "-b", "--mass-coef",
-                  "Mass coefficient (or epsilon).");
-   args.AddOption(&sigma_, "-c", "--damping-coef",
-                  "Damping coefficient (or sigma).");
-   args.AddOption(&mu_, "-mu", "--permeability",
-                  "Permeability of free space (or 1/(spring constant)).");
-   args.AddOption(&epsilon_, "-eps", "--permittivity",
-                  "Permittivity of free space (or mass constant).");
-   args.AddOption(&sigma_, "-sigma", "--conductivity",
-                  "Conductivity (or damping constant).");
-   args.AddOption(&freq, "-f", "--frequency",
-                  "Frequency (in Hz).");
-   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
-                  "--no-hermitian", "Use convention for Hermitian operators.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   MFEM_VERIFY(prob >= 0 && prob <=2,
-               "Unrecognized problem type: " << prob);
-
-   if ( a_coef != 0.0 )
-   {
-      mu_ = 1.0 / a_coef;
-   }
-   if ( freq > 0.0 )
-   {
-      omega_ = 2.0 * M_PI * freq;
-   }
-
-   exact_sol = check_for_inline_mesh(mesh_file);
-   if (exact_sol)
-   {
-      cout << "Identified a mesh with known exact solution" << endl;
-   }
-
-   ComplexOperator::Convention conv =
-      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
-   //    with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Refine the mesh to increase resolution. In this example we do
-   //    'ref_levels' of uniform refinement where the user specifies
-   //    the number of levels with the '-r' option.
-   for (int l = 0; l < ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 4. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
-   //    order.
-   if (dim == 1 && prob != 0 )
-   {
-      cout << "Switching to problem type 0, H1 basis functions, "
-           << "for 1 dimensional mesh." << endl;
-      prob = 0;
-   }
-
-   FiniteElementCollection *fec = NULL;
-   switch (prob)
-   {
-      case 0:  fec = new H1_FECollection(order, dim);      break;
-      case 1:  fec = new ND_FECollection(order, dim);      break;
-      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
-      default: break; // This should be unreachable
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
-        << endl;
-
-   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined based on the type
-   //    of mesh and the problem type.
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system.
-   ComplexLinearForm b(fespace, conv);
-   b.Vector::operator=(0.0);
-
-   // 7. Define the solution vector u as a complex finite element grid function
-   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
-   //    the exact solution if it is known.
-   ComplexGridFunction u(fespace);
-   ComplexGridFunction * u_exact = NULL;
-   if (exact_sol) { u_exact = new ComplexGridFunction(fespace); }
-
-   FunctionCoefficient u0_r(u0_real_exact);
-   FunctionCoefficient u0_i(u0_imag_exact);
-   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
-   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
-   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
-   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
-
-   ConstantCoefficient zeroCoef(0.0);
-   ConstantCoefficient oneCoef(1.0);
-
-   Vector zeroVec(dim); zeroVec = 0.0;
-   Vector  oneVec(dim);  oneVec = 0.0; oneVec[(prob==2)?(dim-1):0] = 1.0;
-   VectorConstantCoefficient zeroVecCoef(zeroVec);
-   VectorConstantCoefficient oneVecCoef(oneVec);
-
-   u = 0.0;
-   switch (prob)
-   {
-      case 0:
-         if (exact_sol)
-         {
-            u.ProjectBdrCoefficient(u0_r, u0_i, ess_bdr);
-            u_exact->ProjectCoefficient(u0_r, u0_i);
-         }
-         else
-         {
-            u.ProjectBdrCoefficient(oneCoef, zeroCoef, ess_bdr);
-         }
-         break;
-      case 1:
-         if (exact_sol)
-         {
-            u.ProjectBdrCoefficientTangent(u1_r, u1_i, ess_bdr);
-            u_exact->ProjectCoefficient(u1_r, u1_i);
-         }
-         else
-         {
-            u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
-         }
-         break;
-      case 2:
-         if (exact_sol)
-         {
-            u.ProjectBdrCoefficientNormal(u2_r, u2_i, ess_bdr);
-            u_exact->ProjectCoefficient(u2_r, u2_i);
-         }
-         else
-         {
-            u.ProjectBdrCoefficientNormal(oneVecCoef, zeroVecCoef, ess_bdr);
-         }
-         break;
-      default: break; // This should be unreachable
-   }
-
-   if (visualization && exact_sol)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *mesh << u_exact->real()
-                 << "window_title 'Exact: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *mesh << u_exact->imag()
-                 << "window_title 'Exact: Imaginary Part'" << flush;
-   }
-
-   // 8. Set up the sesquilinear form a(.,.) on the finite element space
-   //    corresponding to the damped harmonic oscillator operator of the
-   //    appropriate type:
-   //
-   //    0) A scalar H1 field
-   //       -Div(a Grad) - omega^2 b + i omega c
-   //
-   //    1) A vector H(Curl) field
-   //       Curl(a Curl) - omega^2 b + i omega c
-   //
-   //    2) A vector H(Div) field
-   //       -Grad(a Div) - omega^2 b + i omega c
-   //
-   ConstantCoefficient stiffnessCoef(1.0/mu_);
-   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
-   ConstantCoefficient lossCoef(omega_ * sigma_);
-   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
-
-   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
-   switch (prob)
-   {
-      case 0:
-         a->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new MassIntegrator(massCoef),
-                                new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         a->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
-
-   // 8a. Set up the bilinear form for the preconditioner corresponding to the
-   //     appropriate operator
-   //
-   //      0) A scalar H1 field
-   //         -Div(a Grad) - omega^2 b + omega c
-   //
-   //      1) A vector H(Curl) field
-   //         Curl(a Curl) + omega^2 b + omega c
-   //
-   //      2) A vector H(Div) field
-   //         -Grad(a Div) - omega^2 b + omega c
-   //
-   BilinearForm *pcOp = new BilinearForm(fespace);
-   switch (prob)
-   {
-      case 0:
-         pcOp->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         pcOp->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         pcOp->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
-
-   // 9. Assemble the form and the corresponding linear system, applying any
-   //    necessary transformations such as: assembly, eliminating boundary
-   //    conditions, conforming constraints for non-conforming AMR, etc.
-   a->Assemble();
-   pcOp->Assemble();
-
-   OperatorHandle A;
-   Vector B, U;
-
-   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
-   u = 0.0;
-   U = 0.0;
-
-   OperatorHandle PCOp;
-   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-
-   {
-      ComplexSparseMatrix * Asp =
-         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
-
-      cout << "Size of linear system: "
-           << 2 * Asp->real().Width() << endl << endl;
-   }
-
-   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
-   //     preconditioner based on the appropriate sparse smoother.
-   {
-      Array<int> blockOffsets;
-      blockOffsets.SetSize(3);
-      blockOffsets[0] = 0;
-      blockOffsets[1] = PCOp.Ptr()->Height();
-      blockOffsets[2] = PCOp.Ptr()->Height();
-      blockOffsets.PartialSum();
-
-      BlockDiagonalPreconditioner BDP(blockOffsets);
-
-      Operator * pc_r = NULL;
-      Operator * pc_i = NULL;
-
-      double s = 1.0;
-      switch (prob)
-      {
-         case 0:
-            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-            break;
-         case 1:
-            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-            s = -1.0;
-            break;
-         case 2:
-            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-            break;
-
-         default: break; // This should be unreachable
-      }
-      pc_i = new ScaledOperator(pc_r,
-                                (conv == ComplexOperator::HERMITIAN) ?
-                                s:-s);
-
-      BDP.SetDiagonalBlock(0, pc_r);
-      BDP.SetDiagonalBlock(1, pc_i);
-      BDP.owns_blocks = 1;
-
-      GMRESSolver gmres;
-      gmres.SetPreconditioner(BDP);
-      gmres.SetOperator(*A.Ptr());
-      gmres.SetRelTol(1e-12);
-      gmres.SetMaxIter(1000);
-      gmres.SetPrintLevel(1);
-      gmres.Mult(B, U);
-   }
-
-   // 11. Recover the solution as a finite element grid function and compute the
-   //     errors if the exact solution is known.
-   a->RecoverFEMSolution(U, b, u);
-
-   if (exact_sol)
-   {
-      double err_r = -1.0;
-      double err_i = -1.0;
-
-      switch (prob)
-      {
-         case 0:
-            err_r = u.real().ComputeL2Error(u0_r);
-            err_i = u.imag().ComputeL2Error(u0_i);
-            break;
-         case 1:
-            err_r = u.real().ComputeL2Error(u1_r);
-            err_i = u.imag().ComputeL2Error(u1_i);
-            break;
-         case 2:
-            err_r = u.real().ComputeL2Error(u2_r);
-            err_i = u.imag().ComputeL2Error(u2_i);
-            break;
-         default: break; // This should be unreachable
-      }
-
-      cout << endl;
-      cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
-      cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
-      cout << endl;
-   }
-
-   // 12. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m mesh -g sol".
-   {
-      ofstream mesh_ofs("refined.mesh");
-      mesh_ofs.precision(8);
-      mesh->Print(mesh_ofs);
-
-      ofstream sol_r_ofs("sol_r.gf");
-      ofstream sol_i_ofs("sol_i.gf");
-      sol_r_ofs.precision(8);
-      sol_i_ofs.precision(8);
-      u.real().Save(sol_r_ofs);
-      u.imag().Save(sol_i_ofs);
-   }
-
-   // 13. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *mesh << u.real()
-                 << "window_title 'Solution: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *mesh << u.imag()
-                 << "window_title 'Solution: Imaginary Part'" << flush;
-   }
-   if (visualization && exact_sol)
-   {
-      *u_exact -= u;
-
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *mesh << u_exact->real()
-                 << "window_title 'Error: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *mesh << u_exact->imag()
-                 << "window_title 'Error: Imaginary Part'" << flush;
-   }
-   if (visualization)
-   {
-      GridFunction u_t(fespace);
-      u_t = u.real();
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << u_t
-               << "window_title 'Harmonic Solution (t = 0.0 T)'"
-               << "pause\n" << flush;
-
-      cout << "GLVis visualization paused."
-           << " Press space (in the GLVis window) to resume it.\n";
-      int num_frames = 32;
-      int i = 0;
-      while (sol_sock)
-      {
-         double t = (double)(i % num_frames) / num_frames;
-         ostringstream oss;
-         oss << "Harmonic Solution (t = " << t << " T)";
-
-         add(cos( 2.0 * M_PI * t), u.real(),
-             sin(-2.0 * M_PI * t), u.imag(), u_t);
-         sol_sock << "solution\n" << *mesh << u_t
-                  << "window_title '" << oss.str() << "'" << flush;
-         i++;
-      }
-   }
-
-   // 14. Free the used memory.
-   delete a;
-   delete u_exact;
-   delete pcOp;
-   delete fespace;
-   delete fec;
-   delete mesh;
-
-   return 0;
-}
-
-bool check_for_inline_mesh(const char * mesh_file)
-{
-   string file(mesh_file);
-   size_t p0 = file.find_last_of("/");
-   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
-   return s0 == "inline-";
-}
-
-complex<double> u0_exact(const Vector &x)
-{
-   int dim = x.Size();
-   complex<double> i(0.0, 1.0);
-   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
-   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
-   return std::exp(-i * kappa * x[dim - 1]);
-}
-
-double u0_real_exact(const Vector &x)
-{
-   return u0_exact(x).real();
-}
-
-double u0_imag_exact(const Vector &x)
-{
-   return u0_exact(x).imag();
-}
-
-void u1_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
-}
-
-void u1_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
-}
-
-void u2_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
-}
-
-void u2_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
-}
@@ -1,634 +0,0 @@
-//                       MFEM Example 22 - Parallel Version
-//
-// Compile with: make ex22p
-//
-// Sample runs:  mpirun -np 4 ex22p -m ../data/inline-segment.mesh -o 3
-//               mpirun -np 4 ex22p -m ../data/inline-tri.mesh -o 3
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
-//
-// Description:  This example code demonstrates the use of MFEM to define and
-//               solve simple complex-valued linear systems. It implements three
-//               variants of a damped harmonic oscillator:
-//
-//               1) A scalar H1 field
-//                  -Div(a Grad u) - omega^2 b u + i omega c u = 0
-//
-//               2) A vector H(Curl) field
-//                  Curl(a Curl u) - omega^2 b u + i omega c u = 0
-//
-//               3) A vector H(Div) field
-//                  -Grad(a Div u) - omega^2 b u + i omega c u = 0
-//
-//               In each case the field is driven by a forced oscillation, with
-//               angular frequency omega, imposed at the boundary or a portion
-//               of the boundary.
-//
-//               In electromagnetics the coefficients are typically named the
-//               permeability, mu = 1/a, permittivity, epsilon = b, and
-//               conductivity, sigma = c. The user can specify these constants
-//               using either set of names.
-//
-//               The example also demonstrates how to display a time-varying
-//               solution as a sequence of fields sent to a single GLVis socket.
-//
-//               We recommend viewing examples 1, 3 and 4 before viewing this
-//               example.
-
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-static double mu_ = 1.0;
-static double epsilon_ = 1.0;
-static double sigma_ = 20.0;
-static double omega_ = 10.0;
-
-double u0_real_exact(const Vector &);
-double u0_imag_exact(const Vector &);
-
-void u1_real_exact(const Vector &, Vector &);
-void u1_imag_exact(const Vector &, Vector &);
-
-void u2_real_exact(const Vector &, Vector &);
-void u2_imag_exact(const Vector &, Vector &);
-
-bool check_for_inline_mesh(const char * mesh_file);
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/inline-quad.mesh";
-   int ser_ref_levels = 1;
-   int par_ref_levels = 1;
-   int order = 1;
-   int prob = 0;
-   double freq = -1.0;
-   double a_coef = 0.0;
-   bool visualization = 1;
-   bool herm_conv = true;
-   bool exact_sol = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: H_1, 1: H(Curl), or 2: H(Div) "
-                  "damped harmonic oscillator.");
-   args.AddOption(&a_coef, "-a", "--stiffness-coef",
-                  "Stiffness coefficient (spring constant or 1/mu).");
-   args.AddOption(&epsilon_, "-b", "--mass-coef",
-                  "Mass coefficient (or epsilon).");
-   args.AddOption(&sigma_, "-c", "--damping-coef",
-                  "Damping coefficient (or sigma).");
-   args.AddOption(&mu_, "-mu", "--permeability",
-                  "Permeability of free space (or 1/(spring constant)).");
-   args.AddOption(&epsilon_, "-eps", "--permittivity",
-                  "Permittivity of free space (or mass constant).");
-   args.AddOption(&sigma_, "-sigma", "--conductivity",
-                  "Conductivity (or damping constant).");
-   args.AddOption(&freq, "-f", "--frequency",
-                  "Frequency (in Hz).");
-   args.AddOption(&herm_conv, "-herm", "--hermitian", "-no-herm",
-                  "--no-hermitian", "Use convention for Hermitian operators.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   MFEM_VERIFY(prob >= 0 && prob <=2,
-               "Unrecognized problem type: " << prob);
-
-   if ( a_coef != 0.0 )
-   {
-      mu_ = 1.0 / a_coef;
-   }
-   if ( freq > 0.0 )
-   {
-      omega_ = 2.0 * M_PI * freq;
-   }
-
-   exact_sol = check_for_inline_mesh(mesh_file);
-   if (myid == 0 && exact_sol)
-   {
-      cout << "Identified a mesh with known exact solution" << endl;
-   }
-
-   ComplexOperator::Convention conv =
-      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution.
-   for (int l = 0; l < ser_ref_levels; l++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int l = 0; l < par_ref_levels; l++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
-   //    the specified order.
-   if (dim == 1 && prob != 0 )
-   {
-      if (myid == 0)
-      {
-         cout << "Switching to problem type 0, H1 basis functions, "
-              << "for 1 dimensional mesh." << endl;
-      }
-      prob = 0;
-   }
-
-   FiniteElementCollection *fec = NULL;
-   switch (prob)
-   {
-      case 0:  fec = new H1_FECollection(order, dim);      break;
-      case 1:  fec = new ND_FECollection(order, dim);      break;
-      case 2:  fec = new RT_FECollection(order - 1, dim);  break;
-      default: break; // This should be unreachable
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    based on the type of mesh and the problem type.
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system.
-   ParComplexLinearForm b(fespace, conv);
-   b.Vector::operator=(0.0);
-
-   // 9. Define the solution vector u as a parallel complex finite element grid
-   //    function corresponding to fespace. Initialize u with initial guess of
-   //    1+0i or the exact solution if it is known.
-   ParComplexGridFunction u(fespace);
-   ParComplexGridFunction * u_exact = NULL;
-   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
-
-   FunctionCoefficient u0_r(u0_real_exact);
-   FunctionCoefficient u0_i(u0_imag_exact);
-   VectorFunctionCoefficient u1_r(dim, u1_real_exact);
-   VectorFunctionCoefficient u1_i(dim, u1_imag_exact);
-   VectorFunctionCoefficient u2_r(dim, u2_real_exact);
-   VectorFunctionCoefficient u2_i(dim, u2_imag_exact);
-
-   ConstantCoefficient zeroCoef(0.0);
-   ConstantCoefficient oneCoef(1.0);
-
-   Vector zeroVec(dim); zeroVec = 0.0;
-   Vector  oneVec(dim);  oneVec = 0.0; oneVec[(prob==2)?(dim-1):0] = 1.0;
-   VectorConstantCoefficient zeroVecCoef(zeroVec);
-   VectorConstantCoefficient oneVecCoef(oneVec);
-
-   u = 0.0;
-   switch (prob)
-   {
-      case 0:
-         if (exact_sol)
-         {
-            u.ProjectBdrCoefficient(u0_r, u0_i, ess_bdr);
-            u_exact->ProjectCoefficient(u0_r, u0_i);
-         }
-         else
-         {
-            u.ProjectBdrCoefficient(oneCoef, zeroCoef, ess_bdr);
-         }
-         break;
-      case 1:
-         if (exact_sol)
-         {
-            u.ProjectBdrCoefficientTangent(u1_r, u1_i, ess_bdr);
-            u_exact->ProjectCoefficient(u1_r, u1_i);
-         }
-         else
-         {
-            u.ProjectBdrCoefficientTangent(oneVecCoef, zeroVecCoef, ess_bdr);
-         }
-         break;
-      case 2:
-         if (exact_sol)
-         {
-            u.ProjectBdrCoefficientNormal(u2_r, u2_i, ess_bdr);
-            u_exact->ProjectCoefficient(u2_r, u2_i);
-         }
-         else
-         {
-            u.ProjectBdrCoefficientNormal(oneVecCoef, zeroVecCoef, ess_bdr);
-         }
-         break;
-      default: break; // This should be unreachable
-   }
-
-   if (visualization && exact_sol)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
-                 << "window_title 'Exact: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
-                 << "window_title 'Exact: Imaginary Part'" << flush;
-   }
-
-   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
-   //     space corresponding to the damped harmonic oscillator operator of the
-   //     appropriate type:
-   //
-   //     0) A scalar H1 field
-   //        -Div(a Grad) - omega^2 b + i omega c
-   //
-   //     1) A vector H(Curl) field
-   //        Curl(a Curl) - omega^2 b + i omega c
-   //
-   //     2) A vector H(Div) field
-   //        -Grad(a Div) - omega^2 b + i omega c
-   //
-   ConstantCoefficient stiffnessCoef(1.0/mu_);
-   ConstantCoefficient massCoef(-omega_ * omega_ * epsilon_);
-   ConstantCoefficient lossCoef(omega_ * sigma_);
-   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);
-
-   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
-   switch (prob)
-   {
-      case 0:
-         a->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new MassIntegrator(massCoef),
-                                new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         a->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         a->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef),
-                                NULL);
-         a->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef),
-                                new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
-
-   // 10a. Set up the parallel bilinear form for the preconditioner
-   //      corresponding to the appropriate operator
-   //
-   //      0) A scalar H1 field
-   //         -Div(a Grad) - omega^2 b + omega c
-   //
-   //      1) A vector H(Curl) field
-   //         Curl(a Curl) + omega^2 b + omega c
-   //
-   //      2) A vector H(Div) field
-   //         -Grad(a Div) - omega^2 b + omega c
-   //
-   ParBilinearForm *pcOp = new ParBilinearForm(fespace);
-   switch (prob)
-   {
-      case 0:
-         pcOp->AddDomainIntegrator(new DiffusionIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new MassIntegrator(lossCoef));
-         break;
-      case 1:
-         pcOp->AddDomainIntegrator(new CurlCurlIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(negMassCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      case 2:
-         pcOp->AddDomainIntegrator(new DivDivIntegrator(stiffnessCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(massCoef));
-         pcOp->AddDomainIntegrator(new VectorFEMassIntegrator(lossCoef));
-         break;
-      default: break; // This should be unreachable
-   }
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, etc.
-   a->Assemble();
-   pcOp->Assemble();
-
-   OperatorHandle A;
-   Vector B, U;
-
-   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
-   u = 0.0;
-   U = 0.0;
-
-   OperatorHandle PCOp;
-   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-
-   if (myid == 0)
-   {
-      ComplexHypreParMatrix * Ahyp =
-         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
-
-      cout << "Size of linear system: "
-           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
-   }
-
-   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
-   //     diagonal preconditioner based on the appropriate multigrid
-   //     preconditioner from hypre.
-   {
-      Array<HYPRE_Int> blockTrueOffsets;
-      blockTrueOffsets.SetSize(3);
-      blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = PCOp.Ptr()->Height();
-      blockTrueOffsets[2] = PCOp.Ptr()->Height();
-      blockTrueOffsets.PartialSum();
-
-      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
-
-      Operator * pc_r = NULL;
-      Operator * pc_i = NULL;
-
-      switch (prob)
-      {
-         case 0:
-            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-            break;
-         case 1:
-            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-            break;
-         case 2:
-            if (dim == 2 )
-            {
-               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-            }
-            else
-            {
-               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-            }
-            break;
-         default: break; // This should be unreachable
-      }
-      pc_i = new ScaledOperator(pc_r,
-                                (conv == ComplexOperator::HERMITIAN) ?
-                                -1.0:1.0);
-
-      BDP.SetDiagonalBlock(0, pc_r);
-      BDP.SetDiagonalBlock(1, pc_i);
-      BDP.owns_blocks = 1;
-
-      FGMRESSolver fgmres(MPI_COMM_WORLD);
-      fgmres.SetPreconditioner(BDP);
-      fgmres.SetOperator(*A.Ptr());
-      fgmres.SetRelTol(1e-12);
-      fgmres.SetMaxIter(1000);
-      fgmres.SetPrintLevel(1);
-      fgmres.Mult(B, U);
-   }
-   // 13. Recover the parallel grid function corresponding to U. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(U, b, u);
-
-   if (exact_sol)
-   {
-      double err_r = -1.0;
-      double err_i = -1.0;
-
-      switch (prob)
-      {
-         case 0:
-            err_r = u.real().ComputeL2Error(u0_r);
-            err_i = u.imag().ComputeL2Error(u0_i);
-            break;
-         case 1:
-            err_r = u.real().ComputeL2Error(u1_r);
-            err_i = u.imag().ComputeL2Error(u1_i);
-            break;
-         case 2:
-            err_r = u.real().ComputeL2Error(u2_r);
-            err_i = u.imag().ComputeL2Error(u2_i);
-            break;
-         default: break; // This should be unreachable
-      }
-
-      if ( myid == 0 )
-      {
-         cout << endl;
-         cout << "|| Re (u_h - u) ||_{L^2} = " << err_r << endl;
-         cout << "|| Im (u_h - u) ||_{L^2} = " << err_i << endl;
-         cout << endl;
-      }
-   }
-
-   // 14. Save the refined mesh and the solution in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_r_name, sol_i_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_r_name << "sol_r." << setfill('0') << setw(6) << myid;
-      sol_i_name << "sol_i." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_r_ofs(sol_r_name.str().c_str());
-      ofstream sol_i_ofs(sol_i_name.str().c_str());
-      sol_r_ofs.precision(8);
-      sol_i_ofs.precision(8);
-      u.real().Save(sol_r_ofs);
-      u.imag().Save(sol_i_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *pmesh << u.real()
-                 << "window_title 'Solution: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *pmesh << u.imag()
-                 << "window_title 'Solution: Imaginary Part'" << flush;
-   }
-   if (visualization && exact_sol)
-   {
-      *u_exact -= u;
-
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock_r(vishost, visport);
-      socketstream sol_sock_i(vishost, visport);
-      sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock_r.precision(8);
-      sol_sock_i.precision(8);
-      sol_sock_r << "solution\n" << *pmesh << u_exact->real()
-                 << "window_title 'Error: Real Part'" << flush;
-      sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
-                 << "window_title 'Error: Imaginary Part'" << flush;
-   }
-   if (visualization)
-   {
-      ParGridFunction u_t(fespace);
-      u_t = u.real();
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << u_t
-               << "window_title 'Harmonic Solution (t = 0.0 T)'"
-               << "pause\n" << flush;
-      if (myid == 0)
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      int num_frames = 32;
-      int i = 0;
-      while (sol_sock)
-      {
-         double t = (double)(i % num_frames) / num_frames;
-         ostringstream oss;
-         oss << "Harmonic Solution (t = " << t << " T)";
-
-         add(cos( 2.0 * M_PI * t), u.real(),
-             sin(-2.0 * M_PI * t), u.imag(), u_t);
-         sol_sock << "parallel " << num_procs << " " << myid << "\n";
-         sol_sock << "solution\n" << *pmesh << u_t
-                  << "window_title '" << oss.str() << "'" << flush;
-         i++;
-      }
-   }
-
-   // 16. Free the used memory.
-   delete a;
-   delete u_exact;
-   delete pcOp;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-bool check_for_inline_mesh(const char * mesh_file)
-{
-   string file(mesh_file);
-   size_t p0 = file.find_last_of("/");
-   string s0 = file.substr((p0==string::npos)?0:(p0+1),7);
-   return s0 == "inline-";
-}
-
-complex<double> u0_exact(const Vector &x)
-{
-   int dim = x.Size();
-   complex<double> i(0.0, 1.0);
-   complex<double> alpha = (epsilon_ * omega_ - i * sigma_);
-   complex<double> kappa = std::sqrt(mu_ * omega_* alpha);
-   return std::exp(-i * kappa * x[dim - 1]);
-}
-
-double u0_real_exact(const Vector &x)
-{
-   return u0_exact(x).real();
-}
-
-double u0_imag_exact(const Vector &x)
-{
-   return u0_exact(x).imag();
-}
-
-void u1_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_real_exact(x);
-}
-
-void u1_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[0] = u0_imag_exact(x);
-}
-
-void u2_real_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_real_exact(x);
-}
-
-void u2_imag_exact(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-   v.SetSize(dim); v = 0.0; v[dim-1] = u0_imag_exact(x);
-}
@@ -1,419 +0,0 @@
-//                                MFEM Example 23
-//
-// Compile with: make ex23
-//
-// Sample runs:  ex23
-//               ex23 -m ../data/inline-tri.mesh
-//               ex23 -m ../data/disc-nurbs.mesh -r 3 -o 2 -tf 2
-//
-// Description:  This example solves the wave equation problem of the form:
-//
-//                               d^2u/dt^2 = c^2 \Delta u.
-//
-//               The example demonstrates the use of 2nd order time integration.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-/** After spatial discretization, the conduction model can be written as:
- *
- *     d^2u/dt^2 = M^{-1}(-Ku)
- *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
- *
- *  Class WaveOperator represents the right-hand side of the above ODE.
- */
-class WaveOperator : public SecondOrderTimeDependentOperator
-{
-protected:
-   FiniteElementSpace &fespace;
-   Array<int> ess_tdof_list; // this list remains empty for pure Neumann b.c.
-
-   BilinearForm *M;
-   BilinearForm *K;
-
-   SparseMatrix Mmat, Kmat, Kmat0;
-   SparseMatrix *T; // T = M + dt K
-   double current_dt;
-
-   CGSolver M_solver; // Krylov solver for inverting the mass matrix M
-   DSmoother M_prec;  // Preconditioner for the mass matrix M
-
-   CGSolver T_solver; // Implicit solver for T = M + fac0*K
-   DSmoother T_prec;  // Preconditioner for the implicit solver
-
-   Coefficient *c2;
-   mutable Vector z; // auxiliary vector
-
-public:
-   WaveOperator(FiniteElementSpace &f, Array<int> &ess_bdr,double speed);
-
-   virtual void Mult(const Vector &u, const Vector &du_dt,
-                     Vector &d2udt2) const;
-   /** Solve the Backward-Euler equation:
-       d2udt2 = f(u + fac0*d2udt2,dudt + fac1*d2udt2, t), for the unknown d2udt2.*/
-   virtual void ImplicitSolve(const double fac0, const double fac1,
-                              const Vector &u, const Vector &dudt, Vector &d2udt2);
-
-   ///
-   void SetParameters(const Vector &u);
-
-   virtual ~WaveOperator();
-};
-
-
-WaveOperator::WaveOperator(FiniteElementSpace &f,
-                           Array<int> &ess_bdr, double speed)
-   : SecondOrderTimeDependentOperator(f.GetTrueVSize(), 0.0), fespace(f), M(NULL),
-     K(NULL),
-     T(NULL), current_dt(0.0), z(height)
-{
-   const double rel_tol = 1e-8;
-
-   fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-
-   c2 = new ConstantCoefficient(speed*speed);
-
-   K = new BilinearForm(&fespace);
-   K->AddDomainIntegrator(new DiffusionIntegrator(*c2));
-   K->Assemble();
-
-   Array<int> dummy;
-   K->FormSystemMatrix(dummy, Kmat0);
-   K->FormSystemMatrix(ess_tdof_list, Kmat);
-
-   M = new BilinearForm(&fespace);
-   M->AddDomainIntegrator(new MassIntegrator());
-   M->Assemble();
-   M->FormSystemMatrix(ess_tdof_list, Mmat);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(rel_tol);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(30);
-   M_solver.SetPrintLevel(0);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(Mmat);
-
-   T_solver.iterative_mode = false;
-   T_solver.SetRelTol(rel_tol);
-   T_solver.SetAbsTol(0.0);
-   T_solver.SetMaxIter(100);
-   T_solver.SetPrintLevel(0);
-   T_solver.SetPreconditioner(T_prec);
-
-   T = NULL;
-}
-
-void WaveOperator::Mult(const Vector &u, const Vector &du_dt,
-                        Vector &d2udt2)  const
-{
-   // Compute:
-   //    d2udt2 = M^{-1}*-K(u)
-   // for d2udt2
-   Kmat.Mult(u, z);
-   z.Neg(); // z = -z
-   M_solver.Mult(z, d2udt2);
-}
-
-void WaveOperator::ImplicitSolve(const double fac0, const double fac1,
-                                 const Vector &u, const Vector &dudt, Vector &d2udt2)
-{
-   // Solve the equation:
-   //    d2udt2 = M^{-1}*[-K(u + fac0*d2udt2)]
-   // for d2udt2
-   if (!T)
-   {
-      T = Add(1.0, Mmat, fac0, Kmat);
-      T_solver.SetOperator(*T);
-   }
-   Kmat0.Mult(u, z);
-   z.Neg();
-
-   for (int i = 0; i < ess_tdof_list.Size(); i++)
-   {
-      z[ess_tdof_list[i]] = 0.0;
-   }
-   T_solver.Mult(z, d2udt2);
-}
-
-void WaveOperator::SetParameters(const Vector &u)
-{
-   delete T;
-   T = NULL; // re-compute T on the next ImplicitSolve
-}
-
-WaveOperator::~WaveOperator()
-{
-   delete T;
-   delete M;
-   delete K;
-   delete c2;
-}
-
-double InitialSolution(const Vector &x)
-{
-   if (x.Norml2() < 0.5)
-   {
-      return 1.0;
-   }
-   else
-   {
-      return 0.0;
-   }
-}
-
-double InitialRate(const Vector &x)
-{
-   return 0.0;
-}
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   const char *ref_dir  = "";
-   int ref_levels = 2;
-   int order = 1;
-   int ode_solver_type = 10;
-   double t_final = 0.5;
-   double dt = 1.0e-2;
-   double speed = 1.0;
-   bool visualization = true;
-   bool visit = true;
-   bool dirichlet = true;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: [0--10] - GeneralizedAlpha(0.1 * s),\n\t"
-                  "\t   11 - Average Acceleration, 12 - Linear Acceleration\n"
-                  "\t   13 - CentralDifference, 14 - FoxGoodwin");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&speed, "-c", "--speed",
-                  "Wave speed.");
-   args.AddOption(&dirichlet, "-dir", "--dirichlet", "-neu",
-                  "--neumann",
-                  "BC switch.");
-
-   args.AddOption(&ref_dir, "-r", "--ref",
-                  "Reference directory for checking final solution.");
-
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral and hexahedral meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Define the ODE solver used for time integration. Several implicit
-   //    singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as
-   //    explicit Runge-Kutta methods are available.
-   SecondOrderODESolver *ode_solver;
-   switch (ode_solver_type)
-   {
-      // Implicit methods
-      case 0: ode_solver = new GeneralizedAlpha2Solver(0.0); break;
-      case 1: ode_solver = new GeneralizedAlpha2Solver(0.1); break;
-      case 2: ode_solver = new GeneralizedAlpha2Solver(0.2); break;
-      case 3: ode_solver = new GeneralizedAlpha2Solver(0.3); break;
-      case 4: ode_solver = new GeneralizedAlpha2Solver(0.4); break;
-      case 5: ode_solver = new GeneralizedAlpha2Solver(0.5); break;
-      case 6: ode_solver = new GeneralizedAlpha2Solver(0.6); break;
-      case 7: ode_solver = new GeneralizedAlpha2Solver(0.7); break;
-      case 8: ode_solver = new GeneralizedAlpha2Solver(0.8); break;
-      case 9: ode_solver = new GeneralizedAlpha2Solver(0.9); break;
-      case 10: ode_solver = new GeneralizedAlpha2Solver(1.0); break;
-
-      case 11: ode_solver = new AverageAccelerationSolver(); break;
-      case 12: ode_solver = new LinearAccelerationSolver(); break;
-      case 13: ode_solver = new CentralDifferenceSolver(); break;
-      case 14: ode_solver = new FoxGoodwinSolver(); break;
-
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define the vector finite element space representing the current and the
-   //    initial temperature, u_ref.
-   H1_FECollection fe_coll(order, dim);
-   FiniteElementSpace fespace(mesh, &fe_coll);
-
-   int fe_size = fespace.GetTrueVSize();
-   cout << "Number of temperature unknowns: " << fe_size << endl;
-
-   GridFunction u_gf(&fespace);
-   GridFunction dudt_gf(&fespace);
-   // 6. Set the initial conditions for u. All boundaries are considered
-   //    natural.
-   FunctionCoefficient u_0(InitialSolution);
-   u_gf.ProjectCoefficient(u_0);
-   Vector u;
-   u_gf.GetTrueDofs(u);
-
-   FunctionCoefficient dudt_0(InitialRate);
-   dudt_gf.ProjectCoefficient(dudt_0);
-   Vector dudt;
-   dudt_gf.GetTrueDofs(dudt);
-
-   // 7. Initialize the conduction operator and the visualization.
-   Array<int> ess_bdr;
-   if (mesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh->bdr_attributes.Max());
-
-      if (dirichlet)
-      {
-         ess_bdr = 1;
-      }
-      else
-      {
-         ess_bdr = 0;
-      }
-   }
-
-   WaveOperator oper(fespace, ess_bdr, speed);
-
-   u_gf.SetFromTrueDofs(u);
-   {
-      ofstream omesh("ex23.mesh");
-      omesh.precision(precision);
-      mesh->Print(omesh);
-      ofstream osol("ex23-init.gf");
-      osol.precision(precision);
-      u_gf.Save(osol);
-      dudt_gf.Save(osol);
-   }
-
-   VisItDataCollection visit_dc("Example23", mesh);
-   visit_dc.RegisterField("solution", &u_gf);
-   visit_dc.RegisterField("rate", &dudt_gf);
-   if (visit)
-   {
-      visit_dc.SetCycle(0);
-      visit_dc.SetTime(0.0);
-      visit_dc.Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *mesh << dudt_gf;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   // 8. Perform time-integration (looping over the time iterations, ti, with a
-   //    time-step dt).
-   ode_solver->Init(oper);
-   double t = 0.0;
-
-   bool last_step = false;
-   for (int ti = 1; !last_step; ti++)
-   {
-
-      if (t + dt >= t_final - dt/2)
-      {
-         last_step = true;
-      }
-
-      ode_solver->Step(u, dudt, t, dt);
-
-      if (last_step || (ti % vis_steps) == 0)
-      {
-         cout << "step " << ti << ", t = " << t << endl;
-
-         u_gf.SetFromTrueDofs(u);
-         dudt_gf.SetFromTrueDofs(dudt);
-         if (visualization)
-         {
-            sout << "solution\n" << *mesh << u_gf << flush;
-         }
-
-         if (visit)
-         {
-            visit_dc.SetCycle(ti);
-            visit_dc.SetTime(t);
-            visit_dc.Save();
-         }
-      }
-      oper.SetParameters(u);
-   }
-
-   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m ex23.mesh -g ex23-final.gf".
-   {
-      ofstream osol("ex23-final.gf");
-      osol.precision(precision);
-      u_gf.Save(osol);
-      dudt_gf.Save(osol);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete mesh;
-
-   return 0;
-}
-
-
@@ -1,285 +0,0 @@
-//                               MFEM Example 24
-//
-// Compile with: make ex24
-//
-// Sample runs:  ex24 -m ../data/star.mesh
-//               ex24 -m ../data/square-disc.mesh -o 2
-//               ex24 -m ../data/beam-tet.mesh
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa
-//               ex24 -m ../data/escher.mesh
-//               ex24 -m ../data/escher.mesh -o 2
-//               ex24 -m ../data/fichera.mesh
-//               ex24 -m ../data/fichera-q2.vtk
-//               ex24 -m ../data/fichera-q3.mesh
-//               ex24 -m ../data/square-disc-nurbs.mesh
-//               ex24 -m ../data/beam-hex-nurbs.mesh
-//               ex24 -m ../data/amr-quad.mesh -o 2
-//               ex24 -m ../data/amr-hex.mesh
-//
-// Device sample runs:
-//               ex24 -m ../data/star.mesh -pa -d cuda
-//               ex24 -m ../data/star.mesh -pa -d raja-cuda
-//               ex24 -m ../data/star.mesh -pa -d raja-omp
-//               ex24 -m ../data/beam-hex.mesh -pa -d cuda
-//
-// Description:  This example code illustrates usage of mixed finite element
-//               spaces. Using two different approaches, we project a gradient
-//               of a function in H^1 to H(curl). Other spaces and example
-//               computations are to be added in the future.
-//
-//               We recommend viewing examples 1 and 3 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-double p_exact(const Vector &x);
-void gradp_exact(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/beam-hex.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels = (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-   mesh->ReorientTetMesh();
-
-   // 5. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   FiniteElementSpace *H1fespace = new FiniteElementSpace(mesh, H1fec);
-
-   int size = fespace->GetTrueVSize();
-   int H1size = H1fespace->GetTrueVSize();
-   cout << "Number of Nedelec finite element unknowns: " << size << endl;
-   cout << "Number of H1 finite element unknowns: " << H1size << endl;
-
-   // 6. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   GridFunction x(fespace);
-   FunctionCoefficient p_coef(p_exact);
-   GridFunction p(H1fespace);
-   p.ProjectCoefficient(p_coef);
-   p.SetTrueVector();
-   p.SetFromTrueVector();
-
-   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-
-   // 7. Set up the bilinear forms.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(1.0);
-   BilinearForm *a = new BilinearForm(fespace);
-   MixedBilinearForm *a_NDH1 = new MixedBilinearForm(H1fespace, fespace);
-   if (pa)
-   {
-      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a_NDH1->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-
-   // First approach: L2 projection
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-   a_NDH1->AddDomainIntegrator(new MixedVectorGradientIntegrator(*muinv));
-
-   // 8. Assemble the parallel bilinear form and the corresponding linear
-   //    system, applying any necessary transformations such as: parallel
-   //    assembly, eliminating boundary conditions, applying conforming
-   //    constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-
-   a->Assemble();
-   if (!pa) { a->Finalize(); }
-
-   a_NDH1->Assemble();
-   if (!pa) { a_NDH1->Finalize(); }
-
-   if (pa)
-   {
-      a_NDH1->Mult(p, x);
-   }
-   else
-   {
-      SparseMatrix& NDH1 = a_NDH1->SpMat();
-      NDH1.Mult(p, x);
-   }
-
-   // 9. Define and apply a PCG solver for Ax = b with Jacobi preconditioner.
-   {
-      GridFunction rhs(fespace);
-      rhs = x;
-      x = 0.0;
-
-      CGSolver cg;
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(1000);
-      cg.SetPrintLevel(1);
-      if (pa)
-      {
-         Array<int> ess_tdof_list; // empty
-         OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
-
-         cg.SetOperator(*a);
-         cg.SetPreconditioner(Jacobi);
-         cg.Mult(rhs, x);
-      }
-      else
-      {
-         SparseMatrix& Amat = a->SpMat();
-         DSmoother Jacobi(Amat);
-
-         cg.SetOperator(Amat);
-         cg.SetPreconditioner(Jacobi);
-         cg.Mult(rhs, x);
-      }
-   }
-
-   // 10. Compute the same solution by applying GradientInterpolator in H(curl).
-   DiscreteLinearOperator grad(H1fespace, fespace);
-   grad.AddDomainInterpolator(new GradientInterpolator());
-   grad.Assemble();
-
-   GridFunction gradp(fespace);
-   grad.Mult(p, gradp);
-
-   // 11. Compute the projection of the exact grad p.
-   GridFunction exact_gradp(fespace);
-   exact_gradp.ProjectCoefficient(gradp_coef);
-   exact_gradp.SetTrueVector();
-   exact_gradp.SetFromTrueVector();
-
-   // 12. Compute and print the L^2 norm of the error.
-   {
-      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = gradp.ComputeL2Error(gradp_coef);
-      double errProj = exact_gradp.ComputeL2Error(gradp_coef);
-
-      cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
-           "|| E_h - grad p ||_{L^2} = " << errSol << '\n' << endl;
-      cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-           "||_{L^2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-           "||_{L^2} = " << errProj << '\n' << endl;
-   }
-
-   // 13. Save the refined mesh and the solution. This output can be viewed
-   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   delete a;
-   delete a_NDH1;
-   delete sigma;
-   delete muinv;
-   delete fespace;
-   delete H1fespace;
-   delete fec;
-   delete H1fec;
-   delete mesh;
-
-   return 0;
-}
-
-double p_exact(const Vector &x)
-{
-   if (dim == 3)
-   {
-      return sin(x(0)) * sin(x(1)) * sin(x(2));
-   }
-   else if (dim == 2)
-   {
-      return sin(x(0)) * sin(x(1));
-   }
-
-   return 0.0;
-}
-
-void gradp_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = cos(x(0)) * sin(x(1)) * sin(x(2));
-      f(1) = sin(x(0)) * cos(x(1)) * sin(x(2));
-      f(2) = sin(x(0)) * sin(x(1)) * cos(x(2));
-   }
-   else
-   {
-      f(0) = cos(x(0)) * sin(x(1));
-      f(1) = sin(x(0)) * cos(x(1));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -1,347 +0,0 @@
-//                       MFEM Example 24 - Parallel Version
-//
-// Compile with: make ex24p
-//
-// Sample runs:  mpirun -np 4 ex24p -m ../data/star.mesh
-//               mpirun -np 4 ex24p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex24p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa
-//               mpirun -np 4 ex24p -m ../data/escher.mesh
-//               mpirun -np 4 ex24p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex24p -m ../data/fichera.mesh
-//               mpirun -np 4 ex24p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex24p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex24p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex24p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex24p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex24p -m ../data/amr-hex.mesh
-//
-// Device sample runs:
-//               mpirun -np 4 ex24p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex24p -m ../data/star.mesh -pa -d raja-cuda
-//               mpirun -np 4 ex24p -m ../data/star.mesh -pa -d raja-omp
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -pa -d cuda
-//
-// Description:  This example code illustrates usage of mixed finite element
-//               spaces. Using two different approaches, we project a gradient
-//               of a function in H^1 to H(curl). Other spaces and example
-//               computations are to be added in the future.
-//
-//               We recommend viewing examples 1 and 3 before viewing this
-//               example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-double p_exact(const Vector &x);
-void gradp_exact(const Vector &, Vector &);
-
-int dim;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-hex.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels = (int)floor(log(1000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 1;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   ParFiniteElementSpace *H1fespace = new ParFiniteElementSpace(pmesh, H1fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   HYPRE_Int H1size = H1fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << size << endl;
-      cout << "Number of H1 finite element unknowns: " << H1size << endl;
-   }
-
-   // 8. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   FunctionCoefficient p_coef(p_exact);
-   ParGridFunction p(H1fespace);
-   p.ProjectCoefficient(p_coef);
-   p.SetTrueVector();
-   p.SetFromTrueVector();
-
-   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-
-   // 9. Set up the parallel bilinear forms.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(1.0);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   ParMixedBilinearForm *a_NDH1 = new ParMixedBilinearForm(H1fespace, fespace);
-   if (pa)
-   {
-      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a_NDH1->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-
-   // First approach: L2 projection
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-   a_NDH1->AddDomainIntegrator(new MixedVectorGradientIntegrator(*muinv));
-
-   // 10. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-
-   a->Assemble();
-   if (!pa) { a->Finalize(); }
-
-   a_NDH1->Assemble();
-   if (!pa) { a_NDH1->Finalize(); }
-
-   Vector B(fespace->GetTrueVSize());
-   Vector X(fespace->GetTrueVSize());
-
-   if (pa)
-   {
-      ParLinearForm *b = new ParLinearForm(fespace); // used as a vector
-      a_NDH1->Mult(p, *b); // process-local multiplication
-      b->ParallelAssemble(B);
-      delete b;
-   }
-   else
-   {
-      HypreParMatrix *NDH1 = a_NDH1->ParallelAssemble();
-
-      Vector P(H1fespace->GetTrueVSize());
-      p.GetTrueDofs(P);
-
-      NDH1->Mult(P,B);
-
-      delete NDH1;
-   }
-
-   // 11. Define and apply a parallel PCG solver for AX=B with Jacobi
-   //     preconditioner.
-   if (pa)
-   {
-      Array<int> ess_tdof_list; // empty
-
-      OperatorPtr A;
-      a->FormSystemMatrix(ess_tdof_list, A);
-
-      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(1000);
-      cg.SetPrintLevel(1);
-      cg.SetOperator(*A);
-      cg.SetPreconditioner(Jacobi);
-      X = 0.0;
-      cg.Mult(B, X);
-   }
-   else
-   {
-      HypreParMatrix *Amat = a->ParallelAssemble();
-      HypreDiagScale Jacobi(*Amat);
-      HyprePCG pcg(*Amat);
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(1000);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(Jacobi);
-      X = 0.0;
-      pcg.Mult(B, X);
-
-      delete Amat;
-   }
-
-   x.SetFromTrueDofs(X);
-
-   // 12. Compute the same solution by applying GradientInterpolator in H(curl).
-   ParDiscreteLinearOperator grad(H1fespace, fespace);
-   grad.AddDomainInterpolator(new GradientInterpolator());
-   grad.Assemble();
-
-   ParGridFunction gradp(fespace);
-   grad.Mult(p, gradp);
-
-   // 13. Compute the projection of the exact grad p.
-   ParGridFunction exact_gradp(fespace);
-   exact_gradp.ProjectCoefficient(gradp_coef);
-   exact_gradp.SetTrueVector();
-   exact_gradp.SetFromTrueVector();
-
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = gradp.ComputeL2Error(gradp_coef);
-      double errProj = exact_gradp.ComputeL2Error(gradp_coef);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in "
-              "H(curl): || E_h - grad p ||_{L^2} = " << errSol << '\n' << endl;
-         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - "
-              "grad p ||_{L^2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-              "||_{L^2} = " << errProj << '\n' << endl;
-      }
-   }
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete a;
-   delete a_NDH1;
-   delete sigma;
-   delete muinv;
-   delete fespace;
-   delete H1fespace;
-   delete fec;
-   delete H1fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-double p_exact(const Vector &x)
-{
-   if (dim == 3)
-   {
-      return sin(x(0)) * sin(x(1)) * sin(x(2));
-   }
-   else if (dim == 2)
-   {
-      return sin(x(0)) * sin(x(1));
-   }
-
-   return 0.0;
-}
-
-void gradp_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = cos(x(0)) * sin(x(1)) * sin(x(2));
-      f(1) = sin(x(0)) * cos(x(1)) * sin(x(2));
-      f(2) = sin(x(0)) * sin(x(1)) * cos(x(2));
-   }
-   else
-   {
-      f(0) = cos(x(0)) * sin(x(1));
-      f(1) = sin(x(0)) * cos(x(1));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -6,7 +6,6 @@
 //               ex3 -m ../data/beam-tri.mesh -o 2
 //               ex3 -m ../data/beam-tet.mesh
 //               ex3 -m ../data/beam-hex.mesh
-//               ex3 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex3 -m ../data/escher.mesh
 //               ex3 -m ../data/escher.mesh -o 2
 //               ex3 -m ../data/fichera.mesh
@@ -20,12 +19,6 @@
 //               ex3 -m ../data/mobius-strip.mesh -f 0.1
 //               ex3 -m ../data/klein-bottle.mesh -f 0.1
 //
-// Device sample runs:
-//               ex3 -m ../data/star.mesh -pa -d cuda
-//               ex3 -m ../data/star.mesh -pa -d raja-cuda
-//               ex3 -m ../data/star.mesh -pa -d raja-omp
-//               ex3 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic diffusion
 //               problem corresponding to the second order definite Maxwell
 //               equation curl curl E + E = f with boundary condition
@@ -60,8 +53,6 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/beam-tet.mesh";
   int order = 1;
   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -73,10 +64,6 @@ int main(int argc, char *argv[])
                  " solution.");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -89,19 +76,14 @@ int main(int argc, char *argv[])
   args.PrintOptions(cout);
   kappa = freq * M_PI;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 50,000
   //    elements.
@@ -115,14 +97,14 @@ int main(int argc, char *argv[])
   }
   mesh->ReorientTetMesh();

-   // 5. Define a finite element space on the mesh. Here we use the Nedelec
+   // 4. Define a finite element space on the mesh. Here we use the Nedelec
   //    finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
        << fespace->GetTrueVSize() << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
@@ -134,7 +116,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side
   //    of the FEM linear system, which in this case is (f,phi_i) where f is
   //    given by the function f_exact and phi_i are the basis functions in the
   //    finite element fespace.
@@ -143,7 +125,7 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
   b->Assemble();

-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x by projecting the exact
   //    solution. Note that only values from the boundary edges will be used
   //    when eliminating the non-homogeneous boundary condition to modify the
@@ -152,59 +134,48 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient E(sdim, E_exact);
   x.ProjectCoefficient(E);

-   // 9. Set up the bilinear form corresponding to the EM diffusion operator
+   // 8. Set up the bilinear form corresponding to the EM diffusion operator
   //    curl muinv curl + sigma I, by adding the curl-curl and the mass domain
   //    integrators.
   Coefficient *muinv = new ConstantCoefficient(1.0);
   Coefficient *sigma = new ConstantCoefficient(1.0);
   BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));

-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
+   // 9. Assemble the bilinear form and the corresponding linear system,
+   //    applying any necessary transformations such as: eliminating boundary
+   //    conditions, applying conforming constraints for non-conforming AMR,
+   //    static condensation, etc.
   if (static_cond) { a->EnableStaticCondensation(); }
   a->Assemble();

-   OperatorPtr A;
+   SparseMatrix A;
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A.Height() << endl;

-   // 11. Solve the linear system A X = B.
-   if (pa) // Jacobi preconditioning in partial assembly mode
-   {
-      OperatorJacobiSmoother M(*a, ess_tdof_list);
-      PCG(*A, M, B, X, 1, 1000, 1e-12, 0.0);
-   }
-   else
-   {
 #ifndef MFEM_USE_SUITESPARSE
-      // 11. Define a simple symmetric Gauss-Seidel preconditioner and use it to
-      //     solve the system Ax=b with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 500, 1e-12, 0.0);
+   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system Ax=b with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 500, 1e-12, 0.0);
 #else
-      // 11. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the
-      //     system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
+   // 10. If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
 #endif
-   }

-   // 12. Recover the solution as a finite element grid function.
+   // 11. Recover the solution as a finite element grid function.
   a->RecoverFEMSolution(X, *b, x);

-   // 13. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
   cout << "\n|| E_h - E ||_{L^2} = " << x.ComputeL2Error(E) << '\n' << endl;

-   // 14. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -215,7 +186,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 15. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -225,7 +196,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *mesh << x << flush;
   }

-   // 16. Free the used memory.
+   // 15. Free the used memory.
   delete a;
   delete sigma;
   delete muinv;
@@ -6,7 +6,6 @@
 //               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex3p -m ../data/escher.mesh
 //               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/fichera.mesh
@@ -20,12 +19,6 @@
 //               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
 //               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
 //
-// Device sample runs:
-//               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d raja-cuda
-//               mpirun -np 4 ex3p -m ../data/star.mesh -pa -d raja-omp
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple electromagnetic diffusion
 //               problem corresponding to the second order definite Maxwell
 //               equation curl curl E + E = f with boundary condition
@@ -66,8 +59,6 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/beam-tet.mesh";
   int order = 1;
   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -79,10 +70,6 @@ int main(int argc, char *argv[])
                  " solution.");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -103,19 +90,14 @@ int main(int argc, char *argv[])
   }
   kappa = freq * M_PI;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -127,7 +109,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
   //    meshes need to be reoriented before we can define high-order Nedelec
@@ -143,7 +125,7 @@ int main(int argc, char *argv[])
   }
   pmesh->ReorientTetMesh();

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Nedelec finite elements of the specified order.
   FiniteElementCollection *fec = new ND_FECollection(order, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -153,7 +135,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
@@ -165,7 +147,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
   //    basis functions in the finite element fespace.
@@ -174,76 +156,57 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
   b->Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x by projecting the exact
-   //     solution. Note that only values from the boundary edges will be used
-   //     when eliminating the non-homogeneous boundary condition to modify the
-   //     r.h.s. vector b.
+   // 9. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
   ParGridFunction x(fespace);
   VectorFunctionCoefficient E(sdim, E_exact);
   x.ProjectCoefficient(E);

-   // 11. Set up the parallel bilinear form corresponding to the EM diffusion
+   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
   //     mass domain integrators.
   Coefficient *muinv = new ConstantCoefficient(1.0);
   Coefficient *sigma = new ConstantCoefficient(1.0);
   ParBilinearForm *a = new ParBilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
   if (static_cond) { a->EnableStaticCondensation(); }
   a->Assemble();

-   OperatorPtr A;
+   HypreParMatrix A;
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
-   //     (in the full assembly case) or CG with Jacobi preconditioner (in the
-   //     partial assembly case).
-
-   if (pa) // Jacobi preconditioning in partial assembly mode
+   if (myid == 0)
   {
-      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(1000);
-      cg.SetPrintLevel(1);
-      cg.SetOperator(*A);
-      cg.SetPreconditioner(Jacobi);
-      cg.Mult(B, X);
-   }
-   else
-   {
-      if (myid == 0)
-      {
-         cout << "Size of linear system: "
-              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
-      }
-
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      HypreAMS ams(*A.As<HypreParMatrix>(), prec_fespace);
-      HyprePCG pcg(*A.As<HypreParMatrix>());
-      pcg.SetTol(1e-12);
-      pcg.SetMaxIter(500);
-      pcg.SetPrintLevel(2);
-      pcg.SetPreconditioner(ams);
-      pcg.Mult(B, X);
+      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
   }

-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 12. Define and apply a parallel PCG solver for AX=B with the AMS
+   //     preconditioner from hypre.
+   ParFiniteElementSpace *prec_fespace =
+      (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
+   HypreSolver *ams = new HypreAMS(A, prec_fespace);
+   HyprePCG *pcg = new HyprePCG(A);
+   pcg->SetTol(1e-12);
+   pcg->SetMaxIter(500);
+   pcg->SetPrintLevel(2);
+   pcg->SetPreconditioner(*ams);
+   pcg->Mult(B, X);
+
+   // 13. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 15. Compute and print the L^2 norm of the error.
+   // 14. Compute and print the L^2 norm of the error.
   {
      double err = x.ComputeL2Error(E);
      if (myid == 0)
@@ -252,7 +215,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 16. Save the refined mesh and the solution in parallel. This output can
+   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -268,7 +231,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -279,7 +242,9 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 18. Free the used memory.
+   // 17. Free the used memory.
+   delete pcg;
+   delete ams;
   delete a;
   delete sigma;
   delete muinv;
@@ -21,7 +21,7 @@
 //
 //               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in a
-//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
+//               VisIt (visit.llnl.gov) visualization format.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

@@ -264,19 +264,7 @@ int main(int argc, char *argv[])
   visit_dc.RegisterField("pressure", &p);
   visit_dc.Save();

-   // 14. Save data in the ParaView format
-   ParaViewDataCollection paraview_dc("Example5", mesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(order);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("velocity",&u);
-   paraview_dc.RegisterField("pressure",&p);
-   paraview_dc.Save();
-
-   // 15. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -289,7 +277,7 @@ int main(int argc, char *argv[])
      p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
   }

-   // 16. Free the used memory.
+   // 15. Free the used memory.
   delete fform;
   delete gform;
   delete invM;
@@ -21,7 +21,7 @@
 //
 //               The example demonstrates the use of the BlockMatrix class, as
 //               well as the collective saving of several grid functions in a
-//               VisIt (visit.llnl.gov) and ParaView (paraview.org) formats.
+//               VisIt (visit.llnl.gov) visualization format.
 //
 //               We recommend viewing examples 1-4 before viewing this example.

@@ -239,6 +239,7 @@ int main(int argc, char *argv[])

   // 12. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
+
   int maxIter(500);
   double rtol(1.e-6);
   double atol(1.e-10);
@@ -325,19 +326,7 @@ int main(int argc, char *argv[])
                      DataCollection::PARALLEL_FORMAT);
   visit_dc.Save();

-   // 16. Save data in the ParaView format
-   ParaViewDataCollection paraview_dc("Example5P", pmesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(order);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetTime(0.0);
-   paraview_dc.RegisterField("velocity",u);
-   paraview_dc.RegisterField("pressure",p);
-   paraview_dc.Save();
-
-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -357,7 +346,7 @@ int main(int argc, char *argv[])
             << endl;
   }

-   // 18. Free the used memory.
+   // 17. Free the used memory.
   delete fform;
   delete gform;
   delete u;
@@ -19,8 +19,6 @@
 //               ex6 -pa -d cuda
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
-//               ex6 -pa -d ceed-cpu
-//               ex6 -pa -d ceed-cuda
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
@@ -19,8 +19,6 @@
 //               mpirun -np 4 ex6p -pa -d cuda
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
-//               mpirun -np 4 ex6p -pa -d ceed-cpu
-//               mpirun -np 4 ex6p -pa -d ceed-cuda
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
 //               refinement loop. The problem being solved is again the Laplace
@@ -9,7 +9,6 @@
 //    ex9 -m ../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/periodic-hexagon.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.002 -tf 9
-//    ex9 -m ../data/amr-quad.mesh -p 1 -r 2 -dt 0.02 -s 13 -tf 9
 //    ex9 -m ../data/star-q3.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/star-mixed.mesh -p 1 -r 2 -dt 0.005 -tf 9
 //    ex9 -m ../data/disc-nurbs.mesh -p 1 -r 3 -dt 0.005 -tf 9
@@ -17,23 +16,17 @@
 //    ex9 -m ../data/periodic-square.mesh -p 3 -r 4 -dt 0.0025 -tf 9 -vs 20
 //    ex9 -m ../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8
 //
-// Device sample runs:
-//    ex9 -pa
-//    ex9 -pa -m ../data/periodic-cube.mesh
-//    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -59,54 +52,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &K, A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         A = K;
-         A *= -dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -116,21 +61,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   BilinearForm &M, &K;
+   SparseMatrix &M, &K;
   const Vector &b;
-   Solver *M_prec;
+   DSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };


@@ -141,14 +84,11 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/periodic-hexagon.mesh";
   int ref_levels = 2;
   int order = 3;
-   bool pa = false;
-   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
   double dt = 0.01;
   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;

@@ -164,17 +104,9 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
-                  "            11 - Backward Euler,\n\t"
-                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
@@ -185,9 +117,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -201,9 +130,6 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   Device device(device_config);
-   device.Print();
-
   // 2. Read the mesh from the given mesh file. We can handle geometrically
   //    periodic meshes in this code.
   Mesh mesh(mesh_file, 1, 1);
@@ -214,21 +140,11 @@ int main(int argc, char *argv[])
   ODESolver *ode_solver = NULL;
   switch (ode_solver_type)
   {
-      // Explicit methods
      case 1: ode_solver = new ForwardEulerSolver; break;
      case 2: ode_solver = new RK2Solver(1.0); break;
      case 3: ode_solver = new RK3SSPSolver; break;
      case 4: ode_solver = new RK4Solver; break;
      case 6: ode_solver = new RK6Solver; break;
-      // Implicit (L-stable) methods
-      case 11: ode_solver = new BackwardEulerSolver; break;
-      case 12: ode_solver = new SDIRK23Solver(2); break;
-      case 13: ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
-
      default:
         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
         return 3;
@@ -250,7 +166,7 @@ int main(int argc, char *argv[])

   // 5. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
   FiniteElementSpace fes(&mesh, &fec);

   cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -263,13 +179,8 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   if (pa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
+   BilinearForm k(&fes);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -281,11 +192,11 @@ int main(int argc, char *argv[])
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

   m.Assemble();
+   m.Finalize();
   int skip_zeros = 0;
   k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
   k.Finalize(skip_zeros);
+   b.Assemble();

   // 7. Define the initial conditions, save the corresponding grid function to
   //    a file and (optionally) save data in the VisIt format and initialize
@@ -326,20 +237,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9", &mesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", &u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
   socketstream sout;
   if (visualization)
   {
@@ -367,7 +264,7 @@ int main(int argc, char *argv[])
   // 8. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and perform time-integration (looping over the time
   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m, k, b);
+   FE_Evolution adv(m.SpMat(), k.SpMat(), b);

   double t = 0.0;
   adv.SetTime(t);
@@ -397,13 +294,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
      }
   }

@@ -417,7 +307,6 @@ int main(int argc, char *argv[])

   // 10. Free the used memory.
   delete ode_solver;
-   delete pd;
   delete dc;

   return 0;
@@ -425,24 +314,12 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
 {
-   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
-   Array<int> ess_tdof_list;
-   if (pa)
-   {
-      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-      M_solver.SetOperator(M);
-      dg_solver = NULL;
-   }
-   else
-   {
-      M_prec = new DSmoother(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-      M_solver.SetOperator(M.SpMat());
-   }
-   M_solver.SetPreconditioner(*M_prec);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);
+
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -458,21 +335,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
   M_solver.Mult(z, y);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   MFEM_VERIFY(dg_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   K.Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -9,7 +9,6 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-hexagon.mesh -p 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.002 -tf 9
-//    mpirun -np 4 ex9p -m ../data/amr-quad.mesh -p 1 -rp 1 -dt 0.02 -s 13 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-q3.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/star-mixed.mesh -p 1 -rp 1 -dt 0.004 -tf 9
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 1 -rp 1 -dt 0.005 -tf 9
@@ -17,23 +16,17 @@
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 3 -rp 2 -dt 0.0025 -tf 9 -vs 20
 //    mpirun -np 4 ex9p -m ../data/periodic-cube.mesh -p 0 -o 2 -rp 1 -dt 0.01 -tf 8
 //
-// Device sample runs:
-//    mpirun -np 4 ex9p -pa
-//    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh
-//    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -58,163 +51,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-struct AIR_parameters
-{
-   int blocksize;
-   int distanceR;
-   std::string prerelax;
-   std::string postrelax;
-   int interp_type;
-   int relax_type;
-   int coarsen_type;
-   double strength_tolC;
-   double strength_tolR;
-   double filter_tolR;
-   double filterA_tol;
-};
-
-class AIR_prec : public Solver
-{
-private:
-   const HypreParMatrix *A;
-   HypreParMatrix A_s;
-
-   // Preconditioner/solvers for A
-   HypreBoomerAMG *AIR_solver;
-   const AIR_parameters &AIR;
-   int blocksize;
-
-public:
-
-   AIR_prec(const AIR_parameters &_AIR) :
-      AIR_solver(NULL), AIR(_AIR)
-   {
-      blocksize = AIR.blocksize;
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      A = dynamic_cast<const HypreParMatrix *>(&op);
-      delete AIR_solver;
-
-      // Scale A by block-diagonal inverse
-#if MFEM_HYPRE_VERSION >= 21800
-      BlockInverseScale(A, &A_s, NULL, NULL, blocksize, 0);
-      AIR_solver = new HypreBoomerAMG(A_s);
-      AIR_solver->SetLAIROptions(AIR.distanceR, AIR.prerelax,
-                                 AIR.postrelax, AIR.strength_tolC,
-                                 AIR.strength_tolR, AIR.filter_tolR,
-                                 AIR.interp_type, AIR.relax_type,
-                                 AIR.filterA_tol, AIR.coarsen_type,
-                                 -1, 1);
-#else
-      MFEM_ABORT("Must have MFEM_HYPRE_VERSION >= 21800 to use AIR.\n");
-#endif
-      AIR_solver->SetPrintLevel(0);
-      AIR_solver->SetMaxLevels(50);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      // scale the rhs by block inverse and solve system
-      HypreParVector z_s;
-#if MFEM_HYPRE_VERSION >= 21800
-      BlockInverseScale(A, NULL, &x, &z_s, blocksize, 2);
-#endif
-      AIR_solver->Mult(z_s, y);
-   }
-
-   ~AIR_prec()
-   {
-#if MFEM_HYPRE_VERSION >= 21800
-      BlockInverseScale(NULL, NULL, NULL, NULL, 0, -1);
-#endif
-      delete AIR_solver;
-   }
-};
-
-class DG_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &K;
-   SparseMatrix M_diag;
-   HypreParMatrix *A;
-   GMRESSolver linear_solver;
-   Solver *prec;
-   double dt;
-public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        A(NULL),
-        linear_solver(M.GetComm()),
-        dt(-1.0)
-   {
-      prec = new BlockILU(fes.GetFE(0)->GetDof(),
-                          BlockILU::Reordering::MINIMUM_DISCARDED_FILL);
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(*prec);
-
-      M.GetDiag(M_diag);
-   }
-
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes,
-             const AIR_parameters &_AIR)
-      : M(M_),
-        K(K_),
-        A(NULL),
-        linear_solver(M.GetComm()),
-        dt(-1.0)
-   {
-      prec = new AIR_prec(_AIR);
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(*prec);
-
-      M.GetDiag(M_diag);
-   }
-
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         delete A;
-         A = Add(-dt, K, 0.0, K);
-         SparseMatrix A_diag;
-         A->GetDiag(A_diag);
-         A_diag.Add(1.0, M_diag);
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   ~DG_Solver()
-   {
-      delete prec;
-      delete A;
-   }
-};
-

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -224,23 +60,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   OperatorHandle M, K;
+   HypreParMatrix &M, &K;
   const Vector &b;
-   Solver *M_prec;
+   HypreSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b,
-                const AIR_parameters &_AIR);
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };


@@ -258,20 +90,14 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 0;
   int order = 3;
-   bool pa = false;
-   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
   double dt = 0.01;
   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;
-   int solver_type = 1;
-   AIR_parameters AIR0 = {-1, 1, "", "FA", 100, 10, 10,
-                          0.1, 0.01, 0.0, 1e-4
-                         };
+
   int precision = 8;
   cout.precision(precision);

@@ -286,32 +112,19 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly in parallel.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6,\n\t"
-                  "            11 - Backward Euler,\n\t"
-                  "            12 - SDIRK23 (L-stable), 13 - SDIRK33,\n\t"
-                  "            22 - Implicit Midpoint Method,\n\t"
-                  "            23 - SDIRK23 (A-stable), 24 - SDIRK34");
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
   args.AddOption(&t_final, "-tf", "--t-final",
                  "Final time; start time is 0.");
   args.AddOption(&dt, "-dt", "--time-step",
                  "Time step.");
-   args.AddOption(&solver_type, "-st", "--solver-type",
-                  "Solver for implicit solves. 0 for ILU, 1 for pAIR-AMG.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -332,9 +145,6 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle geometrically periodic meshes in this code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -345,20 +155,11 @@ int main(int argc, char *argv[])
   ODESolver *ode_solver = NULL;
   switch (ode_solver_type)
   {
-      // Explicit methods
      case 1: ode_solver = new ForwardEulerSolver; break;
      case 2: ode_solver = new RK2Solver(1.0); break;
      case 3: ode_solver = new RK3SSPSolver; break;
      case 4: ode_solver = new RK4Solver; break;
      case 6: ode_solver = new RK6Solver; break;
-      // Implicit (L-stable) methods
-      case 11: ode_solver = new BackwardEulerSolver; break;
-      case 12: ode_solver = new SDIRK23Solver(2); break;
-      case 13: ode_solver = new SDIRK33Solver; break;
-      // Implicit A-stable methods (not L-stable)
-      case 22: ode_solver = new ImplicitMidpointSolver; break;
-      case 23: ode_solver = new SDIRK23Solver; break;
-      case 24: ode_solver = new SDIRK34Solver; break;
      default:
         if (myid == 0)
         {
@@ -395,7 +196,7 @@ int main(int argc, char *argv[])

   // 7. Define the parallel discontinuous DG finite element space on the
   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);

   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
@@ -404,9 +205,6 @@ int main(int argc, char *argv[])
      cout << "Number of unknowns: " << global_vSize << endl;
   }

-   // Get DG blocksize for AIR
-   AIR0.blocksize = fes->GetFE(0)->GetDof();
-
   // 8. Set up and assemble the parallel bilinear and linear forms (and the
   //    parallel hypre matrices) corresponding to the DG discretization. The
   //    DGTraceIntegrator involves integrals over mesh interior faces.
@@ -415,13 +213,8 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   if (pa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
   m->AddDomainIntegrator(new MassIntegrator);
+   ParBilinearForm *k = new ParBilinearForm(fes);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -432,14 +225,15 @@ int main(int argc, char *argv[])
   b->AddBdrFaceIntegrator(
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

-   int skip_zeros = 0;
   m->Assemble();
-   k->Assemble(skip_zeros);
-   b->Assemble();
   m->Finalize();
+   int skip_zeros = 0;
+   k->Assemble(skip_zeros);
   k->Finalize(skip_zeros);
+   b->Assemble();

-
+   HypreParMatrix *M = m->ParallelAssemble();
+   HypreParMatrix *K = k->ParallelAssemble();
   HypreParVector *B = b->ParallelAssemble();

   // 9. Define the initial conditions, save the corresponding grid function to
@@ -487,20 +281,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9P", pmesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
   socketstream sout;
   if (visualization)
   {
@@ -534,19 +314,11 @@ int main(int argc, char *argv[])
   // 10. Define the time-dependent evolution operator describing the ODE
   //     right-hand side, and perform time-integration (looping over the time
   //     iterations, ti, with a time-step dt).
-   FE_Evolution *adv;
-   if (solver_type == 1)
-   {
-      adv = new FE_Evolution(*m, *k, *B, AIR0);
-   }
-   else
-   {
-      adv = new FE_Evolution(*m, *k, *B);
-   }
+   FE_Evolution adv(*M, *K, *B);

   double t = 0.0;
-   adv->SetTime(t);
-   ode_solver->Init(*adv);
+   adv.SetTime(t);
+   ode_solver->Init(adv);

   bool done = false;
   for (int ti = 0; !done; )
@@ -580,13 +352,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
      }
   }

@@ -606,14 +371,14 @@ int main(int argc, char *argv[])
   delete u;
   delete B;
   delete b;
+   delete K;
   delete k;
+   delete M;
   delete m;
   delete fes;
   delete pmesh;
   delete ode_solver;
-   delete pd;
   delete dc;
-   delete adv;

   MPI_Finalize();
   return 0;
@@ -621,44 +386,15 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
+FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
                           const Vector &_b)
-   : TimeDependentOperator(_M.Height()), b(_b),
-     M_solver(_M.ParFESpace()->GetComm()),
-     z(_M.Height())
+   : TimeDependentOperator(_M.Height()),
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
 {
-   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);

-   if (pa)
-   {
-      M.Reset(&_M, false);
-      K.Reset(&_K, false);
-   }
-   else
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-
-   M_solver.SetOperator(*M);
-
-   Array<int> ess_tdof_list;
-   if (pa)
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }
-   else
-   {
-      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
-      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
-      M_prec = hypre_prec;
-
-      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
-   }
-
-   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -666,67 +402,14 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
   M_solver.SetPrintLevel(0);
 }

-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
-                           const Vector &_b, const AIR_parameters &_AIR)
-   : TimeDependentOperator(_M.Height()), b(_b),
-     M_solver(_M.ParFESpace()->GetComm()),
-     z(_M.Height())
-{
-   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
-
-   if (pa)
-   {
-      MFEM_ABORT("AIR solver not available for partial assembly.\n");
-   }
-   else
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-
-   HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-   HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
-   HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::GS);
-   M_prec = hypre_prec;
-
-   dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace(), _AIR);
-
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.SetOperator(*M);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-// Solve the equation:
-//    u_t = M^{-1}(Ku + b),
-// by solving associated linear system
-//    (M - dt*K) d = K*u + b
-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   K->Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
   // y = M^{-1} (K x + b)
-   K->Mult(x, z);
+   K.Mult(x, z);
   z += b;
   M_solver.Mult(z, y);
 }

-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}
-

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(GINKGO_EXAMPLES_SRCS)
-list(APPEND GINKGO_EXAMPLES_SRCS
-  ex1.cpp
-)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_ginkgo" target, see below.
-add_custom_target(test_ginkgo
-  ${CMAKE_CTEST_COMMAND} -R ginkgo USES_TERMINAL)
-
-# Add one executable per cpp file, adding "ginkgo_" as prefix. Sets
-# "test_ginkgo" as a target that depends on the given examples.
-set(PFX ginkgo_)
-add_mfem_examples(GINKGO_EXAMPLES_SRCS ${PFX} "" test_ginkgo)
-
-# Testing.
-# The GINKGO tests can be run separately using the target "test_ginkgo"
-# which builds the examples and runs:
-#   ctest -R ginkgo
-
-# Command line options for the tests.
-set(EX1_COMMON_OPTS ex1 -m ../data/star.mesh --use_gko_solver)
-set(EX1_TEST_OPTS ${EX9_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${GINKGO_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,259 +0,0 @@
-//                                MFEM Example 1
-//                             GINKGO Modification
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_GINKGO
-#error This example requires that MFEM is built with MFEM_USE_GINKGO=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool use_ginkgo_solver= true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&use_ginkgo_solver, "-gko", "--use_gko_solver", "-no-gko",
-                  "--no-gko-solver",
-                  "Solve using ginkgo.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (mesh->GetNodes())
-   {
-      fec = mesh->GetNodes()->OwnFEC();
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace->GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm *b = new LinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(one));
-   b->Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
-      if (use_ginkgo_solver)
-      {
-#ifdef MFEM_USE_GINKGO
-         // Solve the linear system with CG + ILU from Ginkgo.
-         std::string executor = "reference";
-         auto exec = gko::ReferenceExecutor::create();
-         auto ilu_precond =
-            gko::preconditioner::Ilu<gko::solver::LowerTrs<>,
-            gko::solver::UpperTrs<>, false>::build()
-            .on(exec);
-         GinkgoWrappers::CGSolver ginkgo_solver(executor, 1, 2000, 1e-12, 0.0,
-                                                ilu_precond.release() );
-         ginkgo_solver.solve(&((SparseMatrix&)(*A)), X, B);
-#endif
-      }
-      else
-      {
-#ifndef MFEM_USE_SUITESPARSE
-         // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-         GSSmoother M((SparseMatrix&)(*A));
-         PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-         // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-         UMFPackSolver umf_solver;
-         umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-         umf_solver.SetOperator(*A);
-         umf_solver.Mult(B, X);
-#endif
-      }
-   }
-   else // No preconditioning for now in partial assembly mode.
-   {
-      CG(*A, B, X, 1, 2000, 1e-12, 0.0);
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh->Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete mesh;
-
-   return 0;
-}
@@ -1,81 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/ginkgo/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-# Currently there are only serial Ginkgo examples
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES =
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_GINKGO),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with GINKO)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI_NP = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
-RUN_MPI = $(RUN_MPI_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial GINKGO example
-PARALLEL_NAME := Parallel GINKGO example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: Example-specific execution options:
-ex1-test-par: ex1
-	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f refined.mesh sol.gf mesh.* sol.*
@@ -1,63 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(HIOP_EXAMPLES_SRCS)
-list(APPEND HIOP_EXAMPLES_SRCS ex9.cpp)
-
-if (MFEM_USE_MPI)
-  list(APPEND HIOP_EXAMPLES_SRCS ex9p.cpp)
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_hiop" target, see below.
-add_custom_target(test_hiop
-  ${CMAKE_CTEST_COMMAND} -R hiop USES_TERMINAL)
-
-# Add one executable per cpp file, adding "hiop_" as prefix. Sets
-# "test_hiop" as a target that depends on the given examples.
-set(PFX hiop_)
-add_mfem_examples(HIOP_EXAMPLES_SRCS ${PFX} "" test_hiop)
-
-# Testing.
-# The HIOP tests can be run separately using the target "test_hiop"
-# which builds the examples and runs:
-#   ctest -R hiop
-
-# Command line options for the tests.
-# Example 9:
-set(EX9_COMMON_OPTS -m ../../data/periodic-segment.mesh -p 0 -dt 0.005)
-set(EX9_TEST_OPTS ${EX9_COMMON_OPTS}  -r  2 )
-set(EX9P_TEST_OPTS ${EX9_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${HIOP_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,684 +0,0 @@
-//            MFEM Example 9 with Nonlinear Constrained Optimization
-//
-// Compile with: make ex9
-//
-// Sample runs:
-//
-//    ex9 -m ../../data/periodic-segment.mesh -r 3 -p 0 -o 2 -dt 0.002 -opt 1
-//    ex9 -m ../../data/periodic-segment.mesh -r 3 -p 0 -o 2 -dt 0.002 -opt 2
-//
-//    ex9 -m ../../data/periodic-square.mesh -p 0 -r 2 -dt 0.01 -tf 10 -opt 1
-//    ex9 -m ../../data/periodic-square.mesh -p 0 -r 2 -dt 0.01 -tf 10 -opt 2
-//
-//    ex9 -m ../../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 1
-//    ex9 -m ../../data/periodic-square.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/amr-quad.mesh -p 1 -r 1 -dt 0.002 -tf 9 -opt 1
-//    ex9 -m ../../data/amr-quad.mesh -p 1 -r 1 -dt 0.002 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/disc-nurbs.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 1
-//    ex9 -m ../../data/disc-nurbs.mesh -p 1 -r 2 -dt 0.005 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/disc-nurbs.mesh -p 2 -r 2 -dt 0.01 -tf 9 -opt 1
-//    ex9 -m ../../data/disc-nurbs.mesh -p 2 -r 2 -dt 0.01 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/periodic-square.mesh -p 3 -r 3 -dt 0.0025 -tf 9 -opt 1
-//    ex9 -m ../../data/periodic-square.mesh -p 3 -r 3 -dt 0.0025 -tf 9 -opt 2
-//
-//    ex9 -m ../../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8 -opt 1
-//    ex9 -m ../../data/periodic-cube.mesh -p 0 -r 2 -o 2 -dt 0.02 -tf 8 -opt 2
-
-// Description:  This example modifies the standard MFEM ex9 by adding nonlinear
-//               constrained optimization capabilities through the SLBQP and
-//               HIOP solvers. It demonstrates how a user can define a custom
-//               class OptimizationProblem that includes linear/nonlinear
-//               equality/inequality constraints. This optimization is applied
-//               as post-processing to the solution of the transport equation.
-//
-//               Description of ex9:
-//               This example code solves the time-dependent advection equation
-//               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem;
-
-// Nonlinear optimizer.
-int optimizer_type;
-
-// Velocity coefficient
-bool invert_velocity = false;
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-/// Computes C(x) = sum w_i x_i, where w is a given Vector.
-class LinearScaleOperator : public Operator
-{
-private:
-   const Vector &w;
-   mutable DenseMatrix grad;
-
-public:
-   LinearScaleOperator(const Vector &weight)
-      : Operator(1, weight.Size()), w(weight), grad(1, width)
-   {
-      for (int i = 0; i < width; i++) { grad(0, i) = w(i); }
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      y(0) = w * x;
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      return grad;
-   }
-};
-
-/// Nonlinear monotone bounded operator to test nonlinear ineq constraints.
-/// Computes D(x) = tanh(sum(x_i)).
-class TanhSumOperator : public Operator
-{
-private:
-   mutable DenseMatrix grad;
-
-public:
-   TanhSumOperator(int size) : Operator(1, size), grad(1, width) { }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      y(0) = std::tanh(x.Sum());
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      const double ts = std::tanh(x.Sum());
-      const double dtanh = 1.0 - ts * ts;
-      for (int i = 0; i < width; i++) { grad(0, i) = dtanh; }
-      return grad;
-   }
-};
-
-/** Monotone and conservative a-posteriori correction for transport solutions:
- *  Find x that minimizes 0.5 || x - x_HO ||^2, subject to
- *  sum w_i x_i = mass,
- *  tanh(sum(x_i_min)) <= tanh(sum(x_i)) <= tanh(sum(x_i_max)),
- *  x_i_min <= x_i <= x_i_max,
- */
-class OptimizedTransportProblem : public OptimizationProblem
-{
-private:
-   const Vector &x_HO;
-   Vector massvec, d_lo, d_hi;
-   const LinearScaleOperator LSoper;
-   const TanhSumOperator TSoper;
-
-public:
-   OptimizedTransportProblem(const Vector &xho, const Vector &w, double mass,
-                             const Vector &xmin, const Vector &xmax)
-      : OptimizationProblem(xho.Size(), NULL, NULL),
-        x_HO(xho), massvec(1), d_lo(1), d_hi(1),
-        LSoper(w), TSoper(w.Size())
-   {
-      C = &LSoper;
-      massvec(0) = mass;
-      SetEqualityConstraint(massvec);
-
-      D = &TSoper;
-      d_lo(0) = std::tanh(xmin.Sum());
-      d_hi(0) = std::tanh(xmax.Sum());
-      MFEM_ASSERT(d_lo(0) < d_hi(0),
-                  "The bounds produce an infeasible optimization problem");
-      SetInequalityConstraint(d_lo, d_hi);
-
-      SetSolutionBounds(xmin, xmax);
-   }
-
-   virtual double CalcObjective(const Vector &x) const
-   {
-      double res = 0.0;
-      for (int i = 0; i < input_size; i++)
-      {
-         const double d = x(i) - x_HO(i);
-         res += d * d;
-      }
-      return 0.5 * res;
-   }
-
-   virtual void CalcObjectiveGrad(const Vector &x, Vector &grad) const
-   {
-      for (int i = 0; i < input_size; i++) { grad(i) = x(i) - x_HO(i); }
-   }
-};
-
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
-    and advection matrices, and b describes the flow on the boundary. This can
-    be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
-    used to evaluate the right-hand side. */
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   SparseMatrix &M, &K;
-   const Vector &b;
-   DSmoother M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-   double dt;
-   BilinearForm &bf;
-   Vector &M_rowsums;
-
-public:
-   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b,
-                BilinearForm &_bf, Vector &M_rs);
-
-   void SetTimeStep(double _dt) { dt = _dt; }
-   void SetK(SparseMatrix &_K) { K = _K; }
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution() { }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   problem = 0;
-   optimizer_type = 1;
-   const char *mesh_file = "../../data/periodic-hexagon.mesh";
-   int ref_levels = 2;
-   int order = 3;
-   int ode_solver_type = 3;
-   double t_final = 1.0;
-   double dt = 0.01;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&optimizer_type, "-opt", "--optimizer",
-                  "Nonlinear optimizer: 1 - SLBQP,\n\t"
-                  "                     2 - HIOP.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Read the mesh from the given mesh file. We can handle geometrically
-   //    periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 3. Define the ODE solver used for time integration. Several explicit
-   //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      default:
-         cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         delete mesh;
-         return 3;
-   }
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement, where 'ref_levels' is a
-   //    command-line parameter. If the mesh is of NURBS type, we convert it to
-   //    a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::Positive);
-   FiniteElementSpace fes(mesh, &fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   // 6. Set up and assemble the bilinear and linear forms corresponding to the
-   //    DG discretization. The DGTraceIntegrator involves integrals over mesh
-   //    interior faces.
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   BilinearForm m(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
-   BilinearForm k(&fes);
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   k.AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-   k.AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-
-   LinearForm b(&fes);
-   b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
-
-   m.Assemble();
-   m.Finalize();
-   int skip_zeros = 0;
-   k.Assemble(skip_zeros);
-   k.Finalize(skip_zeros);
-   b.Assemble();
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u(&fes);
-   u.ProjectCoefficient(u0);
-
-   {
-      ofstream omesh("ex9.mesh");
-      omesh.precision(precision);
-      mesh->Print(omesh);
-      ofstream osol("ex9-init.gf");
-      osol.precision(precision);
-      u.Save(osol);
-   }
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9", mesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example9", mesh);
-         dc->SetPrecision(precision);
-      }
-      dc->RegisterField("solution", &u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << *mesh << u;
-         sout << "pause\n";
-         sout << flush;
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   Vector M_rowsums(m.Size());
-   m.SpMat().GetRowSums(M_rowsums);
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m.SpMat(), k.SpMat(), b, k, M_rowsums);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   // Compute initial volume.
-   const double vol0 = M_rowsums * u;
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-      adv.SetTimeStep(dt_real);
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-
-         if (visualization)
-         {
-            sout << "solution\n" << *mesh << u << flush;
-         }
-
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-      }
-   }
-
-   // Print the error vs exact solution.
-   const double max_error = u.ComputeMaxError(u0),
-                l1_error  = u.ComputeL1Error(u0),
-                l2_error  = u.ComputeL2Error(u0);
-   std::cout << "Linf error = " << max_error << endl
-             << "L1   error = " << l1_error << endl
-             << "L2   error = " << l2_error << endl;
-
-   // Print error in volume.
-   const double vol = M_rowsums * u;
-   std::cout << "Vol  error = " << vol - vol0 << endl;
-
-   // 9. Save the final solution. This output can be viewed later using GLVis:
-   //    "glvis -m ex9.mesh -g ex9-final.gf".
-   {
-      ofstream osol("ex9-final.gf");
-      osol.precision(precision);
-      u.Save(osol);
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete dc;
-   delete mesh;
-
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K,
-                           const Vector &_b, BilinearForm &_bf, Vector &M_rs)
-   : TimeDependentOperator(_M.Size()),
-     M(_M), K(_K), b(_b), M_prec(), M_solver(), z(_M.Size()),
-     bf(_bf), M_rowsums(M_rs)
-{
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // Compute bounds y_min, y_max for y from x on the ldofs.
-   const int dofs = x.Size();
-   Vector y_min(dofs), y_max(dofs);
-   const int *In = bf.SpMat().GetI(), *Jn = bf.SpMat().GetJ();
-   for (int i = 0, k = 0; i < dofs; i++)
-   {
-      double x_i_min = +std::numeric_limits<double>::infinity();
-      double x_i_max = -std::numeric_limits<double>::infinity();
-      for (int end = In[i+1]; k < end; k++)
-      {
-         const int j = Jn[k];
-         if (x(j) > x_i_max) { x_i_max = x(j); }
-         if (x(j) < x_i_min) { x_i_min = x(j); }
-      }
-      y_min(i) = x_i_min;
-      y_max(i) = x_i_max;
-   }
-   for (int i = 0; i < dofs; i++)
-   {
-      y_min(i) = (y_min(i) - x(i) ) / dt;
-      y_max(i) = (y_max(i) - x(i) ) / dt;
-   }
-
-   // Compute the high-order solution y = M^{-1} (K x + b).
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-
-   // The solution y is an increment; it should not introduce new mass.
-   const double mass_y = 0.0;
-
-   // Perform optimization.
-   Vector y_out(dofs);
-   const int max_iter = 500;
-   const double rtol = 1.e-7;
-   double atol = 1.e-7;
-
-   OptimizationSolver *optsolver = NULL;
-   if (optimizer_type == 2)
-   {
-#ifdef MFEM_USE_HIOP
-      HiopNlpOptimizer *tmp_opt_ptr = new HiopNlpOptimizer();
-      optsolver = tmp_opt_ptr;
-#else
-      MFEM_ABORT("MFEM is not built with HiOp support!");
-#endif
-   }
-   else
-   {
-      SLBQPOptimizer *slbqp = new SLBQPOptimizer();
-      slbqp->SetBounds(y_min, y_max);
-      slbqp->SetLinearConstraint(M_rowsums, mass_y);
-      atol = 1.e-15;
-      optsolver = slbqp;
-   }
-
-   OptimizedTransportProblem ot_prob(y, M_rowsums, mass_y, y_min, y_max);
-   optsolver->SetOptimizationProblem(ot_prob);
-
-   optsolver->SetMaxIter(max_iter);
-   optsolver->SetAbsTol(atol);
-   optsolver->SetRelTol(rtol);
-   optsolver->SetPrintLevel(0);
-   optsolver->Mult(y, y_out);
-
-   y = y_out;
-
-   delete optsolver;
-}
-
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = (invert_velocity) ? -1.0 : 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return (X(0) > -0.15 && X(0) < 0.15) ? 1.0 : 0.0;
-            //return exp(-40.*pow(X(0)-0.0,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}
@@ -1,802 +0,0 @@
-//  MFEM Example 9 with Nonlinear Constrained Optimization - Parallel Version
-//
-// Compile with: make ex9p
-//
-// Sample runs:
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 0 -rs 2 -dt 0.01 -tf 10 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 0 -rs 2 -dt 0.01 -tf 10 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh -p 1 -rs 1 -dt 0.002 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/amr-quad.mesh -p 1 -rs 1 -dt 0.002 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 1 -rs 2 -dt 0.005 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 2 -rs 2 -dt 0.01 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh -p 2 -rs 2 -dt 0.01 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 3 -rs 3 -dt 0.0025 -tf 9 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-square.mesh -p 3 -rs 3 -dt 0.0025 -tf 9 -opt 2
-//
-//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.02 -tf 8 -opt 1
-//    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.02 -tf 8 -opt 2
-
-// Description:  This example modifies the standard MFEM ex9 by adding nonlinear
-//               constrained optimization capabilities through the SLBQP and
-//               HIOP solvers. It demonstrates how a user can define a custom
-//               class OptimizationProblem that includes linear/nonlinear
-//               equality/inequality constraints. This optimization is applied
-//               as post-processing to the solution of the transport equation.
-//
-//               Description of ex9:
-//               This example code solves the time-dependent advection equation
-//               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
-//               u0(x)=u(0,x) is a given initial condition.
-//
-//               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of explicit
-//               ODE time integrators, the definition of periodic boundary
-//               conditions through periodic meshes, as well as the use of GLVis
-//               for persistent visualization of a time-evolving solution. The
-//               saving of time-dependent data files for external visualization
-//               with VisIt (visit.llnl.gov) is also illustrated.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Choice for the problem setup. The fluid velocity, initial condition and
-// inflow boundary condition are chosen based on this parameter.
-int problem;
-
-// Nonlinear optimizer.
-int optimizer_type;
-
-// Velocity coefficient
-bool invert_velocity = false;
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-/// Computes C(x) = sum w_i x_i, where w is a given Vector.
-class LinearScaleOperator : public Operator
-{
-private:
-   ParFiniteElementSpace &pfes;
-   // Local weights.
-   const Vector &w;
-   // Gradient for the tdofs.
-   mutable DenseMatrix grad;
-
-public:
-   LinearScaleOperator(ParFiniteElementSpace &space, const Vector &weight)
-      : Operator(1, space.TrueVSize()),
-        pfes(space), w(weight), grad(1, width)
-   {
-      Vector w_glob(width);
-      pfes.Dof_TrueDof_Matrix()->MultTranspose(w, w_glob);
-      for (int i = 0; i < width; i++) { grad(0, i) = w_glob(i); }
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      Vector x_loc(w.Size());
-      pfes.GetProlongationMatrix()->Mult(x, x_loc);
-      const double loc_res = w * x_loc;
-      MPI_Allreduce(&loc_res, &y(0), 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      return grad;
-   }
-};
-
-/// Nonlinear monotone bounded operator to test nonlinear ineq constraints.
-/// Computes D(x) = tanh(sum(x_i)).
-class TanhSumOperator : public Operator
-{
-private:
-   // Gradient for the tdofs.
-   mutable DenseMatrix grad;
-
-public:
-   TanhSumOperator(ParFiniteElementSpace &space)
-      : Operator(1, space.TrueVSize()), grad(1, width) { }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      double sum_loc = x.Sum();
-      MPI_Allreduce(&sum_loc, &y(0), 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      y(0) = std::tanh(y(0));
-   }
-
-   virtual Operator &GetGradient(const Vector &x) const
-   {
-      double sum_loc = x.Sum();
-      double dtanh;
-      MPI_Allreduce(&sum_loc, &dtanh, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      dtanh = 1.0 - pow(std::tanh(dtanh), 2);
-
-      for (int i = 0; i < width; i++) { grad(0, i) = dtanh; }
-      return grad;
-   }
-};
-
-
-/** Monotone and conservative a-posteriori correction for transport solutions:
- *  Find x that minimizes 0.5 || x - x_HO ||^2, subject to
- *  sum w_i x_i = mass,
- *  tanh(sum(x_i_min)) <= tanh(sum(x_i)) <= tanh(sum(x_i_max)),
- *  x_i_min <= x_i <= x_i_max,
- */
-class OptimizedTransportProblem : public OptimizationProblem
-{
-private:
-   const Vector &x_HO;
-   Vector massvec, d_lo, d_hi;
-   const LinearScaleOperator LSoper;
-   const TanhSumOperator TSoper;
-
-public:
-   OptimizedTransportProblem(ParFiniteElementSpace &space,
-                             const Vector &xho, const Vector &w, double mass,
-                             const Vector &xmin, const Vector &xmax)
-      : OptimizationProblem(xho.Size(), NULL, NULL),
-        x_HO(xho), massvec(1), d_lo(1), d_hi(1),
-        LSoper(space, w), TSoper(space)
-   {
-      C = &LSoper;
-      massvec(0) = mass;
-      SetEqualityConstraint(massvec);
-
-      D = &TSoper;
-      double lsums[2], gsums[2];
-      lsums[0] = xmin.Sum();
-      lsums[1] = xmax.Sum();
-      MPI_Allreduce(lsums, gsums, 2, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      d_lo(0) = std::tanh(gsums[0]);
-      d_hi(0) = std::tanh(gsums[1]);
-      MFEM_ASSERT(d_lo(0) < d_hi(0),
-                  "The bounds produce an infeasible optimization problem");
-      SetInequalityConstraint(d_lo, d_hi);
-
-      SetSolutionBounds(xmin, xmax);
-   }
-
-   virtual double CalcObjective(const Vector &x) const
-   {
-      double loc_res = 0.0;
-      for (int i = 0; i < input_size; i++)
-      {
-         const double d = x(i) - x_HO(i);
-         loc_res += d * d;
-      }
-      loc_res *= 0.5;
-      double res;
-      MPI_Allreduce(&loc_res, &res, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      return res;
-   }
-
-   virtual void CalcObjectiveGrad(const Vector &x, Vector &grad) const
-   {
-      for (int i = 0; i < input_size; i++) { grad(i) = x(i) - x_HO(i); }
-   }
-};
-
-
-/** A time-dependent operator for the right-hand side of the ODE. The DG weak
-    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
-    and advection matrices, and b describes the flow on the boundary. This can
-    be written as a general ODE, du/dt = M^{-1} (K u + b), and this class is
-    used to evaluate the right-hand side. */
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   HypreParMatrix &M, &K;
-   const Vector &b;
-   HypreSmoother M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-   double dt;
-   ParBilinearForm &pbf;
-   Vector &M_rowsums;
-
-public:
-   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                const Vector &_b, ParBilinearForm &_pbf, Vector &M_rs);
-
-   void SetTimeStep(double _dt) { dt = _dt; }
-   void SetK(HypreParMatrix &_K) { K = _K; }
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution() { }
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   problem = 0;
-   optimizer_type = 1;
-   const char *mesh_file = "../../data/periodic-hexagon.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 0;
-   int order = 3;
-   int ode_solver_type = 3;
-   double t_final = 1.0;
-   double dt = 0.01;
-   bool visualization = true;
-   bool visit = false;
-   bool binary = false;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&problem, "-p", "--problem",
-                  "Problem setup to use. See options in velocity_function().");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&optimizer_type, "-opt", "--optimizer",
-                  "Nonlinear optimizer: 1 - SLBQP,\n\t"
-                  "                     2 - HIOP.");
-   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
-                  "ODE solver: 1 - Forward Euler,\n\t"
-                  "            2 - RK2 SSP, 3 - RK3 SSP.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
-                  "--no-visit-datafiles",
-                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
-                  "--ascii-datafiles",
-                  "Use binary (Sidre) or ascii format for VisIt data files.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0) { args.PrintUsage(cout); }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0) { args.PrintOptions(cout); }
-
-   // 3. Read the serial mesh from the given mesh file on all processors. We can
-   //    handle geometrically periodic meshes in this code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Define the ODE solver used for time integration. Several explicit
-   //    Runge-Kutta methods are available.
-   ODESolver *ode_solver = NULL;
-   switch (ode_solver_type)
-   {
-      case 1: ode_solver = new ForwardEulerSolver; break;
-      case 2: ode_solver = new RK2Solver(1.0); break;
-      case 3: ode_solver = new RK3SSPSolver; break;
-      case 4: ode_solver = new RK4Solver; break;
-      case 6: ode_solver = new RK6Solver; break;
-      default:
-         if (myid == 0)
-         {
-            cout << "Unknown ODE solver type: " << ode_solver_type << '\n';
-         }
-         delete mesh;
-         MPI_Finalize();
-         return 3;
-   }
-
-   // 5. Refine the mesh in serial to increase the resolution. In this example
-   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
-   //    a command-line parameter. If the mesh is of NURBS type, we convert it
-   //    to a (piecewise-polynomial) high-order mesh.
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-   if (mesh->NURBSext)
-   {
-      mesh->SetCurvature(max(order, 1));
-   }
-   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 7. Define the parallel discontinuous DG finite element space on the
-   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim, BasisType::Positive);
-   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
-
-   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << global_vSize << endl;
-   }
-
-   // 8. Set up and assemble the parallel bilinear and linear forms (and the
-   //    parallel hypre matrices) corresponding to the DG discretization. The
-   //    DGTraceIntegrator involves integrals over mesh interior faces.
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   ParBilinearForm *m = new ParBilinearForm(fes);
-   m->AddDomainIntegrator(new MassIntegrator);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
-   k->AddInteriorFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-   k->AddBdrFaceIntegrator(
-      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
-
-   ParLinearForm *b = new ParLinearForm(fes);
-   b->AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));
-
-   m->Assemble();
-   m->Finalize();
-   int skip_zeros = 0;
-   k->Assemble(skip_zeros);
-   k->Finalize(skip_zeros);
-   b->Assemble();
-
-   HypreParMatrix *M = m->ParallelAssemble();
-   HypreParMatrix *K = k->ParallelAssemble();
-   HypreParVector *B = b->ParallelAssemble();
-
-   // 9. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   ParGridFunction *u = new ParGridFunction(fes);
-   u->ProjectCoefficient(u0);
-   HypreParVector *U = u->GetTrueDofs();
-
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "ex9-mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "ex9-init." << setfill('0') << setw(6) << myid;
-      ofstream omesh(mesh_name.str().c_str());
-      omesh.precision(precision);
-      pmesh->Print(omesh);
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u->Save(osol);
-   }
-
-   // Create data collection for solution output: either VisItDataCollection for
-   // ascii data files, or SidreDataCollection for binary data files.
-   DataCollection *dc = NULL;
-   if (visit)
-   {
-      if (binary)
-      {
-#ifdef MFEM_USE_SIDRE
-         dc = new SidreDataCollection("Example9-Parallel", pmesh);
-#else
-         MFEM_ABORT("Must build with MFEM_USE_SIDRE=YES for binary output.");
-#endif
-      }
-      else
-      {
-         dc = new VisItDataCollection("Example9-Parallel", pmesh);
-         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
-      }
-      dc->RegisterField("solution", u);
-      dc->SetCycle(0);
-      dc->SetTime(0.0);
-      dc->Save();
-   }
-
-   socketstream sout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      if (!sout)
-      {
-         if (myid == 0)
-            cout << "Unable to connect to GLVis server at "
-                 << vishost << ':' << visport << endl;
-         visualization = false;
-         if (myid == 0)
-         {
-            cout << "GLVis visualization disabled.\n";
-         }
-      }
-      else
-      {
-         sout << "parallel " << num_procs << " " << myid << "\n";
-         sout.precision(precision);
-         sout << "solution\n" << *pmesh << *u;
-         sout << "pause\n";
-         sout << flush;
-         if (myid == 0)
-            cout << "GLVis visualization paused."
-                 << " Press space (in the GLVis window) to resume it.\n";
-      }
-   }
-
-   Vector M_rowsums(m->Size());
-   m->SpMat().GetRowSums(M_rowsums);
-
-   // 10. Define the time-dependent evolution operator describing the ODE
-   //     right-hand side, and perform time-integration (looping over the time
-   //     iterations, ti, with a time-step dt).
-   FE_Evolution adv(*M, *K, *B, *k, M_rowsums);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   ode_solver->Init(adv);
-
-   *u = *U;
-
-   // Compute initial volume.
-   const double vol0_loc = M_rowsums * (*u);
-   double vol0;
-   MPI_Allreduce(&vol0_loc, &vol0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-      adv.SetTimeStep(dt_real);
-      ode_solver->Step(*U, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         if (myid == 0)
-         {
-            cout << "time step: " << ti << ", time: " << t << endl;
-         }
-
-         // 11. Extract the parallel grid function corresponding to the finite
-         //     element approximation U (the local solution on each processor).
-         *u = *U;
-
-         if (visualization)
-         {
-            sout << "parallel " << num_procs << " " << myid << "\n";
-            sout << "solution\n" << *pmesh << *u << flush;
-         }
-
-         if (visit)
-         {
-            dc->SetCycle(ti);
-            dc->SetTime(t);
-            dc->Save();
-         }
-      }
-   }
-
-   // Print the error vs exact solution.
-   const double max_error = u->ComputeMaxError(u0),
-                l1_error  = u->ComputeL1Error(u0),
-                l2_error  = u->ComputeL2Error(u0);
-   if (myid == 0)
-   {
-      std::cout << "Linf error = " << max_error << endl
-                << "L1   error = " << l1_error << endl
-                << "L2   error = " << l2_error << endl;
-   }
-
-   // Print error in volume.
-   const double vol_loc = M_rowsums * (*u);
-   double vol;
-   MPI_Allreduce(&vol_loc, &vol, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-   if (myid == 0)
-   {
-      std::cout << "Vol  error = " << vol - vol0 << endl;
-   }
-
-   // 12. Save the final solution in parallel. This output can be viewed later
-   //     using GLVis: "glvis -np <np> -m ex9-mesh -g ex9-final".
-   {
-      *u = *U;
-      ostringstream sol_name;
-      sol_name << "ex9-final." << setfill('0') << setw(6) << myid;
-      ofstream osol(sol_name.str().c_str());
-      osol.precision(precision);
-      u->Save(osol);
-   }
-
-   // 13. Free the used memory.
-   delete U;
-   delete u;
-   delete B;
-   delete b;
-   delete K;
-   delete k;
-   delete M;
-   delete m;
-   delete fes;
-   delete pmesh;
-   delete ode_solver;
-   delete dc;
-
-   MPI_Finalize();
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
-                           const Vector &_b, ParBilinearForm &_pbf,
-                           Vector &M_rs)
-   : TimeDependentOperator(_M.Height()),
-     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height()),
-     pbf(_pbf), M_rowsums(M_rs)
-{
-   M_prec.SetType(HypreSmoother::Jacobi);
-   M_solver.SetPreconditioner(M_prec);
-   M_solver.SetOperator(M);
-
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // Get values on the ldofs.
-   ParFiniteElementSpace *pfes = pbf.ParFESpace();
-   ParGridFunction x_gf(pfes);
-   pfes->GetProlongationMatrix()->Mult(x, x_gf);
-
-   // Compute bounds y_min, y_max for y from from x on the ldofs.
-   const int ldofs = x_gf.Size();
-   Vector y_min(ldofs), y_max(ldofs);
-   x_gf.ExchangeFaceNbrData();
-   Vector &x_nd = x_gf.FaceNbrData();
-   const int *In = pbf.SpMat().GetI(), *Jn = pbf.SpMat().GetJ();
-   for (int i = 0, k = 0; i < ldofs; i++)
-   {
-      double x_i_min = +std::numeric_limits<double>::infinity();
-      double x_i_max = -std::numeric_limits<double>::infinity();
-      for (int end = In[i+1]; k < end; k++)
-      {
-         const int j = Jn[k];
-         const double x_j = (j < ldofs) ? x(j): x_nd(j-ldofs);
-
-         if (x_j > x_i_max) { x_i_max = x_j; }
-         if (x_j < x_i_min) { x_i_min = x_j; }
-      }
-      y_min(i) = x_i_min;
-      y_max(i) = x_i_max;
-   }
-   for (int i = 0; i < ldofs; i++)
-   {
-      y_min(i) = (y_min(i) - x_gf(i) ) / dt;
-      y_max(i) = (y_max(i) - x_gf(i) ) / dt;
-   }
-   Vector y_min_tdofs(y.Size()), y_max_tdofs(y.Size());
-   // Move the bounds to the tdofs.
-   pfes->GetRestrictionMatrix()->Mult(y_min, y_min_tdofs);
-   pfes->GetRestrictionMatrix()->Mult(y_max, y_max_tdofs);
-
-   // Compute the high-order solution y = M^{-1} (K x + b) on the tdofs.
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-
-   // The solution y is an increment; it should not introduce new mass.
-   const double mass_y = 0.0;
-
-   // Perform optimization on the tdofs.
-   Vector y_out(y.Size());
-   const int max_iter = 500;
-   const double rtol = 1.e-7;
-   double atol = 1.e-7;
-
-   OptimizationSolver* optsolver = NULL;
-   if (optimizer_type == 2)
-   {
-#ifdef MFEM_USE_HIOP
-      HiopNlpOptimizer *tmp_opt_ptr = new HiopNlpOptimizer(MPI_COMM_WORLD);
-      optsolver = tmp_opt_ptr;
-#else
-      MFEM_ABORT("MFEM is not built with HiOp support!");
-#endif
-   }
-   else
-   {
-      SLBQPOptimizer *slbqp = new SLBQPOptimizer(MPI_COMM_WORLD);
-      slbqp->SetBounds(y_min_tdofs, y_max_tdofs);
-      slbqp->SetLinearConstraint(M_rowsums, mass_y);
-      atol = 1.e-15;
-      optsolver = slbqp;
-   }
-
-   OptimizedTransportProblem ot_prob(*pfes, y, M_rowsums, mass_y,
-                                     y_min_tdofs, y_max_tdofs);
-   optsolver->SetOptimizationProblem(ot_prob);
-
-   optsolver->SetMaxIter(max_iter);
-   optsolver->SetAbsTol(atol);
-   optsolver->SetRelTol(rtol);
-   optsolver->SetPrintLevel(0);
-   optsolver->Mult(y, y_out);
-
-   y = y_out;
-
-   delete optsolver;
-}
-
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      {
-         // Translations in 1D, 2D, and 3D
-         switch (dim)
-         {
-            case 1: v(0) = (invert_velocity) ? -1.0 : 1.0; break;
-            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
-            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
-               break;
-         }
-         break;
-      }
-      case 1:
-      case 2:
-      {
-         // Clockwise rotation in 2D around the origin
-         const double w = M_PI/2;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
-            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-      case 3:
-      {
-         // Clockwise twisting rotation in 2D around the origin
-         const double w = M_PI/2;
-         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
-         d = d*d;
-         switch (dim)
-         {
-            case 1: v(0) = 1.0; break;
-            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
-            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
-         }
-         break;
-      }
-   }
-}
-
-// Initial condition
-double u0_function(const Vector &x)
-{
-   int dim = x.Size();
-
-   // map to the reference [-1,1] domain
-   Vector X(dim);
-   for (int i = 0; i < dim; i++)
-   {
-      double center = (bb_min[i] + bb_max[i]) * 0.5;
-      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
-   }
-
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      {
-         switch (dim)
-         {
-            case 1:
-               return (X(0) > -0.15 && X(0) < 0.15) ? 1.0 : 0.0;
-            //return exp(-40.*pow(X(0)-0.0,2));
-            case 2:
-            case 3:
-            {
-               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
-               if (dim == 3)
-               {
-                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
-                  rx *= s;
-                  ry *= s;
-               }
-               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
-                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
-            }
-         }
-      }
-      case 2:
-      {
-         double x_ = X(0), y_ = X(1), rho, phi;
-         rho = hypot(x_, y_);
-         phi = atan2(y_, x_);
-         return pow(sin(M_PI*rho),2)*sin(3*phi);
-      }
-      case 3:
-      {
-         const double f = M_PI;
-         return sin(f*X(0))*sin(f*X(1));
-      }
-   }
-   return 0.0;
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   switch (problem)
-   {
-      case 0:
-      case 1:
-      case 2:
-      case 3: return 0.0;
-   }
-   return 0.0;
-}
@@ -1,66 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/hiop/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex9
-PAR_EXAMPLES = ex9p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_HIOP),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with HIOP)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.* Example9*
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Use the MFEM build directory
 MFEM_DIR ?= ..
@@ -22,9 +22,9 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 ex17\
-  ex18 ex19 ex20 ex21 ex22 ex23 ex24
+  ex18 ex19 ex20 ex21
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p ex12p\
- ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p
+ ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -41,13 +41,6 @@ endif
 ifeq ($(MFEM_USE_PUMI),YES)
   SUBDIRS += pumi
 endif
-ifeq ($(MFEM_USE_HIOP),YES)
-   SUBDIRS += hiop
-endif
-ifeq ($(MFEM_USE_GINKGO),YES)
-   SUBDIRS += ginkgo
-endif
-
 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
 SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
@@ -124,8 +117,8 @@ clean-build:

 clean-exec:
 	@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
-	@rm -rf Example5* Example9* Example15* Example16* Example23* ParaView
-	@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
+	@rm -rf Example5* Example9* Example15* Example16*
+	@rm -f sphere_refined.* sol.* sol_u.* sol_p.*
 	@rm -f ex9.mesh ex9-mesh.* ex9-init.* ex9-final.*
 	@rm -f deformed.* velocity.* elastic_energy.* mode_*
 	@rm -f ex16.mesh ex16-mesh.* ex16-init.* ex16-final.*
@@ -133,4 +126,3 @@ clean-exec:
 	@rm -f deformation.* pressure.*
 	@rm -f ex20.dat ex20p_?????.dat gnuplot_ex20.inp gnuplot_ex20p.inp
 	@rm -f ex21*.mesh ex21*.sol ex21p_*.*
-	@rm -f ex23.mesh ex23-*.gf
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 set(PETSC_EXAMPLES_SRCS "")
 set(PETSC_RC_FILES "")
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Use the MFEM build directory
 MFEM_DIR ?= ../..
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 set(PUMI_EXAMPLES_SRCS)
 # All PUMI examples require MPI
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Use the MFEM build directory
 MFEM_DIR ?= ../..
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 set(SUNDIALS_EXAMPLES_SRCS)
 list(APPEND SUNDIALS_EXAMPLES_SRCS
@@ -1,13 +1,13 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 # Use the MFEM build directory
 MFEM_DIR ?= ../..
@@ -1,29 +1,21 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
+# Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at the
+# Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights reserved.
+# See file COPYRIGHT for details.
 #
 # This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
+# availability see http://mfem.org.
 #
 # MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
+# terms of the GNU Lesser General Public License (as published by the Free
+# Software Foundation) version 2.1 dated February 1999.

 set(SRCS
  bilinearform.cpp
  bilinearform_ext.cpp
  bilininteg.cpp
-  bilininteg_convection.cpp
-  bilininteg_dgtrace.cpp
  bilininteg_diffusion.cpp
-  bilininteg_divergence.cpp
-  bilininteg_gradient.cpp
  bilininteg_mass.cpp
-  bilininteg_vecdiffusion.cpp
-  bilininteg_vecmass.cpp
-  bilininteg_hcurl.cpp
  coefficient.cpp
-  complex_fem.cpp
  datacollection.cpp
  eltrans.cpp
  estimators.cpp
@@ -37,15 +29,10 @@ set(SRCS
  linearform.cpp
  lininteg.cpp
  nonlinearform.cpp
-  nonlinearform_ext.cpp
  nonlininteg.cpp
-  nonlininteg_vectorconvection.cpp
-  quadinterpolator.cpp
-  restriction.cpp
  staticcond.cpp
  tmop.cpp
  tmop_tools.cpp
-  gslib.cpp
  )

 set(HDRS
@@ -53,7 +40,6 @@ set(HDRS
  bilinearform_ext.hpp
  bilininteg.hpp
  coefficient.hpp
-  complex_fem.hpp
  datacollection.hpp
  eltrans.hpp
  estimators.hpp
@@ -68,10 +54,7 @@ set(HDRS
  linearform.hpp
  lininteg.hpp
  nonlinearform.hpp
-  nonlinearform_ext.hpp
  nonlininteg.hpp
-  quadinterpolator.hpp
-  restriction.hpp
  staticcond.hpp
  tbilinearform.hpp
  tbilininteg.hpp
@@ -83,7 +66,6 @@ set(HDRS
  tintrules.hpp
  tmop.hpp
  tmop_tools.hpp
-  gslib.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -113,17 +95,6 @@ if (MFEM_USE_MPI)
    pnonlinearform.hpp)
 endif()

-if (MFEM_USE_CEED)
-   list(APPEND SRCS
-     libceed/ceed.cpp
-     libceed/diffusion.cpp
-     libceed/mass.cpp)
-   list(APPEND HDRS
-     libceed/ceed.hpp
-     libceed/diffusion.hpp
-     libceed/mass.hpp)
-endif()
-
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)

@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 // Implementation of class BilinearForm

@@ -55,7 +55,7 @@ void BilinearForm::AllocMat()

   int *I = dof_dof.GetI();
   int *J = dof_dof.GetJ();
-   double *data = Memory<double>(I[height]);
+   double *data = new double[I[height]];

   mat = new SparseMatrix(I, J, data, height, height, true, true, true);
   *mat = 0.0;
@@ -204,7 +204,7 @@ void BilinearForm::UseSparsity(SparseMatrix &A)
               << A.Height() << " x " << A.Width());
   MFEM_ASSERT(A.Finalized(), "matrix A must be Finalized");

-   UseSparsity(A.GetI(), A.GetJ(), A.ColumnsAreSorted());
+   UseSparsity(A.GetI(), A.GetJ(), A.areColumnsSorted());
 }

 double& BilinearForm::Elem (int i, int j)
@@ -224,13 +224,10 @@ MatrixInverse * BilinearForm::Inverse() const

 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::FULL)
-   {
-      if (!static_cond) { mat->Finalize(skip_zeros); }
-      if (mat_e) { mat_e->Finalize(skip_zeros); }
-      if (static_cond) { static_cond->Finalize(); }
-      if (hybridization) { hybridization->Finalize(); }
-   }
+   if (!static_cond) { mat->Finalize(skip_zeros); }
+   if (mat_e) { mat_e->Finalize(skip_zeros); }
+   if (static_cond) { static_cond->Finalize(); }
+   if (hybridization) { hybridization->Finalize(); }
 }

 void BilinearForm::AddDomainIntegrator(BilinearFormIntegrator *bfi)
@@ -611,41 +608,18 @@ void BilinearForm::ConformingAssemble()
   width = mat->Width();
 }

-void BilinearForm::AssembleDiagonal(Vector &diag) const
-{
-   if (ext)
-   {
-      MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      const Operator *P = fes->GetProlongationMatrix();
-      if (!IsIdentityProlongation(P))
-      {
-         Vector local_diag(P->Height());
-         ext->AssembleDiagonal(local_diag);
-         P->MultTranspose(local_diag, diag);
-      }
-      else
-      {
-         ext->AssembleDiagonal(diag);
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
-                 "matrix and use SparseMatrix::GetDiag?");
-   }
-}
-
 void BilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
                                    Vector &b, OperatorHandle &A, Vector &X,
                                    Vector &B, int copy_interior)
 {
+   const SparseMatrix *P = fes->GetConformingProlongation();
+
   if (ext)
   {
      ext->FormLinearSystem(ess_tdof_list, x, b, A, X, B, copy_interior);
      return;
   }
-   const SparseMatrix *P = fes->GetConformingProlongation();
+
   FormSystemMatrix(ess_tdof_list, A);

   // Transform the system and perform the elimination in B, based on the
@@ -986,18 +960,6 @@ void BilinearForm::EliminateVDofsInRHS(
   mat->PartMult(vdofs, x, b);
 }

-void BilinearForm::Mult(const Vector &x, Vector &y) const
-{
-   if (ext)
-   {
-      ext->Mult(x, y);
-   }
-   else
-   {
-      mat->Mult(x, y);
-   }
-}
-
 void BilinearForm::Update(FiniteElementSpace *nfes)
 {
   bool full_update;
@@ -1073,10 +1035,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   trial_fes = tr_fes;
   test_fes = te_fes;
   mat = NULL;
-   mat_e = NULL;
   extern_bfs = 0;
-   assembly = AssemblyLevel::FULL;
-   ext = NULL;
 }

 MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
@@ -1087,9 +1046,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   trial_fes = tr_fes;
   test_fes = te_fes;
   mat = NULL;
-   mat_e = NULL;
   extern_bfs = 1;
-   ext = NULL;

   // Copy the pointers to the integrators
   dbfi = mbf->dbfi;
@@ -1099,38 +1056,6 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,

   bbfi_marker = mbf->bbfi_marker;
   btfbfi_marker = mbf->btfbfi_marker;
-
-   assembly = AssemblyLevel::FULL;
-   ext = NULL;
-}
-
-void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
-{
-   if (ext)
-   {
-      MFEM_ABORT("the assembly level has already been set!");
-   }
-   assembly = assembly_level;
-   switch (assembly)
-   {
-      case AssemblyLevel::FULL:
-         // ext = new FAMixedBilinearFormExtension(this);
-         // Use the original BilinearForm implementation for now
-         break;
-      case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
-         // ext = new EAMixedBilinearFormExtension(this);
-         break;
-      case AssemblyLevel::PARTIAL:
-         ext = new PAMixedBilinearFormExtension(this);
-         break;
-      case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         // ext = new MFMixedBilinearFormExtension(this);
-         break;
-      default:
-         mfem_error("Unknown assembly level");
-   }
 }

 double & MixedBilinearForm::Elem (int i, int j)
@@ -1143,63 +1068,31 @@ const double & MixedBilinearForm::Elem (int i, int j) const
   return (*mat)(i, j);
 }

-void MixedBilinearForm::Mult(const Vector & x, Vector & y) const
+void MixedBilinearForm::Mult (const Vector & x, Vector & y) const
 {
-   y = 0.0;
-   AddMult(x, y);
+   mat -> Mult (x, y);
 }

-void MixedBilinearForm::AddMult(const Vector & x, Vector & y,
-                                const double a) const
+void MixedBilinearForm::AddMult (const Vector & x, Vector & y,
+                                 const double a) const
 {
-   if (ext)
-   {
-      ext->AddMult(x, y, a);
-   }
-   else
-   {
-      mat->AddMult(x, y, a);
-   }
+   mat -> AddMult (x, y, a);
 }

-void MixedBilinearForm::MultTranspose(const Vector & x, Vector & y) const
+void MixedBilinearForm::AddMultTranspose (const Vector & x, Vector & y,
+                                          const double a) const
 {
-   y = 0.0;
-   AddMultTranspose(x, y);
-}
-
-void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,
-                                         const double a) const
-{
-   if (ext)
-   {
-      ext->AddMultTranspose(x, y, a);
-   }
-   else
-   {
-      mat->AddMultTranspose(x, y, a);
-   }
+   mat -> AddMultTranspose (x, y, a);
 }

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::FULL)
-   {
-      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
-      return NULL;
-   }
-   else
-   {
-      return mat -> Inverse ();
-   }
+   return mat -> Inverse ();
 }

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::FULL)
-   {
-      mat -> Finalize (skip_zeros);
-   }
+   mat -> Finalize (skip_zeros);
 }

 void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
@@ -1252,12 +1145,6 @@ void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,

 void MixedBilinearForm::Assemble (int skip_zeros)
 {
-   if (ext)
-   {
-      ext->Assemble();
-      return;
-   }
-
   Array<int> tr_vdofs, te_vdofs;
   ElementTransformation *eltrans;
   DenseMatrix elemmat;
@@ -1425,12 +1312,6 @@ void MixedBilinearForm::Assemble (int skip_zeros)

 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::FULL)
-   {
-      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
-      return;
-   }
-
   Finalize();

   const SparseMatrix *P2 = test_fes->GetConformingProlongation();
@@ -1589,92 +1470,17 @@ void MixedBilinearForm::EliminateTestDofs (const Array<int> &bdr_attr_is_ess)
      }
 }

-void MixedBilinearForm::FormRectangularSystemMatrix(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    OperatorHandle &A)
-
-{
-   if (ext)
-   {
-      ext->FormRectangularSystemOperator(trial_tdof_list, test_tdof_list, A);
-      return;
-   }
-
-   const SparseMatrix *test_P = test_fes->GetConformingProlongation();
-   const SparseMatrix *trial_P = trial_fes->GetConformingProlongation();
-
-   mat->Finalize();
-
-   if (test_P) // TODO: Must actually check for trial_P too
-   {
-      SparseMatrix *m = RAP(*test_P, *mat, *trial_P);
-      delete mat;
-      mat = m;
-   }
-
-   Array<int> ess_trial_tdof_marker, ess_test_tdof_marker;
-   FiniteElementSpace::ListToMarker(trial_tdof_list, trial_fes->GetTrueVSize(),
-                                    ess_trial_tdof_marker);
-   FiniteElementSpace::ListToMarker(test_tdof_list, test_fes->GetTrueVSize(),
-                                    ess_test_tdof_marker);
-
-   mat_e = new SparseMatrix(mat->Height(), mat->Width());
-   mat->EliminateCols(ess_trial_tdof_marker, *mat_e);
-
-   for (int i=0; i<test_tdof_list.Size(); ++i)
-   {
-      mat->EliminateRow(test_tdof_list[i]);
-   }
-   mat_e->Finalize();
-   A.Reset(mat, false);
-}
-
-void MixedBilinearForm::FormRectangularLinearSystem(const Array<int>
-                                                    &trial_tdof_list,
-                                                    const Array<int> &test_tdof_list,
-                                                    Vector &x, Vector &b,
-                                                    OperatorHandle &A,
-                                                    Vector &X, Vector &B)
-{
-   if (ext)
-   {
-      ext->FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, A, X,
-                                       B);
-      return;
-   }
-
-   const Operator *Pi = this->GetProlongation();
-   const Operator *Po = this->GetOutputProlongation();
-   const Operator *Ri = this->GetRestriction();
-   InitTVectors(Po, Ri, Pi, x, b, X, B);
-
-   if (!mat_e)
-   {
-      FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list,
-                                  A); // Set A = mat_e
-   }
-   // Eliminate essential BCs with B -= Ab xb
-   mat_e->AddMult(X, B, -1.0);
-
-   B.SetSubVector(test_tdof_list, 0.0);
-}
-
 void MixedBilinearForm::Update()
 {
   delete mat;
   mat = NULL;
-   delete mat_e;
-   mat_e = NULL;
   height = test_fes->GetVSize();
   width = trial_fes->GetVSize();
-   if (ext) { ext->Update(); }
 }

 MixedBilinearForm::~MixedBilinearForm()
 {
   if (mat) { delete mat; }
-   if (mat_e) { delete mat_e; }
   if (!extern_bfs)
   {
      int i;
@@ -1683,7 +1489,6 @@ MixedBilinearForm::~MixedBilinearForm()
      for (i = 0; i < tfbfi.Size(); i++) { delete tfbfi[i]; }
      for (i = 0; i < btfbfi.Size(); i++) { delete btfbfi[i]; }
   }
-   delete ext;
 }


@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #ifndef MFEM_BILINEARFORM
 #define MFEM_BILINEARFORM
@@ -58,7 +58,7 @@ protected:
   /// FE space on which the form lives. Not owned.
   FiniteElementSpace *fes;

-   /// The assembly level of the form (full, partial, etc.)
+   /// The form assembly level (full, partial, etc.)
   AssemblyLevel assembly;
   /// Element batch size used in the form action (1, 8, num_elems, etc.)
   int batch;
@@ -153,9 +153,6 @@ public:
   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

-   /// Get the assembly level
-   AssemblyLevel GetAssemblyLevel() {return assembly;}
-
   /** Enable the use of static condensation. For details see the description
       for class StaticCondensation in fem/staticcond.hpp This method should be
       called before assembly. If the number of unknowns after static
@@ -230,7 +227,7 @@ public:
   virtual const double &Elem(int i, int j) const;

   /// Matrix vector multiplication.
-   virtual void Mult(const Vector &x, Vector &y) const;
+   virtual void Mult(const Vector &x, Vector &y) const { mat->Mult(x, y); }

   void FullMult(const Vector &x, Vector &y) const
   { mat->Mult(x, y); mat_e->AddMult(x, y); }
@@ -322,26 +319,12 @@ public:
   /// Assembles the form i.e. sums over all domain/bdr integrators.
   void Assemble(int skip_zeros = 1);

-   /** @brief Assemble the diagonal of the bilinear form into diag
-
-       For adaptively refined meshes, this returns P^T d_e, where d_e is the
-       locally assembled diagonal on each element and P^T is the transpose of
-       the conforming prolongation. In general this is not the correct diagonal
-       for an AMR mesh. */
-   void AssembleDiagonal(Vector &diag) const;
-
   /// Get the finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return fes->GetConformingProlongation(); }
   /// Get the finite element space restriction matrix
   virtual const Operator *GetRestriction() const
   { return fes->GetConformingRestriction(); }
-   /// Get the output finite element space prolongation matrix
-   virtual const Operator *GetOutputProlongation() const
-   { return GetProlongation(); }
-   /// Get the output finite element space restriction matrix
-   virtual const Operator *GetOutputRestriction() const
-   { return GetRestriction(); }

   /** @brief Form the linear system A X = B, corresponding to this bilinear
       form and the linear form @a b(.). */
@@ -542,9 +525,6 @@ public:
   /// Sets diagonal policy used upon construction of the linear system
   void SetDiagonalPolicy(DiagonalPolicy policy);

-   /// Indicate that integrators are not owned by the BilinearForm
-   void UseExternalIntegrators() { extern_bfs = 1; };
-
   /// Destroys bilinear form.
   virtual ~BilinearForm();
 };
@@ -569,18 +549,10 @@ class MixedBilinearForm : public Matrix
 {
 protected:
   SparseMatrix *mat; ///< Owned.
-   SparseMatrix *mat_e; ///< Owned.

   FiniteElementSpace *trial_fes, ///< Not owned
                      *test_fes;  ///< Not owned

-   /// The form assembly level (full, partial, etc.)
-   AssemblyLevel assembly;
-
-   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
-       Partial Assembly (PA), or Matrix Free assembly (MF). */
-   MixedBilinearFormExtension *ext;
-
   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
       #tfbfi and #btfbfi are owned by another MixedBilinearForm. */
   int extern_bfs;
@@ -635,13 +607,16 @@ public:
   virtual const double &Elem(int i, int j) const;

   virtual void Mult(const Vector & x, Vector & y) const;
+
   virtual void AddMult(const Vector & x, Vector & y,
                        const double a = 1.0) const;

-   virtual void MultTranspose(const Vector & x, Vector & y) const;
   virtual void AddMultTranspose(const Vector & x, Vector & y,
                                 const double a = 1.0) const;

+   virtual void MultTranspose(const Vector & x, Vector & y) const
+   { y = 0.0; AddMultTranspose (x, y); }
+
   virtual MatrixInverse *Inverse() const;

   virtual void Finalize(int skip_zeros = 1);
@@ -701,28 +676,8 @@ public:

   void operator=(const double a) { *mat = a; }

-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
-   /** This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level);
-
   void Assemble(int skip_zeros = 1);

-   /// Get the input finite element space prolongation matrix
-   virtual const Operator *GetProlongation() const
-   { return trial_fes->GetProlongationMatrix(); }
-
-   /// Get the input finite element space restriction matrix
-   virtual const Operator *GetRestriction() const
-   { return trial_fes->GetRestrictionMatrix(); }
-
-   /// Get the test finite element space prolongation matrix
-   virtual const Operator *GetOutputProlongation() const
-   { return test_fes->GetProlongationMatrix(); }
-
-   /// Get the test finite element space restriction matrix
-   virtual const Operator *GetOutputRestriction() const
-   { return test_fes->GetRestrictionMatrix(); }
-
   /** For partially conforming trial and/or test FE spaces, complete the
       assembly process by performing A := P2^t A P1 where A is the internal
       sparse matrix; P1 and P2 are the conforming prolongation matrices of the
@@ -784,81 +739,8 @@ public:

   virtual void EliminateTestDofs(const Array<int> &bdr_attr_is_ess);

-   /** @brief Return in @a A that is column-constrained.
-
-      This returns the same operator as FormRectangularLinearSystem(), but does
-      without the transformations of the right-hand side. */
-   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    OperatorHandle &A);
-
-   /** @brief Form the column-constrained linear system matrix A.
-       See FormRectangularSystemMatrix() for details.
-
-       Version of the method FormRectangularSystemMatrix() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
-   template <typename OpType>
-   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list, OpType &A)
-   {
-      OperatorHandle Ah;
-      FormRectangularSystemMatrix(trial_tdof_list, test_tdof_list, Ah);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   /** @brief Form the linear system A X = B, corresponding to this mixed bilinear
-       form and the linear form @a b(.).
-
-       Return in @a A a *reference* to the system matrix that is column-constrained.
-       The reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
-   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OperatorHandle &A, Vector &X, Vector &B);
-
-   /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.).
-
-       Version of the method FormRectangularLinearSystem() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called.
-
-       Currently, this method can be used only with AssemblyLevel::FULL. */
-   template <typename OpType>
-   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OpType &A, Vector &X, Vector &B)
-   {
-      OperatorHandle Ah;
-      FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b, Ah, X, B);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
   void Update();

-   /// Return the trial FE space associated with the BilinearForm.
-   FiniteElementSpace *TrialFESpace() { return trial_fes; }
-   /// Read-only access to the associated trial FiniteElementSpace.
-   const FiniteElementSpace *TrialFESpace() const { return trial_fes; }
-
-   /// Return the test FE space associated with the BilinearForm.
-   FiniteElementSpace *TestFESpace() { return test_fes; }
-   /// Read-only access to the associated test FiniteElementSpace.
-   const FiniteElementSpace *TestFESpace() const { return test_fes; }
-
   virtual ~MixedBilinearForm();
 };

@@ -1,20 +1,19 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 // Implementations of classes FABilinearFormExtension, EABilinearFormExtension,
 // PABilinearFormExtension and MFBilinearFormExtension.

 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
-#include "libceed/ceed.hpp"

 namespace mfem
 {
@@ -39,106 +38,26 @@ const Operator *BilinearFormExtension::GetRestriction() const
 // Data and methods for partially-assembled bilinear forms
 PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
   : BilinearFormExtension(form),
-     trialFes(a->FESpace()),
-     testFes(a->FESpace())
+     trialFes(a->FESpace()), testFes(a->FESpace())
 {
-   elem_restrict = NULL;
-   int_face_restrict_lex = NULL;
-   bdr_face_restrict_lex = NULL;
-}
-
-void PABilinearFormExtension::SetupRestrictionOperators()
-{
-   ElementDofOrdering ordering = UsesTensorBasis(*a->FESpace())?
-                                 ElementDofOrdering::LEXICOGRAPHIC:
-                                 ElementDofOrdering::NATIVE;
-   elem_restrict = trialFes->GetElementRestriction(ordering);
-   if (elem_restrict)
+   elem_restrict_lex = trialFes->GetElementRestriction(
+                          ElementDofOrdering::LEXICOGRAPHIC);
+   if (elem_restrict_lex)
   {
-      localX.SetSize(elem_restrict->Height(), Device::GetDeviceMemoryType());
-      localY.SetSize(elem_restrict->Height(), Device::GetDeviceMemoryType());
+      localX.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
+      localY.SetSize(elem_restrict_lex->Height(), Device::GetMemoryType());
      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
   }
-
-   // Construct face restriction operators only if the bilinear form has
-   // interior or boundary face integrators
-   if (int_face_restrict_lex == NULL && a->GetFBFI()->Size() > 0)
-   {
-      int_face_restrict_lex = trialFes->GetFaceRestriction(
-                                 ElementDofOrdering::LEXICOGRAPHIC, FaceType::Interior);
-      faceIntX.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
-      faceIntY.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
-      faceIntY.UseDevice(true); // ensure 'faceIntY = 0.0' is done on device
-   }
-
-   if (bdr_face_restrict_lex == NULL && a->GetBFBFI()->Size() > 0)
-   {
-      bdr_face_restrict_lex = trialFes->GetFaceRestriction(
-                                 ElementDofOrdering::LEXICOGRAPHIC, FaceType::Boundary);
-      faceBdrX.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
-      faceBdrY.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
-      faceBdrY.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device
-   }
 }

 void PABilinearFormExtension::Assemble()
 {
-   SetupRestrictionOperators();
-
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
   const int integratorCount = integrators.Size();
   for (int i = 0; i < integratorCount; ++i)
   {
      integrators[i]->AssemblePA(*a->FESpace());
   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int intFaceIntegratorCount = intFaceIntegrators.Size();
-   for (int i = 0; i < intFaceIntegratorCount; ++i)
-   {
-      intFaceIntegrators[i]->AssemblePAInteriorFaces(*a->FESpace());
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int boundFaceIntegratorCount = bdrFaceIntegrators.Size();
-   for (int i = 0; i < boundFaceIntegratorCount; ++i)
-   {
-      bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
-   }
-}
-
-void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (elem_restrict)
-   {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalPA(localY);
-      }
-      const ElementRestriction* H1elem_restrict =
-         dynamic_cast<const ElementRestriction*>(elem_restrict);
-      if (H1elem_restrict)
-      {
-         H1elem_restrict->MultTransposeUnsigned(localY, y);
-      }
-      else
-      {
-         elem_restrict->MultTranspose(localY, y);
-      }
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalPA(y);
-      }
-   }
 }

 void PABilinearFormExtension::Update()
@@ -147,18 +66,24 @@ void PABilinearFormExtension::Update()
   height = width = fes->GetVSize();
   trialFes = fes;
   testFes = fes;
-
-   elem_restrict = nullptr;
-   int_face_restrict_lex = nullptr;
-   bdr_face_restrict_lex = nullptr;
+   elem_restrict_lex = trialFes->GetElementRestriction(
+                          ElementDofOrdering::LEXICOGRAPHIC);
+   if (elem_restrict_lex)
+   {
+      localX.SetSize(elem_restrict_lex->Height());
+      localY.SetSize(elem_restrict_lex->Height());
+   }
 }

 void PABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
                                               OperatorHandle &A)
 {
-   Operator *oper;
-   Operator::FormSystemOperator(ess_tdof_list, oper);
-   A.Reset(oper); // A will own oper
+   const Operator* trialP = trialFes->GetProlongationMatrix();
+   const Operator* testP  = testFes->GetProlongationMatrix();
+   Operator *rap = this;
+   if (trialP) { rap = new RAPOperator(*testP, *this, *trialP); }
+   const bool own_A = (rap!=this);
+   A.Reset(new ConstrainedOperator(rap, ess_tdof_list, own_A));
 }

 void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
@@ -177,7 +102,17 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
+   if (elem_restrict_lex)
+   {
+      elem_restrict_lex->Mult(x, localX);
+      localY = 0.0;
+      for (int i = 0; i < iSz; ++i)
+      {
+         integrators[i]->AddMultPA(localX, localY);
+      }
+      elem_restrict_lex->MultTranspose(localY, y);
+   }
+   else
   {
      y.UseDevice(true); // typically this is a large vector, so store on device
      y = 0.0;
@@ -186,63 +121,21 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         integrators[i]->AddMultPA(x, y);
      }
   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultPA(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultPA(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultPA(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
 }

 void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
   const int iSz = integrators.Size();
-   if (elem_restrict)
+   if (elem_restrict_lex)
   {
-      elem_restrict->Mult(x, localX);
+      elem_restrict_lex->Mult(x, localX);
      localY = 0.0;
      for (int i = 0; i < iSz; ++i)
      {
         integrators[i]->AddMultTransposePA(localX, localY);
      }
-      elem_restrict->MultTranspose(localY, y);
+      elem_restrict_lex->MultTranspose(localY, y);
   }
   else
   {
@@ -253,238 +146,6 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         integrators[i]->AddMultTransposePA(x, y);
      }
   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultTransposePA(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultTransposePA(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
-   : Operator(form->Height(), form->Width()), a(form)
-{
-   // empty
-}
-
-const Operator *MixedBilinearFormExtension::GetProlongation() const
-{
-   return a->GetProlongation();
-}
-
-const Operator *MixedBilinearFormExtension::GetRestriction() const
-{
-   return a->GetRestriction();
-}
-
-const Operator *MixedBilinearFormExtension::GetOutputProlongation() const
-{
-   return a->GetOutputProlongation();
-}
-
-const Operator *MixedBilinearFormExtension::GetOutputRestriction() const
-{
-   return a->GetOutputRestriction();
-}
-
-// Data and methods for partially-assembled bilinear forms
-
-PAMixedBilinearFormExtension::PAMixedBilinearFormExtension(
-   MixedBilinearForm *form)
-   : MixedBilinearFormExtension(form),
-     trialFes(form->TrialFESpace()),
-     testFes(form->TestFESpace()),
-     elem_restrict_trial(NULL),
-     elem_restrict_test(NULL)
-{
-   Update();
-}
-
-void PAMixedBilinearFormExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssemblePA(*trialFes, *testFes);
-   }
-}
-
-void PAMixedBilinearFormExtension::Update()
-{
-   trialFes = a->TrialFESpace();
-   testFes  = a->TestFESpace();
-   height = testFes->GetVSize();
-   width = trialFes->GetVSize();
-   elem_restrict_trial = trialFes->GetElementRestriction(
-                            ElementDofOrdering::LEXICOGRAPHIC);
-   elem_restrict_test  =  testFes->GetElementRestriction(
-                             ElementDofOrdering::LEXICOGRAPHIC);
-   if (elem_restrict_trial)
-   {
-      localTrial.UseDevice(true);
-      localTrial.SetSize(elem_restrict_trial->Height(),
-                         Device::GetMemoryType());
-
-   }
-   if (elem_restrict_test)
-   {
-      localTest.UseDevice(true); // ensure 'localY = 0.0' is done on device
-      localTest.SetSize(elem_restrict_test->Height(), Device::GetMemoryType());
-   }
-}
-
-void PAMixedBilinearFormExtension::FormRectangularSystemOperator(
-   const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list,
-   OperatorHandle &A)
-{
-   Operator * oper;
-   Operator::FormRectangularSystemOperator(trial_tdof_list, test_tdof_list,
-                                           oper);
-   A.Reset(oper); // A will own oper
-}
-
-void PAMixedBilinearFormExtension::FormRectangularLinearSystem(
-   const Array<int> &trial_tdof_list,
-   const Array<int> &test_tdof_list,
-   Vector &x, Vector &b,
-   OperatorHandle &A,
-   Vector &X, Vector &B)
-{
-   Operator *oper;
-   Operator::FormRectangularLinearSystem(trial_tdof_list, test_tdof_list, x, b,
-                                         oper, X, B);
-   A.Reset(oper); // A will own oper
-}
-
-void PAMixedBilinearFormExtension::SetupMultInputs(
-   const Operator *elem_restrict_x,
-   const Vector &x,
-   Vector &localX,
-   const Operator *elem_restrict_y,
-   Vector &y,
-   Vector &localY,
-   const double c) const
-{
-   // * G operation: localX = c*local(x)
-   if (elem_restrict_x)
-   {
-      elem_restrict_x->Mult(x, localX);
-      if (c != 1.0)
-      {
-         localX *= c;
-      }
-   }
-   else
-   {
-      if (c == 1.0)
-      {
-         localX.SyncAliasMemory(x);
-      }
-      else
-      {
-         localX.Set(c, x);
-      }
-   }
-   if (elem_restrict_y)
-   {
-      localY = 0.0;
-   }
-   else
-   {
-      y.UseDevice(true);
-      localY.SyncAliasMemory(y);
-   }
-}
-
-void PAMixedBilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   y = 0.0;
-   AddMult(x, y);
-}
-
-void PAMixedBilinearFormExtension::AddMult(const Vector &x, Vector &y,
-                                           const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // * G operation
-   SetupMultInputs(elem_restrict_trial, x, localTrial,
-                   elem_restrict_test, y, localTest, c);
-
-   // * B^TDB operation
-   for (int i = 0; i < iSz; ++i)
-   {
-      integrators[i]->AddMultPA(localTrial, localTest);
-   }
-
-   // * G^T operation
-   if (elem_restrict_test)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict_test->MultTranspose(localTest, tempY);
-      y += tempY;
-   }
-}
-
-void PAMixedBilinearFormExtension::MultTranspose(const Vector &x,
-                                                 Vector &y) const
-{
-   y = 0.0;
-   AddMultTranspose(x, y);
-}
-
-void PAMixedBilinearFormExtension::AddMultTranspose(const Vector &x, Vector &y,
-                                                    const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // * G operation
-   SetupMultInputs(elem_restrict_test, x, localTest,
-                   elem_restrict_trial, y, localTrial, c);
-
-   // * B^TD^TB operation
-   for (int i = 0; i < iSz; ++i)
-   {
-      integrators[i]->AddMultTransposePA(localTest, localTrial);
-   }
-
-   // * G^T operation
-   if (elem_restrict_trial)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict_trial->MultTranspose(localTrial, tempY);
-      y += tempY;
-   }
 }

 } // namespace mfem
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #ifndef MFEM_BILINEARFORM_EXT
 #define MFEM_BILINEARFORM_EXT
@@ -20,7 +20,6 @@ namespace mfem
 {

 class BilinearForm;
-class MixedBilinearForm;


 /** @brief Class extending the BilinearForm class to support the different
@@ -34,7 +33,7 @@ public:
   BilinearFormExtension(BilinearForm *form);

   virtual MemoryClass GetMemoryClass() const
-   { return Device::GetDeviceMemoryClass(); }
+   { return Device::GetMemoryClass(); }

   /// Get the finite element space prolongation matrix
   virtual const Operator *GetProlongation() const;
@@ -43,12 +42,6 @@ public:
   virtual const Operator *GetRestriction() const;

   virtual void Assemble() = 0;
-
-   virtual void AssembleDiagonal(Vector &diag) const
-   {
-      MFEM_ABORT("AssembleDiagonal not implemented for this assembly level!");
-   }
-
   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
                                 OperatorHandle &A) = 0;
   virtual void FormLinearSystem(const Array<int> &ess_tdof_list,
@@ -104,18 +97,12 @@ class PABilinearFormExtension : public BilinearFormExtension
 protected:
   const FiniteElementSpace *trialFes, *testFes; // Not owned
   mutable Vector localX, localY;
-   mutable Vector faceIntX, faceIntY;
-   mutable Vector faceBdrX, faceBdrY;
-   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
+   const Operator *elem_restrict_lex; // Not owned

 public:
   PABilinearFormExtension(BilinearForm*);

-   void SetupRestrictionOperators();
   void Assemble();
-   void AssembleDiagonal(Vector &diag) const;
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
   void FormLinearSystem(const Array<int> &ess_tdof_list,
                         Vector &x, Vector &b,
@@ -127,7 +114,6 @@ public:
   void Update();
 };

-
 /// Data and methods for matrix-free bilinear forms
 class MFBilinearFormExtension : public BilinearFormExtension
 {
@@ -148,98 +134,6 @@ public:
   ~MFBilinearFormExtension() {}
 };

-/** @brief Class extending the MixedBilinearForm class to support the different
-    AssemblyLevel%s. */
-class MixedBilinearFormExtension : public Operator
-{
-protected:
-   MixedBilinearForm *a; ///< Not owned
-
-public:
-   MixedBilinearFormExtension(MixedBilinearForm *form);
-
-   virtual MemoryClass GetMemoryClass() const
-   { return Device::GetMemoryClass(); }
-
-   /// Get the finite element space prolongation matrix
-   virtual const Operator *GetProlongation() const;
-
-   /// Get the finite element space restriction matrix
-   virtual const Operator *GetRestriction() const;
-
-   /// Get the output finite element space restriction matrix
-   virtual const Operator *GetOutputProlongation() const;
-
-   /// Get the output finite element space restriction matrix
-   virtual const Operator *GetOutputRestriction() const;
-
-   virtual void Assemble() = 0;
-   virtual void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
-                                              const Array<int> &test_tdof_list,
-                                              OperatorHandle &A) = 0;
-   virtual void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                            const Array<int> &test_tdof_list,
-                                            Vector &x, Vector &b,
-                                            OperatorHandle &A, Vector &X, Vector &B) = 0;
-
-   virtual void AddMult(const Vector &x, Vector &y, const double c=1.0) const = 0;
-   virtual void AddMultTranspose(const Vector &x, Vector &y,
-                                 const double c=1.0) const = 0;
-
-   virtual void Update() = 0;
-};
-
-/// Data and methods for partially-assembled mixed bilinear forms
-class PAMixedBilinearFormExtension : public MixedBilinearFormExtension
-{
-protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
-   mutable Vector localTrial, localTest, tempY;
-   const Operator *elem_restrict_trial; // Not owned
-   const Operator *elem_restrict_test;  // Not owned
-private:
-   /// Helper function to set up inputs/outputs for Mult or MultTranspose
-   void SetupMultInputs(const Operator *elem_restrict_x,
-                        const Vector &x, Vector &localX,
-                        const Operator *elem_restrict_y,
-                        Vector &y, Vector &localY, const double c) const;
-
-public:
-   PAMixedBilinearFormExtension(MixedBilinearForm *form);
-
-   /// Partial assembly of all internal integrators
-   void Assemble();
-   /**
-      @brief Setup OperatorHandle A to contain constrained linear operator
-
-      OperatorHandle A contains matrix-free constrained operator formed for RAP
-      system where ess_tdof_list are in trial space and eliminated from
-      "columns" of A.
-   */
-   void FormRectangularSystemOperator(const Array<int> &trial_tdof_list,
-                                      const Array<int> &test_tdof_list,
-                                      OperatorHandle &A);
-   /**
-      Setup OperatorHandle A to contain constrained linear operator and
-      eliminate columns corresponding to essential dofs from system,
-      updating RHS B vector with the results.
-   */
-   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
-                                    const Array<int> &test_tdof_list,
-                                    Vector &x, Vector &b,
-                                    OperatorHandle &A, Vector &X, Vector &B);
-   /// y = A*x
-   void Mult(const Vector &x, Vector &y) const;
-   /// y += c*A*x
-   void AddMult(const Vector &x, Vector &y, const double c=1.0) const;
-   /// y = A^T*x
-   void MultTranspose(const Vector &x, Vector &y) const;
-   /// y += c*A^T*x
-   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
-   /// Update internals for when a new MixedBilinearForm is given to this class
-   void Update();
-};
-
 }

 #endif
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 // Implementation of Bilinear Form Integrators

@@ -22,44 +22,19 @@ namespace mfem

 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::Assemble (...)\n"
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
-                                        const FiniteElementSpace&)
-{
-   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssemblePAInteriorFaces(const FiniteElementSpace&)
-{
-   mfem_error ("BilinearFormIntegrator::AssemblePAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace&)
-{
-   mfem_error ("BilinearFormIntegrator::AssemblePABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
-{
-   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
-              "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembled(...)\n"
+   mfem_error ("BilinearFormIntegrator::MultAssembled (...)\n"
               "   is not implemented for this class.");
 }

 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose(...)\n"
+   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose (...)\n"
               "   is not implemented for this class.");
 }

@@ -67,7 +42,7 @@ void BilinearFormIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
 {
-   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix (...)\n"
               "   is not implemented for this class.");
 }

@@ -75,7 +50,7 @@ void BilinearFormIntegrator::AssembleElementMatrix2 (
   const FiniteElement &el1, const FiniteElement &el2,
   ElementTransformation &Trans, DenseMatrix &elmat )
 {
-   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssembleElementMatrix2 (...)\n"
               "   is not implemented for this class.");
 }

@@ -83,7 +58,7 @@ void BilinearFormIntegrator::AssembleFaceMatrix (
   const FiniteElement &el1, const FiniteElement &el2,
   FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleFaceMatrix(...)\n"
+   mfem_error ("BilinearFormIntegrator::AssembleFaceMatrix (...)\n"
               "   is not implemented for this class.");
 }

@@ -405,72 +380,6 @@ void MixedScalarVectorIntegrator::AssembleElementMatrix2(
 }


-void GradientIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe,
-   ElementTransformation &Trans,  DenseMatrix &elmat)
-{
-   int dim = test_fe.GetDim();
-   int trial_dof = trial_fe.GetDof();
-   int test_dof = test_fe.GetDof();
-   double c;
-   Vector d_col;
-
-   dshape.SetSize(trial_dof, dim);
-   gshape.SetSize(trial_dof, dim);
-   Jadj.SetSize(dim);
-   shape.SetSize(test_dof);
-   elmat.SetSize(dim * test_dof, trial_dof);
-
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
-                                                            Trans);
-
-   elmat = 0.0;
-   elmat_comp.SetSize(test_dof, trial_dof);
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      trial_fe.CalcDShape(ip, dshape);
-      test_fe.CalcShape(ip, shape);
-
-      Trans.SetIntPoint(&ip);
-      CalcAdjugate(Trans.Jacobian(), Jadj);
-
-      Mult(dshape, Jadj, gshape);
-
-      c = ip.weight;
-      if (Q)
-      {
-         c *= Q->Eval(Trans, ip);
-      }
-      shape *= c;
-
-      for (int d = 0; d < dim; ++d)
-      {
-         gshape.GetColumnReference(d, d_col);
-         MultVWt(shape, d_col, elmat_comp);
-         for (int jj = 0; jj < trial_dof; ++jj)
-         {
-            for (int ii = 0; ii < test_dof; ++ii)
-            {
-               elmat(d * test_dof + ii, jj) += elmat_comp(ii, jj);
-            }
-         }
-      }
-   }
-}
-
-const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
-                                                   &trial_fe,
-                                                   const FiniteElement &test_fe,
-                                                   ElementTransformation &Trans)
-{
-   int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder() + Trans.OrderJ();
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
  DenseMatrix &elmat )
@@ -638,7 +547,7 @@ void DiffusionIntegrator::AssembleElementVector(

 void DiffusionIntegrator::ComputeElementFlux
 ( const FiniteElement &el, ElementTransformation &Trans,
-  Vector &u, const FiniteElement &fluxelem, Vector &flux, bool with_coef )
+  Vector &u, const FiniteElement &fluxelem, Vector &flux, int with_coef )
 {
   int i, j, nd, dim, spaceDim, fnd;

@@ -822,6 +731,7 @@ void MassIntegrator::AssembleElementMatrix2(
 {
   int tr_nd = trial_fe.GetDof();
   int te_nd = test_fe.GetDof();
+   // int dim = trial_fe.GetDim();
   double w;

 #ifdef MFEM_THREAD_SAFE
@@ -911,6 +821,7 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
   }
 }

+
 void ConvectionIntegrator::AssembleElementMatrix(
   const FiniteElement &el, ElementTransformation &Trans, DenseMatrix &elmat)
 {
@@ -1011,21 +922,6 @@ void GroupConvectionIntegrator::AssembleElementMatrix(
   }
 }

-const IntegrationRule &ConvectionIntegrator::GetRule(const FiniteElement
-                                                     &trial_fe,
-                                                     const FiniteElement &test_fe,
-                                                     ElementTransformation &Trans)
-{
-   int order = Trans.OrderGrad(&trial_fe) + Trans.Order() + test_fe.GetOrder();
-
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-
-const IntegrationRule &ConvectionIntegrator::GetRule(
-   const FiniteElement &el, ElementTransformation &Trans)
-{
-   return GetRule(el,el,Trans);
-}

 void VectorMassIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
@@ -1554,7 +1450,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
 void CurlCurlIntegrator
 ::ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans,
                     Vector &u, const FiniteElement &fluxelem, Vector &flux,
-                     bool with_coef)
+                     int with_coef)
 {
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix projcurl;
@@ -1856,29 +1752,29 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
   const FiniteElement &trial_fe, const FiniteElement &test_fe,
   ElementTransformation &Trans, DenseMatrix &elmat)
 {
-   if (test_fe.GetRangeType() == FiniteElement::SCALAR
-       && trial_fe.GetRangeType() == FiniteElement::VECTOR)
+   if ( test_fe.GetRangeType() == FiniteElement::SCALAR && VQ )
   {
      // assume test_fe is scalar FE and trial_fe is vector FE
-      int spaceDim = Trans.GetSpaceDim();
-      int vdim = MQ ? MQ->GetHeight() : spaceDim;
+      int dim  = test_fe.GetDim();
      int trial_dof = trial_fe.GetDof();
      int test_dof = test_fe.GetDof();
      double w;

+      if (MQ)
+         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                    "   is not implemented for tensor materials");
+
 #ifdef MFEM_THREAD_SAFE
-      DenseMatrix trial_vshape(trial_dof, spaceDim);
+      DenseMatrix trial_vshape(trial_dof, dim);
      Vector shape(test_dof);
-      Vector D(VQ ? VQ->GetVDim() : 0);
-      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
+      Vector D(dim);
 #else
-      trial_vshape.SetSize(trial_dof, spaceDim);
+      trial_vshape.SetSize(trial_dof, dim);
      shape.SetSize(test_dof);
-      D.SetSize(VQ ? VQ->GetVDim() : 0);
-      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
+      D.SetSize(dim);
 #endif

-      elmat.SetSize(vdim*test_dof, trial_dof);
+      elmat.SetSize (test_dof, trial_dof);

      const IntegrationRule *ir = IntRule;
      if (ir == NULL)
@@ -1898,83 +1794,97 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
         test_fe.CalcShape(ip, shape);

         w = ip.weight * Trans.Weight();
-         if (VQ)
+         VQ->Eval(D, Trans, ip);
+         D *= w;
+
+         for (int d = 0; d < dim; d++)
         {
-            VQ->Eval(D, Trans, ip);
-            D *= w;
-            for (int d = 0; d < vdim; d++)
+            for (int j = 0; j < test_dof; j++)
            {
-               for (int j = 0; j < test_dof; j++)
+               for (int k = 0; k < trial_dof; k++)
               {
-                  for (int k = 0; k < trial_dof; k++)
-                  {
-                     elmat(d * test_dof + j, k) +=
-                        shape(j) * D(d) * trial_vshape(k, d);
-                  }
-               }
-            }
-         }
-         else if (MQ)
-         {
-            MQ->Eval(K, Trans, ip);
-            K *= w;
-            for (int d = 0; d < vdim; d++)
-            {
-               for (int j = 0; j < test_dof; j++)
-               {
-                  for (int k = 0; k < trial_dof; k++)
-                  {
-                     double Kv = 0.0;
-                     for (int vd = 0; vd < spaceDim; vd++)
-                     {
-                        Kv += K(d, vd) * trial_vshape(k, vd);
-                     }
-                     elmat(d * test_dof + j, k) += shape(j) * Kv;
-                  }
-               }
-            }
-         }
-         else
-         {
-            if (Q)
-            {
-               w *= Q->Eval(Trans, ip);
-            }
-            for (int d = 0; d < vdim; d++)
-            {
-               for (int j = 0; j < test_dof; j++)
-               {
-                  for (int k = 0; k < trial_dof; k++)
-                  {
-                     elmat(d * test_dof + j, k) +=
-                        w * shape(j) * trial_vshape(k, d);
-                  }
+                  elmat(j, k) += D[d] * shape(j) * trial_vshape(k, d);
               }
            }
         }
      }
   }
-   else if (test_fe.GetRangeType() == FiniteElement::VECTOR
-            && trial_fe.GetRangeType() == FiniteElement::VECTOR)
+   else if ( test_fe.GetRangeType() == FiniteElement::SCALAR )
   {
-      // assume both test_fe and trial_fe are vector FE
-      int spaceDim = Trans.GetSpaceDim();
+      // assume test_fe is scalar FE and trial_fe is vector FE
+      int dim  = test_fe.GetDim();
      int trial_dof = trial_fe.GetDof();
      int test_dof = test_fe.GetDof();
      double w;

+      if (VQ || MQ)
+         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                    "   is not implemented for vector/tensor permeability");
+
 #ifdef MFEM_THREAD_SAFE
-      DenseMatrix trial_vshape(trial_dof,spaceDim);
-      DenseMatrix test_vshape(test_dof,spaceDim);
-      Vector D(VQ ? VQ->GetVDim() : 0);
-      DenseMatrix K(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
+      DenseMatrix trial_vshape(trial_dof, dim);
+      Vector shape(test_dof);
 #else
-      trial_vshape.SetSize(trial_dof,spaceDim);
-      test_vshape.SetSize(test_dof,spaceDim);
-      D.SetSize(VQ ? VQ->GetVDim() : 0);
-      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
+      trial_vshape.SetSize(trial_dof, dim);
+      shape.SetSize(test_dof);
+#endif
+
+      elmat.SetSize (dim*test_dof, trial_dof);
+
+      const IntegrationRule *ir = IntRule;
+      if (ir == NULL)
+      {
+         int order = (Trans.OrderW() + test_fe.GetOrder() + trial_fe.GetOrder());
+         ir = &IntRules.Get(test_fe.GetGeomType(), order);
+      }
+
+      elmat = 0.0;
+      for (int i = 0; i < ir->GetNPoints(); i++)
+      {
+         const IntegrationPoint &ip = ir->IntPoint(i);
+
+         Trans.SetIntPoint (&ip);
+
+         trial_fe.CalcVShape(Trans, trial_vshape);
+         test_fe.CalcShape(ip, shape);
+
+         w = ip.weight * Trans.Weight();
+         if (Q)
+         {
+            w *= Q -> Eval (Trans, ip);
+         }
+
+         for (int d = 0; d < dim; d++)
+         {
+            for (int j = 0; j < test_dof; j++)
+            {
+               for (int k = 0; k < trial_dof; k++)
+               {
+                  elmat(d * test_dof + j, k) += w * shape(j) * trial_vshape(k, d);
+               }
+            }
+         }
+      }
+   }
+   else
+   {
+      // assume both test_fe and trial_fe are vector FE
+      int dim  = test_fe.GetDim();
+      int trial_dof = trial_fe.GetDof();
+      int test_dof = test_fe.GetDof();
+      double w;
+
+      if (VQ || MQ)
+         mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
+                    "   is not implemented for vector/tensor permeability");
+
+#ifdef MFEM_THREAD_SAFE
+      DenseMatrix trial_vshape(trial_dof, dim);
+      DenseMatrix test_vshape(test_dof,dim);
+#else
+      trial_vshape.SetSize(trial_dof, dim);
+      test_vshape.SetSize(test_dof,dim);
 #endif
-      DenseMatrix tmp(trial_vshape.Height(), K.Width());

      elmat.SetSize (test_dof, trial_dof);

@@ -1996,34 +1906,23 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
         test_fe.CalcVShape(Trans, test_vshape);

         w = ip.weight * Trans.Weight();
-         if (MQ)
+         if (Q)
         {
-            MQ->Eval(K, Trans, ip);
-            K *= w;
-            Mult(test_vshape,K,tmp);
-            AddMultABt(tmp,trial_vshape,elmat);
+            w *= Q -> Eval (Trans, ip);
         }
-         else if (VQ)
+
+         for (int d = 0; d < dim; d++)
         {
-            VQ->Eval(D, Trans, ip);
-            D *= w;
-            AddMultADBt(test_vshape,D,trial_vshape,elmat);
-         }
-         else
-         {
-            if (Q)
+            for (int j = 0; j < test_dof; j++)
            {
-               w *= Q -> Eval (Trans, ip);
+               for (int k = 0; k < trial_dof; k++)
+               {
+                  elmat(j, k) += w * test_vshape(j, d) * trial_vshape(k, d);
+               }
            }
-            AddMult_a_ABt(w,test_vshape,trial_vshape,elmat);
         }
      }
   }
-   else
-   {
-      mfem_error("VectorFEMassIntegrator::AssembleElementMatrix2(...)\n"
-                 "   is not implemented for given trial and test bases.");
-   }
 }

 void VectorDivergenceIntegrator::AssembleElementMatrix2(
@@ -2045,8 +1944,12 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(

   elmat.SetSize (test_dof, dim*trial_dof);

-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(trial_fe, test_fe,
-                                                            Trans);
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }

   elmat = 0.0;

@@ -2076,15 +1979,6 @@ void VectorDivergenceIntegrator::AssembleElementMatrix2(
   }
 }

-const IntegrationRule &VectorDivergenceIntegrator::GetRule(
-   const FiniteElement &trial_fe,
-   const FiniteElement &test_fe,
-   ElementTransformation &Trans)
-{
-   int order = Trans.OrderGrad(&trial_fe) + test_fe.GetOrder() + Trans.OrderJ();
-   return IntRules.Get(trial_fe.GetGeomType(), order);
-}
-

 void DivDivIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -2332,7 +2226,7 @@ void ElasticityIntegrator::AssembleElementMatrix(
 void ElasticityIntegrator::ComputeElementFlux(
   const mfem::FiniteElement &el, ElementTransformation &Trans,
   Vector &u, const mfem::FiniteElement &fluxelem, Vector &flux,
-   bool with_coef)
+   int with_coef)
 {
   const int dof = el.GetDof();
   const int dim = el.GetDim();
@@ -2632,14 +2526,6 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
   }
 }

-
-const IntegrationRule &DGTraceIntegrator::GetRule(
-   Geometry::Type geom, int order, FaceElementTransformations &T)
-{
-   int int_order = T.Elem1->OrderW() + 2*order;
-   return IntRules.Get(geom, int_order);
-}
-
 void DGDiffusionIntegrator::AssembleFaceMatrix(
   const FiniteElement &el1, const FiniteElement &el2,
   FaceElementTransformations &Trans, DenseMatrix &elmat)
@@ -1,13 +1,13 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
+// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
+// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
+// reserved. See file COPYRIGHT for details.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
+// availability see http://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
+// terms of the GNU Lesser General Public License (as published by the Free
+// Software Foundation) version 2.1 dated February 1999.

 #ifndef MFEM_BILININTEG
 #define MFEM_BILININTEG
@@ -15,7 +15,6 @@
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
 #include "fespace.hpp"
-#include "libceed/ceed.hpp"

 namespace mfem
 {
@@ -40,22 +39,10 @@ public:
   // make sense for the action of the nonlinear operator (but they all make
   // sense for its Jacobian).

-   using NonlinearFormIntegrator::AssemblePA;
-
   /// Method defining partial assembly.
   /** The result of the partial assembly is stored internally so that it can be
       used later in the methods AddMultPA() and AddMultTransposePA(). */
   virtual void AssemblePA(const FiniteElementSpace &fes);
-   /** Used with BilinearFormIntegrators that have different spaces. */
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
-
-   virtual void AssemblePABoundaryFaces(const FiniteElementSpace &fes);
-
-   /// Assemble diagonal and add it to Vector @a diag.
-   virtual void AssembleDiagonalPA(Vector &diag);

   /// Method for partially assembled action.
   /** Perform the action of integrator on the input @a x and add the result to
@@ -151,7 +138,7 @@ public:
                                   ElementTransformation &Trans,
                                   Vector &u,
                                   const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true) { }
+                                   Vector &flux, int with_coef = 1) { }

   /** @brief Virtual method required for Zienkiewicz-Zhu type error estimators.

@@ -207,33 +194,6 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace& fes)
-   {
-      bfi->AssemblePA(fes);
-   }
-
-   virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes)
-   {
-      bfi->AssemblePAInteriorFaces(fes);
-   }
-
-   virtual void AssemblePABoundaryFaces(const FiniteElementSpace &fes)
-   {
-      bfi->AssemblePABoundaryFaces(fes);
-   }
-
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const
-   {
-      bfi->AddMultPA(x, y);
-   }
-
-   virtual void AddMultPA(const Vector& x, Vector& y) const
-   {
-      bfi->AddMultTransposePA(x, y);
-   }
-
   virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };

@@ -1577,21 +1537,8 @@ protected:
      trial_fe.CalcPhysDShape(Trans, shape);
   }

-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
 private:
   DenseMatrix Jinv;
-
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, quad1D;
 };

 /** Class for integrating the bilinear form a(u,v) := (Q curl u, v) in 3D and
@@ -1711,55 +1658,6 @@ protected:
   }
 };

-/** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same space
-    as u. */
-class GradientIntegrator : public BilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-
-private:
-   Vector shape;
-   DenseMatrix dshape;
-   DenseMatrix gshape;
-   DenseMatrix Jadj;
-   DenseMatrix elmat_comp;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *trial_maps, *test_maps; ///< Not owned
-   const GeometricFactors *geom;            ///< Not owned
-   int dim, ne, nq;
-   int trial_dofs1D, test_dofs1D, quad1D;
-
-public:
-   GradientIntegrator() :
-      Q{NULL}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
-   { }
-   GradientIntegrator(Coefficient *_q) :
-      Q{_q}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
-   { }
-   GradientIntegrator(Coefficient &q) :
-      Q{&q}, trial_maps{NULL}, test_maps{NULL}, geom{NULL}
-   { }
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
-};
-
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
    can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
@@ -1776,60 +1674,22 @@ private:
 #endif

   // PA extension
-   const FiniteElementSpace *fespace;
   const DofToQuad *maps;         ///< Not owned
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, dofs1D, quad1D;
   Vector pa_data;

-#ifdef MFEM_USE_CEED
-   // CEED extension
-   CeedData* ceedDataPtr;
-#endif
-
 public:
   /// Construct a diffusion integrator with coefficient Q = 1
-   DiffusionIntegrator()
-   {
-      Q = NULL;
-      MQ = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+   DiffusionIntegrator() { Q = NULL; MQ = NULL; maps = NULL; geom = NULL; }

   /// Construct a diffusion integrator with a scalar coefficient q
   DiffusionIntegrator(Coefficient &q)
-      : Q(&q)
-   {
-      MQ = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : Q(&q) { MQ = NULL; maps = NULL; geom = NULL; }

   /// Construct a diffusion integrator with a matrix coefficient q
   DiffusionIntegrator(MatrixCoefficient &q)
-      : MQ(&q)
-   {
-      Q = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
-
-   virtual ~DiffusionIntegrator()
-   {
-#ifdef MFEM_USE_CEED
-      delete ceedDataPtr;
-#endif
-   }
+      : MQ(&q) { Q = NULL; maps = NULL; geom = NULL; }

   /** Given a particular Finite Element
       computes the element stiffness matrix elmat. */
@@ -1851,24 +1711,18 @@ public:
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true);
+                                   Vector &flux, int with_coef = 1);

   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
                                    Vector &flux, Vector *d_energy = NULL);

-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-
-   virtual void AssembleDiagonalPA(Vector &diag);
+   virtual void AssemblePA(const FiniteElementSpace&);

   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe);
-
-   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -1880,46 +1734,19 @@ protected:
 #endif
   Coefficient *Q;
   // PA extension
-   const FiniteElementSpace *fespace;
   Vector pa_data;
   const DofToQuad *maps;         ///< Not owned
   const GeometricFactors *geom;  ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;

-#ifdef MFEM_USE_CEED
-   // CEED extension
-   CeedData* ceedDataPtr;
-#endif
-
 public:
   MassIntegrator(const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir)
-   {
-      Q = NULL;
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : BilinearFormIntegrator(ir) { Q = NULL; maps = NULL; geom = NULL; }

   /// Construct a mass integrator with coefficient q
   MassIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BilinearFormIntegrator(ir), Q(&q)
-   {
-      maps = NULL;
-      geom = NULL;
-#ifdef MFEM_USE_CEED
-      ceedDataPtr = NULL;
-#endif
-   }
+      : BilinearFormIntegrator(ir), Q(&q) { maps = NULL; geom = NULL; }

-   virtual ~MassIntegrator()
-   {
-#ifdef MFEM_USE_CEED
-      delete ceedDataPtr;
-#endif
-   }
   /** Given a particular Finite Element
       computes the element mass matrix elmat. */
   virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -1930,19 +1757,13 @@ public:
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);

-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-
-   virtual void AssembleDiagonalPA(Vector &diag);
+   virtual void AssemblePA(const FiniteElementSpace&);

   virtual void AddMultPA(const Vector&, Vector&) const;

   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe,
                                         ElementTransformation &Trans);
-
-   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };

 class BoundaryMassIntegrator : public MassIntegrator
@@ -1964,11 +1785,6 @@ class ConvectionIntegrator : public BilinearFormIntegrator
 protected:
   VectorCoefficient *Q;
   double alpha;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;

 private:
 #ifndef MFEM_THREAD_SAFE
@@ -1982,19 +1798,6 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &,
                                      ElementTransformation &,
                                      DenseMatrix &);
-
-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace&);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &el,
-                                         ElementTransformation &Trans);
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
 };

 /// alpha (q . grad u, v) using the "group" FE discretization
@@ -2032,11 +1835,6 @@ protected:
   Coefficient *Q;
   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;

 public:
   /// Construct an integrator with coefficient 1.0
@@ -2068,10 +1866,6 @@ public:
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssembleDiagonalPA(Vector &diag);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
 };


@@ -2195,13 +1989,6 @@ protected:
   Coefficient *Q;
   MatrixCoefficient *MQ;

-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
-
 public:
   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
@@ -2217,16 +2004,11 @@ public:
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef);
+                                   Vector &flux, int with_coef);

   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
                                    Vector &flux, Vector *d_energy = NULL);
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AssembleDiagonalPA(Vector& diag);
 };

 /** Integrator for (curl u, curl v) for FE spaces defined by 'dim' copies of a
@@ -2256,10 +2038,7 @@ public:
                                   const Vector &elfun);
 };

-/** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
-    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
-    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
-    vi are in H1. */
+/// Integrator for (Q u, v) for VectorFiniteElements
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
@@ -2270,7 +2049,6 @@ private:
   Vector shape;
   Vector D;
   DenseMatrix K;
-   DenseMatrix partelmat;
   DenseMatrix test_vshape;
   DenseMatrix trial_vshape;
 #endif
@@ -2280,13 +2058,6 @@ protected:
   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;

-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
-
 public:
   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
   VectorFEMassIntegrator(Coefficient *_q) { Init(_q, NULL, NULL); }
@@ -2303,11 +2074,6 @@ public:
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AssembleDiagonalPA(Vector& diag);
 };

 /** Integrator for (Q div u, p) where u=(v1,...,vn) and all vi are in the same
@@ -2323,39 +2089,16 @@ private:
   DenseMatrix dshape;
   DenseMatrix gshape;
   DenseMatrix Jadj;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *trial_maps, *test_maps; ///< Not owned
-   const GeometricFactors *geom;            ///< Not owned
-   int dim, ne, nq;
-   int trial_dofs1D, test_dofs1D, quad1D;

 public:
-   VectorDivergenceIntegrator() :
-      Q(NULL), trial_maps(NULL), test_maps(NULL), geom(NULL)
-   {  }
-   VectorDivergenceIntegrator(Coefficient *_q) :
-      Q(_q), trial_maps(NULL), test_maps(NULL), geom(NULL)
-   { }
-   VectorDivergenceIntegrator(Coefficient &q) :
-      Q(&q), trial_maps(NULL), test_maps(NULL), geom(NULL)
-   { }
+   VectorDivergenceIntegrator() { Q = NULL; }
+   VectorDivergenceIntegrator(Coefficient *_q) { Q = _q; }
+   VectorDivergenceIntegrator(Coefficient &q) { Q = &q; }

   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
-                                         const FiniteElement &test_fe,
-                                         ElementTransformation &Trans);
 };

 /// (Q div u, div v) for RT elements
@@ -2389,12 +2132,6 @@ class VectorDiffusionIntegrator : public BilinearFormIntegrator
 protected:
   Coefficient *Q;

-   // PA extension
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, dofs1D, quad1D;
-   Vector pa_data;
-
 private:
   DenseMatrix Jinv;
   DenseMatrix dshape;
@@ -2411,10 +2148,6 @@ public:
   virtual void AssembleElementVector(const FiniteElement &el,
                                      ElementTransformation &Tr,
                                      const Vector &elfun, Vector &elvect);
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssembleDiagonalPA(Vector &diag);
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
 };

 /** Integrator for the linear elasticity form:
@@ -2459,7 +2192,7 @@ public:
                                   ElementTransformation &Trans,
                                   Vector &u,
                                   const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true);
+                                   Vector &flux, int with_coef = 1);

   /** Compute the element energy (integral of the strain energy density)
       corresponding to the stress represented by @a flux which is a vector of
@@ -2489,11 +2222,6 @@ protected:
   Coefficient *rho;
   VectorCoefficient *u;
   double alpha, beta;
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *maps;             ///< Not owned
-   const FaceGeometricFactors *geom;  ///< Not owned
-   int dim, nf, nq, dofs1D, quad1D;

 private:
   Vector shape1, shape2;
@@ -2512,22 +2240,6 @@ public:
                                   const FiniteElement &el2,
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);
-
-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
-
-   virtual void AssemblePABoundaryFaces(const FiniteElementSpace &fes);
-
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-   static const IntegrationRule &GetRule(Geometry::Type geom, int order,
-                                         FaceElementTransformations &T);
-
-private:
-   void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

 /** Integrator for the DG form:
@@ -1,865 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license.  We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "../general/forall.hpp"
-#include "bilininteg.hpp"
-#include "gridfunc.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-// PA Convection Integrator
-
-// PA Convection Assemble 2D kernel
-static void PAConvectionSetup2D(const int Q1D,
-                                const int ne,
-                                const Array<double> &w,
-                                const Vector &j,
-                                const Vector &vel,
-                                const double alpha,
-                                Vector &op)
-{
-   const int NE = ne;
-   const int NQ = Q1D*Q1D;
-   auto W = w.Read();
-
-   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   const bool const_v = vel.Size() == 2;
-   auto V =
-      const_v ? Reshape(vel.Read(), 2,1,1) : Reshape(vel.Read(), 2,NQ,NE);
-   auto y = Reshape(op.Write(), NQ, 2, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double w = alpha * W[q];
-         const double v0 = const_v ? V(0,0,0) : V(0,q,e);
-         const double v1 = const_v ? V(1,0,0) : V(1,q,e);
-         const double wx = w * v0;
-         const double wy = w * v1;
-         //w*J^-1
-         y(q,0,e) =  wx * J22 - wy * J12; // 1
-         y(q,1,e) = -wx * J21 + wy * J11; // 2
-      }
-   });
-}
-
-// PA Convection Assemble 3D kernel
-static void PAConvectionSetup3D(const int Q1D,
-                                const int NE,
-                                const Array<double> &w,
-                                const Vector &j,
-                                const Vector &vel,
-                                const double alpha,
-                                Vector &op)
-{
-   const int NQ = Q1D*Q1D*Q1D;
-   auto W = w.Read();
-   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   const bool const_v = vel.Size() == 3;
-   auto V =
-      const_v ? Reshape(vel.Read(), 3,1,1) : Reshape(vel.Read(), 3,NQ,NE);
-   auto y = Reshape(op.Write(), NQ, 3, NE);
-   MFEM_FORALL(e, NE,
-   {
-      for (int q = 0; q < NQ; ++q)
-      {
-         const double J11 = J(q,0,0,e);
-         const double J21 = J(q,1,0,e);
-         const double J31 = J(q,2,0,e);
-         const double J12 = J(q,0,1,e);
-         const double J22 = J(q,1,1,e);
-         const double J32 = J(q,2,1,e);
-         const double J13 = J(q,0,2,e);
-         const double J23 = J(q,1,2,e);
-         const double J33 = J(q,2,2,e);
-         const double w = alpha * W[q];
-         const double v0 = const_v ? V(0,0,0) : V(0,q,e);
-         const double v1 = const_v ? V(1,0,0) : V(1,q,e);
-         const double v2 = const_v ? V(2,0,0) : V(2,q,e);
-         const double wx = w * v0;
-         const double wy = w * v1;
-         const double wz = w * v2;
-         // adj(J)
-         const double A11 = (J22 * J33) - (J23 * J32);
-         const double A12 = (J32 * J13) - (J12 * J33);
-         const double A13 = (J12 * J23) - (J22 * J13);
-         const double A21 = (J31 * J23) - (J21 * J33);
-         const double A22 = (J11 * J33) - (J13 * J31);
-         const double A23 = (J21 * J13) - (J11 * J23);
-         const double A31 = (J21 * J32) - (J31 * J22);
-         const double A32 = (J31 * J12) - (J11 * J32);
-         const double A33 = (J11 * J22) - (J12 * J21);
-         // q . J^{-1} = q . adj(J)
-         y(q,0,e) =  wx * A11 + wy * A12 + wz * A13;
-         y(q,1,e) =  wx * A21 + wy * A22 + wz * A23;
-         y(q,2,e) =  wx * A31 + wy * A32 + wz * A33;
-      }
-   });
-}
-
-static void PAConvectionSetup(const int dim,
-                              const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &W,
-                              const Vector &J,
-                              const Vector &coeff,
-                              const double alpha,
-                              Vector &op)
-{
-   if (dim == 1) { MFEM_ABORT("dim==1 not supported in PAConvectionSetup"); }
-   if (dim == 2)
-   {
-      PAConvectionSetup2D(Q1D, NE, W, J, coeff, alpha, op);
-   }
-   if (dim == 3)
-   {
-      PAConvectionSetup3D(Q1D, NE, W, J, coeff, alpha, op);
-   }
-}
-
-// PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
-void PAConvectionApply2D(const int ne,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Array<double> &gt,
-                         const Vector &_op,
-                         const Vector &_x,
-                         Vector &_y,
-                         const int d1d = 0,
-                         const int q1d = 0)
-{
-   const int NE = ne;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, 2, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      double u[max_D1D][max_D1D];
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            u[dy][dx] = x(dx,dy,e);
-         }
-      }
-      double Bu[max_D1D][max_Q1D];
-      double Gu[max_D1D][max_Q1D];
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            Bu[dy][qx] = 0.0;
-            Gu[dy][qx] = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double bx  = B(qx,dx);
-               const double gx  = G(qx,dx);
-               const double x = u[dy][dx];
-               Bu[dy][qx] += bx * x;
-               Gu[dy][qx] += gx * x;
-            }
-         }
-      }
-      double GBu[max_Q1D][max_Q1D];
-      double BGu[max_Q1D][max_Q1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            GBu[qy][qx] = 0.0;
-            BGu[qy][qx] = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double bx  = B(qy,dy);
-               const double gx  = G(qy,dy);
-               GBu[qy][qx] += gx * Bu[dy][qx];
-               BGu[qy][qx] += bx * Gu[dy][qx];
-            }
-         }
-      }
-      // Calculate Dxy, xDy in plane
-      double DGu[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double O1 = op(qx,qy,0,e);
-            const double O2 = op(qx,qy,1,e);
-
-            const double gradX = BGu[qy][qx];
-            const double gradY = GBu[qy][qx];
-
-            DGu[qy][qx] = (O1 * gradX) + (O2 * gradY);
-         }
-      }
-      double BDGu[max_D1D][max_Q1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            BDGu[dy][qx] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double w  = Bt(dy,qy);
-               BDGu[dy][qx] += w * DGu[qy][qx];
-            }
-         }
-      }
-      for (int dx = 0; dx < D1D; ++dx)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double BBDGu = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double w  = Bt(dx,qx);
-               BBDGu += w * BDGu[dy][qx];
-            }
-            y(dx,dy,e) += BBDGu;
-         }
-      }
-   });
-}
-
-// Optimized PA Convection Apply 2D kernel
-template<int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0> static
-void SmemPAConvectionApply2D(const int ne,
-                             const Array<double> &b,
-                             const Array<double> &g,
-                             const Array<double> &bt,
-                             const Array<double> &gt,
-                             const Vector &_op,
-                             const Vector &_x,
-                             Vector &_y,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int NE = ne;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, 2, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, NE);
-   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      // constexpr int MDQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D;
-      MFEM_SHARED double u[NBZ][max_D1D][max_D1D];
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            // e is really equal to e+tidz
-            u[tidz][dy][dx] = x(dx,dy,e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED double Bu[NBZ][max_D1D][max_Q1D];
-      MFEM_SHARED double Gu[NBZ][max_D1D][max_Q1D];
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            Bu[tidz][dy][qx] = 0.0;
-            Gu[tidz][dy][qx] = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double bx = B(qx,dx);
-               const double gx = G(qx,dx);
-               const double x  = u[tidz][dy][dx];
-               Bu[tidz][dy][qx] += bx * x;
-               Gu[tidz][dy][qx] += gx * x;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED double GBu[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED double BGu[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            GBu[tidz][qy][qx] = 0.0;
-            BGu[tidz][qy][qx] = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double bx  = B(qy,dy);
-               const double gx  = G(qy,dy);
-               GBu[tidz][qy][qx] += gx * Bu[tidz][dy][qx];
-               BGu[tidz][qy][qx] += bx * Gu[tidz][dy][qx];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Calculate Dxy, xDy in plane
-      MFEM_SHARED double DGu[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            const double O1 = op(qx,qy,0,e);
-            const double O2 = op(qx,qy,1,e);
-
-            const double gradX = BGu[tidz][qy][qx];
-            const double gradY = GBu[tidz][qy][qx];
-
-            DGu[tidz][qy][qx] = (O1 * gradX) + (O2 * gradY);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED double BDGu[NBZ][max_D1D][max_Q1D];
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            BDGu[tidz][dy][qx] = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double w  = Bt(dy,qy);
-               BDGu[tidz][dy][qx] += w * DGu[tidz][qy][qx];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            double BBDGu = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double w  = Bt(dx,qx);
-               BBDGu += w * BDGu[tidz][dy][qx];
-            }
-            y(dx,dy,e) += BBDGu;
-         }
-      }
-   });
-}
-
-// PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
-void PAConvectionApply3D(const int ne,
-                         const Array<double> &b,
-                         const Array<double> &g,
-                         const Array<double> &bt,
-                         const Array<double> &gt,
-                         const Vector &_op,
-                         const Vector &_x,
-                         Vector &_y,
-                         const int d1d = 0,
-                         const int q1d = 0)
-{
-   const int NE = ne;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, 3, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL(e, NE,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-
-      double u[max_D1D][max_D1D][max_D1D];
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               u[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-      }
-      double Bu[max_D1D][max_D1D][max_Q1D];
-      double Gu[max_D1D][max_D1D][max_Q1D];
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               Bu[dz][dy][qx] = 0.0;
-               Gu[dz][dy][qx] = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double bx  = B(qx,dx);
-                  const double gx  = G(qx,dx);
-                  const double x = u[dz][dy][dx];
-                  Bu[dz][dy][qx] += bx * x;
-                  Gu[dz][dy][qx] += gx * x;
-               }
-            }
-         }
-      }
-      double BBu[max_D1D][max_Q1D][max_Q1D];
-      double GBu[max_D1D][max_Q1D][max_Q1D];
-      double BGu[max_D1D][max_Q1D][max_Q1D];
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               BBu[dz][qy][qx] = 0.0;
-               GBu[dz][qy][qx] = 0.0;
-               BGu[dz][qy][qx] = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const double bx  = B(qy,dy);
-                  const double gx  = G(qy,dy);
-                  BBu[dz][qy][qx] += bx * Bu[dz][dy][qx];
-                  GBu[dz][qy][qx] += gx * Bu[dz][dy][qx];
-                  BGu[dz][qy][qx] += bx * Gu[dz][dy][qx];
-               }
-            }
-         }
-      }
-      double GBBu[max_Q1D][max_Q1D][max_Q1D];
-      double BGBu[max_Q1D][max_Q1D][max_Q1D];
-      double BBGu[max_Q1D][max_Q1D][max_Q1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               GBBu[qz][qy][qx] = 0.0;
-               BGBu[qz][qy][qx] = 0.0;
-               BBGu[qz][qy][qx] = 0.0;
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const double bx  = B(qz,dz);
-                  const double gx  = G(qz,dz);
-                  GBBu[qz][qy][qx] += gx * BBu[dz][qy][qx];
-                  BGBu[qz][qy][qx] += bx * GBu[dz][qy][qx];
-                  BBGu[qz][qy][qx] += bx * BGu[dz][qy][qx];
-               }
-            }
-         }
-      }
-      // Calculate Dxy, xDy in plane
-      double DGu[max_Q1D][max_Q1D][max_Q1D];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double O1 = op(qx,qy,qz,0,e);
-               const double O2 = op(qx,qy,qz,1,e);
-               const double O3 = op(qx,qy,qz,2,e);
-
-               const double gradX = BBGu[qz][qy][qx];
-               const double gradY = BGBu[qz][qy][qx];
-               const double gradZ = GBBu[qz][qy][qx];
-
-               DGu[qz][qy][qx] = (O1 * gradX) + (O2 * gradY) + (O3 * gradZ);
-            }
-         }
-      }
-      double BDGu[max_D1D][max_Q1D][max_Q1D];
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               BDGu[dz][qy][qx] = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  const double w  = Bt(dz,qz);
-                  BDGu[dz][qy][qx] += w * DGu[qz][qy][qx];
-               }
-            }
-         }
-      }
-      double BBDGu[max_D1D][max_D1D][max_Q1D];
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               BBDGu[dz][dy][qx] = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double w  = Bt(dy,qy);
-                  BBDGu[dz][dy][qx] += w * BDGu[dz][qy][qx];
-               }
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               double BBBDGu = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double w  = Bt(dx,qx);
-                  BBBDGu += w * BBDGu[dz][dy][qx];
-               }
-               y(dx,dy,dz,e) += BBBDGu;
-            }
-         }
-      }
-   });
-}
-
-// Optimized PA Convection Apply 3D kernel
-template<int T_D1D = 0, int T_Q1D = 0> static
-void SmemPAConvectionApply3D(const int ne,
-                             const Array<double> &b,
-                             const Array<double> &g,
-                             const Array<double> &bt,
-                             const Array<double> &gt,
-                             const Vector &_op,
-                             const Vector &_x,
-                             Vector &_y,
-                             const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int NE = ne;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(_op.Read(), Q1D, Q1D, Q1D, 3, NE);
-   auto x = Reshape(_x.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(_y.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D;
-      MFEM_SHARED double sm0[max_DQ*max_DQ*max_DQ];
-      MFEM_SHARED double sm1[max_DQ*max_DQ*max_DQ];
-      MFEM_SHARED double sm2[max_DQ*max_DQ*max_DQ];
-      MFEM_SHARED double sm3[max_DQ*max_DQ*max_DQ];
-      MFEM_SHARED double sm4[max_DQ*max_DQ*max_DQ];
-      MFEM_SHARED double sm5[max_DQ*max_DQ*max_DQ];
-
-      double (*u)[max_D1D][max_D1D] = (double (*)[max_D1D][max_D1D]) sm0;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               u[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      double (*Bu)[max_D1D][max_Q1D] = (double (*)[max_D1D][max_Q1D])sm1;
-      double (*Gu)[max_D1D][max_Q1D] = (double (*)[max_D1D][max_Q1D])sm2;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double Bu_ = 0.0;
-               double Gu_ = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const double bx  = B(qx,dx);
-                  const double gx  = G(qx,dx);
-                  const double x = u[dz][dy][dx];
-                  Bu_ += bx * x;
-                  Gu_ += gx * x;
-               }
-               Bu[dz][dy][qx] = Bu_;
-               Gu[dz][dy][qx] = Gu_;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      double (*BBu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm3;
-      double (*GBu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm4;
-      double (*BGu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm5;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               double BBu_ = 0.0;
-               double GBu_ = 0.0;
-               double BGu_ = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const double bx  = B(qy,dy);
-                  const double gx  = G(qy,dy);
-                  BBu_ += bx * Bu[dz][dy][qx];
-                  GBu_ += gx * Bu[dz][dy][qx];
-                  BGu_ += bx * Gu[dz][dy][qx];
-               }
-               BBu[dz][qy][qx] = BBu_;
-               GBu[dz][qy][qx] = GBu_;
-               BGu[dz][qy][qx] = BGu_;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      double (*GBBu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm0;
-      double (*BGBu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm1;
-      double (*BBGu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm2;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               double GBBu_ = 0.0;
-               double BGBu_ = 0.0;
-               double BBGu_ = 0.0;
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const double bx  = B(qz,dz);
-                  const double gx  = G(qz,dz);
-                  GBBu_ += gx * BBu[dz][qy][qx];
-                  BGBu_ += bx * GBu[dz][qy][qx];
-                  BBGu_ += bx * BGu[dz][qy][qx];
-               }
-               GBBu[qz][qy][qx] = GBBu_;
-               BGBu[qz][qy][qx] = BGBu_;
-               BBGu[qz][qy][qx] = BBGu_;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      double (*DGu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm3;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double O1 = op(qx,qy,qz,0,e);
-               const double O2 = op(qx,qy,qz,1,e);
-               const double O3 = op(qx,qy,qz,2,e);
-
-               const double gradX = BBGu[qz][qy][qx];
-               const double gradY = BGBu[qz][qy][qx];
-               const double gradZ = GBBu[qz][qy][qx];
-
-               DGu[qz][qy][qx] = (O1 * gradX) + (O2 * gradY) + (O3 * gradZ);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      double (*BDGu)[max_Q1D][max_Q1D] = (double (*)[max_Q1D][max_Q1D])sm4;
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dz,z,D1D)
-            {
-               double BDGu_ = 0.0;
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  const double w  = Bt(dz,qz);
-                  BDGu_ += w * DGu[qz][qy][qx];
-               }
-               BDGu[dz][qy][qx] = BDGu_;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      double (*BBDGu)[max_D1D][max_Q1D] = (double (*)[max_D1D][max_Q1D])sm5;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dy,y,D1D)
-            {
-               double BBDGu_ = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double w  = Bt(dy,qy);
-                  BBDGu_ += w * BDGu[dz][qy][qx];
-               }
-               BBDGu[dz][dy][qx] = BBDGu_;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
-            {
-               double BBBDGu = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const double w  = Bt(dx,qx);
-                  BBBDGu += w * BBDGu[dz][dy][qx];
-               }
-               y(dx,dy,dz,e) = BBBDGu;
-            }
-         }
-      }
-   });
-}
-
-void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
-{
-   // Assumes tensor-product elements
-   Mesh *mesh = fes.GetMesh();
-   const FiniteElement &el = *fes.GetFE(0);
-   ElementTransformation &Trans = *fes.GetElementTransformation(0);
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, Trans);
-   const int dims = el.GetDim();
-   const int symmDims = dims;
-   const int nq = ir->GetNPoints();
-   dim = mesh->Dimension();
-   ne = fes.GetNE();
-   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
-   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
-   dofs1D = maps->ndof;
-   quad1D = maps->nqpt;
-   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
-   Vector vel;
-   if (VectorConstantCoefficient *cQ = dynamic_cast<VectorConstantCoefficient*>(Q))
-   {
-      vel = cQ->GetVec();
-   }
-   else
-   {
-      vel.SetSize(dim * nq * ne);
-      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      Vector Vq(dim);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            Q->Eval(Vq, T, ir->IntPoint(q));
-            for (int i = 0; i < dim; ++i)
-            {
-               C(i,q,e) = Vq(i);
-            }
-         }
-      }
-   }
-   PAConvectionSetup(dim, dofs1D, quad1D, ne, ir->GetWeights(), geom->J,
-                     vel, alpha, pa_data);
-}
-
-static void PAConvectionApply(const int dim,
-                              const int D1D,
-                              const int Q1D,
-                              const int NE,
-                              const Array<double> &B,
-                              const Array<double> &G,
-                              const Array<double> &Bt,
-                              const Array<double> &Gt,
-                              const Vector &op,
-                              const Vector &x,
-                              Vector &y)
-{
-   if (dim == 2)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x22: return SmemPAConvectionApply2D<2,2,8>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x33: return SmemPAConvectionApply2D<3,3,3>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x44: return SmemPAConvectionApply2D<4,4,2>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x55: return SmemPAConvectionApply2D<5,5,2>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x66: return SmemPAConvectionApply2D<6,6,1>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x77: return SmemPAConvectionApply2D<7,7,1>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x88: return SmemPAConvectionApply2D<8,8,1>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x99: return SmemPAConvectionApply2D<9,9,1>(NE,B,G,Bt,Gt,op,x,y);
-         default:   return PAConvectionApply2D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch ((D1D << 4 ) | Q1D)
-      {
-         case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x34: return SmemPAConvectionApply3D<3,4>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x45: return SmemPAConvectionApply3D<4,5>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x56: return SmemPAConvectionApply3D<5,6>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x67: return SmemPAConvectionApply3D<6,7>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x78: return SmemPAConvectionApply3D<7,8>(NE,B,G,Bt,Gt,op,x,y);
-         case 0x89: return SmemPAConvectionApply3D<8,9>(NE,B,G,Bt,Gt,op,x,y);
-         default:   return PAConvectionApply3D(NE,B,G,Bt,Gt,op,x,y,D1D,Q1D);
-      }
-   }
-   MFEM_ABORT("Unknown kernel.");
-}
-
-// PA Convection Apply kernel
-void ConvectionIntegrator::AddMultPA(const Vector &x, Vector &y) const
-{
-   PAConvectionApply(dim, dofs1D, quad1D, ne,
-                     maps->B, maps->G, maps->Bt, maps->Gt,
-                     pa_data, x, y);
-}
-
-} // namespace mfem
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Jason Hicken	a67cf7bff6	fixed euclid issue	2020-02-26 14:15:38 -05:00
Jason Hicken	858e47628f	Merge branch 'master' into functional-dev	2019-12-19 15:08:07 -05:00
Jason Hicken	d9e78bae82	removed Functional alias to CalcEnergy, and removed comment about prolongate	2019-08-30 15:27:32 -04:00
Jason Hicken	ecfe5a98c7	added face and boundary terms to GetEnergy	2019-08-22 15:46:01 -04:00