add optional MemoryType to the SparseMatrix copy-ctor

Cherry-picking changes on top of our current hash.
There are things that hurt performance in laster mfem@master commits, not sure what yet. This includes: - feature/artv3/cusparse-Spmv - feature/tomstitt/temp-mem-type - patches to tmop.cpp and tmop_tools.cpp to address memory issues - patch to mem_manager/device to use mfem's default allcator instead of the host umpire one
2020-07-24 09:13:21 -07:00 · 2020-07-22 15:51:04 -07:00 · 2020-06-23 12:11:33 -07:00 · 2020-06-22 14:42:48 -07:00 · 2020-06-22 14:29:28 -07:00 · 2020-06-18 13:35:11 -07:00
297 changed files with 15474 additions and 80982 deletions
@@ -15,10 +15,8 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

-# Install METIS, use a mirror because the original source server is not always
-# up. Original url:
-#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
- ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
+# Install METIS
+- ps: Start-FileDownload 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
@@ -28,17 +26,17 @@ install:
 - cd ..

 # Install hypre
- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
- 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
- cd hypre-2.19.0/src
- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
+- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2-10-0b
+- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
- cd ../..
+- cd ..

 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -122,7 +122,7 @@ examples/sundials/ex16-final.*
 examples/sundials/Example16*

 examples/petsc/ex[1-69]p
-examples/petsc/ex1[0-1]p
+examples/petsc/ex10p

 examples/petsc/mesh.*
 examples/petsc/sol.*
@@ -137,7 +137,6 @@ examples/petsc/Example9*
 examples/petsc/deformed.*
 examples/petsc/velocity.*
 examples/petsc/elastic_energy.*
-examples/petsc/mode_*

 examples/pumi/ex1
 examples/pumi/ex[126]p
@@ -175,7 +174,6 @@ miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
-miniapps/meshing/polar-nc

 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
@@ -188,7 +186,6 @@ miniapps/meshing/extruder.mesh
 miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
-miniapps/meshing/polar-nc.mesh

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -235,7 +232,6 @@ miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*

 miniapps/gslib/field-diff
-miniapps/gslib/field-interp
 miniapps/gslib/findpts
 miniapps/gslib/pfindpts

@@ -247,9 +243,6 @@ miniapps/navier/navier_3dfoc
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output

-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -262,10 +255,5 @@ tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg

-# Other tests
-tests/convergence/rates
-tests/convergence/prates
-tests/par-mesh-format/ex1p
-
 # VPATH builds
 build-*/*
@@ -71,8 +71,6 @@ stages:
  - build
  - test
  - deallocate
-  - lassen_build
-  - lassen_test
  - baseline_check
  - baseline_publish

@@ -81,11 +79,7 @@ stages:
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
-setup:
-  tags:
-    - shell
-    - quartz
+.setup:
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -106,15 +100,6 @@ setup:
  before_script:
    - module load gcc/6.1.0

-# On lassen
-.with_gcc_8_3_1:
-  variables:
-    TOOLCHAIN: gcc_8_3_1
-    CXX: g++
-    CC: gcc
-  before_script:
-    - module load gcc/8.3.1
-
 .with_gcc_4_9_3:
  variables:
    TOOLCHAIN: gcc_4_9_3
@@ -305,4 +290,3 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
  - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml
@@ -1,57 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  variables:
-    PLAT: lassen
-
-# Build MFEM
-build_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
-
-build_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CUDA_ARCH=sm_70
-
-# Sanity check
-sanitycheck_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 15 -q pdebug make -j test
-
-sanitycheck_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_debug_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 30 -q pdebug make -j test
@@ -22,6 +22,10 @@
    MAKE_PAR: 6
    BASELINE_PAR: 18

+# Setup
+setup_quartz:
+  extends: [.setup, .on_quartz]
+
 # Allocate
 allocate_quartz:
  variables:
@@ -11,20 +11,11 @@

 language: cpp

-os: linux
-dist: bionic
-
 stages:
  - checks
  - tests
  - optional

-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
 jobs:
  include:

@@ -37,7 +28,6 @@ jobs:

    - stage: checks
      os: linux
-      dist: xenial
      name: "code-style"
      addons:
        apt:
@@ -56,6 +46,9 @@ jobs:
          packages:
            - doxygen
            - graphviz
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
      script:
        - cd ${TRAVIS_BUILD_DIR}
        - cd tests/scripts
@@ -70,24 +63,13 @@ jobs:
            - mpich
            - libmpich-dev
      env: MPI=YES
-      before_script:
+      script:
        - cd ${TRAVIS_BUILD_DIR}
-        - mpicxx -v
        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
        - make all -j3
        - make test-noclean
-      script:
        - cd tests/scripts
        - ./runtest gitignore
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}

    # ========================
    #  Optional Checks/Tests
@@ -96,7 +78,6 @@ jobs:

    - stage: optional
      name: "branch-history"
-      if: branch != next
      # need full git history for the binary/big files check
      git:
        depth: false
@@ -125,8 +106,6 @@ jobs:
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true

    - os: linux
      compiler: gcc
@@ -135,8 +114,6 @@ jobs:
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true

    - os: linux
      compiler: gcc
@@ -160,9 +137,9 @@ jobs:
           MFEM_TEST_TARGET=check
           NPROCS=2
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -191,9 +168,9 @@ jobs:
           MFEM_TEST_TARGET=test
           NPROCS=2
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -216,16 +193,16 @@ jobs:
        - cd ${TRAVIS_BUILD_DIR}/build
        - cmake ..
             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../hypre-2.10.0b/src/hypre
             -DMFEM_MPI_NP=$NPROCS
        - make -j3 mfem examples
        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
        - make -j3
        - ctest --output-on-failure
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -241,43 +218,27 @@ jobs:
    # - parallel

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Serial + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=YES
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Serial"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=NO
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Parallel + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=YES
           MPI=YES
           CODECOV=NO
@@ -285,9 +246,9 @@ jobs:
           NPROCS=4
           TMPDIR=/tmp
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -296,13 +257,9 @@ jobs:
          rm -f Lib/*.{c,o}

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Parallel"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=NO
           MPI=YES
           CODECOV=YES
@@ -310,9 +267,9 @@ jobs:
           NPROCS=4
           TMPDIR=/tmp
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -326,19 +283,14 @@ before_install:
   #      brew install open-mpi;
   #   fi

-   # Disable ccache while building dependencies that are cached:
-   - echo "before \$PATH = $PATH";
-     export PATH=${PATH//\/usr\/lib\/ccache:/};
-     echo "after  \$PATH = $PATH"
-
-   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   # On Mac OS X, build and cache OpenMPI 2.1.1:
   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
           mkdir -p $HOME/builds && cd $HOME/builds &&
-           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
-           tar jxf openmpi-2.1.6.tar.bz2 &&
+           wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2 &&
+           tar jxf openmpi-2.1.1.tar.bz2 &&
           mkdir openmpi-build && cd openmpi-build &&
-           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           ../openmpi-2.1.1/configure --prefix=$HOME/local-cached &&
           make -j3 all && make install;
        fi;
        PATH=$HOME/local-cached/bin:$PATH;
@@ -383,28 +335,26 @@ install:

   # hypre
   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+        if [ ! -e hypre-2.10.0b/src/hypre/lib/libHYPRE.a ]; then
+           wget https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz --no-check-certificate;
+           rm -rf hypre-2.10.0b;
+           tar xvzf hypre-2.10.0b.tar.gz;
+           cd hypre-2.10.0b/src;
+           ./configure --disable-fortran --without-fei CC=mpicc CXX=mpic++;
           make -j3;
           cd ../..;
        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
+            echo "Reusing cached hypre-2.10.0b/";
        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
+        ln -s hypre-2.10.0b hypre;
     else
        echo "Serial build, not using hypre";
     fi

-   # METIS, use a mirror because the original source server is not always up.
-   # Original url:
-   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   # METIS
   - if [ $MPI == "YES" ]; then
        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz;
           tar xvzf metis-4.0.3.tar.gz;
           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
           rm -rf metis-4.0;
@@ -414,18 +364,6 @@ install:
        fi;
     fi

-   # Re-enable ccache on linux; enable ccache on mac os:
-   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-        export PATH="/usr/lib/ccache:$PATH";
-     else
-        if [ $TRAVIS_OS_NAME == "osx" ]; then
-           export PATH="/usr/local/opt/ccache/libexec:$PATH";
-        fi;
-     fi
-
-   - printf "which \$CC  = "; which $CC;
-     printf "which \$CXX = "; which $CXX
-
 script:
   # Compiler
   - if [ $MPI == "YES" ]; then
@@ -446,9 +384,6 @@ script:
     if [ "$CODECOV" == "YES" ]; then
        CPPFLAGS="--coverage -g";
     fi;
-     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
-        CPPFLAGS+=" -pedantic -Wall -Werror";
-     fi

   # Configure the library
   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
@@ -16,12 +16,7 @@ Meshing improvements
 - The graph linear ordering library Gecko, previously an external dependency, is
  now included directly in MFEM. As a result, Mesh::GetGeckoElementOrdering is
  always available. The interface has also been improved, see for example the
-  Mesh Explorer miniapp.
-
- Improved Gmsh reader (version 2.2), which now supports both high-order and
-  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
-  supported up to order 10. Wedges and hexahedra are supported up to order 9.
-  For sample periodic meshes, see the periodic*.msh files in the data directory.
+  mesh-explorer miniapp.

 - Added support for finite difference-based gradient and Hessian approximation
  in the TMOP mesh optimization algorithms. This improves the accuracy of the
@@ -32,16 +27,11 @@ Meshing improvements
  the user to specify different discrete functions for controlling the
  size, aspect-ratio, orientation, and skew of elements in the mesh.

- Added TMOP capability for approximate tangential mesh relaxation. Added
-  support and examples for using TMOP on mixed meshes.
+- Added TMOP capability for approximate tangential mesh relaxation.

- Added complete action of the TMOP Integrator to account for the spatial
-  derivatives of discrete and analytic targets.
-
- Added support for initialization of (serial) non-conforming meshes. Hanging
-  nodes can be marked with Mesh::AddVertexParents when building the mesh with
-  the "init" constructor. The usage is demonstrated in a new meshing miniapp
-  (polar-nc) which generates meshes that are non-conforming from the start.
+- Added support for reading periodic meshes in Gmsh format (version 2.2). See
+  for example the periodic-annulus-sector and periodic-torus-sector files in
+  the data directory.

 Performance improvements
 ------------------------
@@ -51,30 +41,13 @@ Performance improvements
    - x86 (SSE/AVX/AVX2/AVX512),
    - Power8 & Power9 (VSX),
    - BG/Q (QPX).
-  These are disabled by default, and can be enabled with MFEM_USE_SIMD=YES.
+  These are now enabled by default, and can be disabled with MFEM_USE_SIMD=NO.
  See the new file linalg/simd.hpp and the new directory linalg/simd.

 Improved GPU capabilities
 -------------------------
 - Added support for Chebyshev accelerated polynomial smoother on GPU.

- Optimized AMD/HIP kernel support.
-
- Added a Full Assembly mode compatible with Device kernel execution. This
-  assembly level builds on top of the current Element Assembly kernels to
-  compute a global sparse matrix. All integrators supported by element assembly
-  are also supported by full assembly. See the '-fa' option in Example 9.
-
- Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
-
- Added AmgX solver class for simple integration with NVIDIA's multigrid library.
-
- Added support for BlockOperator on GPU. See the updated Example 5.
-
- Added partial assembly and GPU support for complex operators, including the
-  classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
-  [Par]SesquilinearForm. See the updated Example 22.
-
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -98,19 +71,11 @@ Discretization improvements
  and, in the continuous field case, arbitrary mesh edges and faces.

 - Added new coefficient and vector coefficient classes for QuadratureFunctions.
-  Additionally, new LinearForm integrators were also added which make use of
+  Additionaly, new LinearForm integrators were also added which make use of
  these new QuadratureFunction coefficient classes.

 - Added support face integrals on the boundaries of NURBS meshes.

- Added support for interpolation of functions in L2, H(div) and H(curl)
-  spaces using GSLIB-FindPoints.
-
- Added support for computing asymptotic error estimates and convergence rates
-  for the whole de Rham sequence based on the new class ConvergenceStudy and new
-  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
-  the tests/convergence directory for sample usage.
-
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -123,10 +88,6 @@ Linear and nonlinear solvers
  and solution during the solving process of an IterativeSolver after every
  iteration.

- Added support for the CVODES package in SUNDIALS which provides ODE
-  solvers with sensitivity analysis capabilities. See the CVODESSolver
-  class and the new adjoint miniapps below.
-
 - Block arrays of parallel matrices can now be merged into a single parallel
  matrix with the function HypreParMatrixFromBlocks. This could be useful for
  solving block systems with parallel direct solvers such as STRUMPACK.
@@ -134,11 +95,6 @@ Linear and nonlinear solvers
 - In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
  solver can work with non-SPD preconditioner B.

- Added support for the SLEPc eigensolver package.
-
- Added partially assembled convergent diagonal preconditioner for adaptively
-  refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -153,21 +109,6 @@ New and updated examples and miniapps
  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
  and periodic boundary conditions with either H1 or DG discretizations.

- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
-  equations of incompressible fluid dynamics. See the miniapps/navier directory
-  for more details.
-
- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
-  solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
-  these miniapps require the MFEM_USE_SUNDIALS configuration option.
-  * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
-    system of ODEs, evaluating both forward and adjoint quadratures in serial.
-  * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
-    for an advection diffusion PDE, evaluating adjoint quadratures in parallel.
-
- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
-  equation with the shift-and-invert spectral transformation method.
-
 - Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
  stitching together opposite surfaces of a mesh to create a topologically
  periodic mesh.
@@ -175,16 +116,11 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
  the Dirichlet problem for the minimal surface equation.

- Added a new meshing miniapp, Polar NC, which demonstrates the construction of
-  polar non-conforming meshes.
-
- Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
+- Added partial assembly support to examples 4/4p and 5/5p, with diagonal
  preconditioning.

- Added full assembly support in Example 9/9p.
-
- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
-  form for H1, H(curl), H(div) and L_2, with partial assembly support.
+- Added a new test problem in example 24/24p, demonstrating a mixed bilinear
+  form for H(div) and L_2, with partial assembly support.

 - Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.

@@ -192,47 +128,26 @@ New and updated examples and miniapps
  mesh based on element attributes. Any newly exposed boundary elements are
  assigned attribute numbers related to the trimmed element attributes.

- Added a new miniapp (field-interp) that demonstrates transfer of grid function
-  between different meshes using GSLIB-FindPoints.
-
- Added diagonal preconditioner in Example 6/6p for partial assembly with AMR.
-
- Added device support in Example 5/5p.
-
- Added partial assembly and device support to Example 22/22p, with diagonal
-  preconditioning.
-
- Added the option to plot a function in Mesh Explorer.
-
 Improved testing
 ----------------
- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
-
 - Added a GitLab pipeline that automates PR testing on supercomputing systems
  and Linux clusters at Lawrence Livermore National Lab (LLNL). This can be
  triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
  and the updated CONTRIBUTING.md file.

- Added testing of the parallel mesh format in tests/par-mesh-format.
-
 Miscellaneous
 -------------
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. The
  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal node data. Cell centered data is accessible by
-  ADIOS2 data readers (e.g. Python), but currently not yet implement as of
-  ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
-  of data substreams/subfiles at scale. See examples 5, 9, 12, 16.
+  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
+  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.

 - The integration order used in the ComputeLpError and ComputeElementLpError
  methods of class GridFunction has been increased.

 - Various other simplifications, extensions, and bugfixes in the code.

- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
-  as DEBUG is sometimes used as a macro.
-

 Version 4.1, released on March 10, 2020
 =======================================
@@ -89,38 +89,8 @@ enable_language(CXX)
 if (MFEM_USE_CUDA)
   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
-   if (NOT CMAKE_CUDA_HOST_COMPILER)
-      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
   enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CUDA_FLAGS "--expt-extended-lambda")
-   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
-   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
-      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
-      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
-         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
-      else()
-         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
-      endif()
-   else()
-      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
-   endif()
   message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-   if (CMAKE_VERSION VERSION_LESS 3.12.0)
-      # CMake versions 3.8 and 3.9 require this to work; 3.10 and 3.11 are not
-      # tested and may not actually need this (but should be ok to keep).
-      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
-      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
-   endif()
-   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
-       "CUDA flags set for MFEM" FORCE)
-   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
 endif()

 if (XSDK_ENABLE_C)
@@ -179,13 +149,9 @@ if (MFEM_USE_MPI)
      message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
    endif()
    set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
-    if (MFEM_USE_SLEPC)
-      find_package(SLEPc REQUIRED config)
-      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
-    endif()
  endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
  foreach(PKG IN LISTS PKGS_NEED_MPI)
    if (MFEM_USE_${PKG})
      message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -241,10 +207,10 @@ endif()
 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
  if (NOT MFEM_USE_MPI)
-    find_package(SUNDIALS REQUIRED NVector_Serial CVODES ARKODE KINSOL)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODE ARKODE KINSOL)
  else()
    find_package(SUNDIALS REQUIRED
-      NVector_Serial NVector_Parallel NVector_ParHyp CVODES ARKODE KINSOL)
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODE ARKODE KINSOL)
  endif()
 endif()

@@ -295,11 +261,6 @@ if (MFEM_USE_CEED)
  find_package(libCEED REQUIRED)
 endif()

-if (MFEM_USE_AMGX)
-  include_directories(${AMGX_DIR}/include)
-  link_directories(${AMGX_DIR}/lib64 -L${AMGX_DIR}/lib -Xlinker=-rpath=${CUDA_HOME}/lib64)
-endif()
-
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -331,6 +292,22 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

+# CUDA
+if (MFEM_USE_CUDA)
+   set(CMAKE_CUDA_STANDARD 11)
+   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+   set(CMAKE_CUDA_EXTENSIONS OFF)
+   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
+       CACHE STRING "CUDA flags set for MFEM" FORCE)
+   if (MFEM_USE_MPI)
+      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
+   else()
+      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
+   endif()
+   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
+   set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CUDA_CCBIN_COMPILER})
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -375,9 +352,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE AMGX)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -109,7 +109,6 @@ The MFEM source code has the following structure:
  ├── linalg
  ├── mesh
  ├── miniapps
-  │   ├── adjoint
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
@@ -350,12 +350,6 @@ MFEM_USE_STRUMPACK = YES/NO
   classes. When enabled, this option uses the STRUMPACK_* library options, see
   below.

-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AMGX multigrid library from 
-   NVIDIA. When enabled, this options enables the user to use SparseMatrices
-   and HypreParMatrices to solve linear systems using routines from the AMGX
-   library.
-
 MFEM_USE_GINKGO = YES/NO
   Enable MFEM functionality based on the Ginkgo library, which provides
   iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
@@ -389,10 +383,6 @@ MFEM_USE_PETSC = YES/NO
   and other features based on the PETSc package. When enabled, this option uses
   the PETSC_* library options, see below.

-MFEM_USE_SLEPC = YES/NO
-   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
-   option uses the SLEPC_* library options, see below.
-
 MFEM_USE_MPFR = YES/NO
   MPFR is a library for multiple-precision floating-point computations. This
   option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -607,12 +597,6 @@ The specific libraries and their options are:
  Options: PETSC_OPT, PETSC_LIB.
  Versions: PETSc >= 3.8.0.

- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
-  uses some of the PETSc options when compiled.
-  URL: https://slepc.upv.es/
-  Options: SLEPC_OPT, SLEPC_LIB.
-  Versions: SLEPc >= 3.8.0.
-
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
  URL: https://github.com/LLNL/axom
@@ -656,11 +640,6 @@ The specific libraries and their options are:
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
  Versions: CUDA >= 9.1, older versions may work too.

- AMGX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AMGX >= 2.1, older versions may work too.
-
 - HIP (optional), used when MFEM_USE_HIP = YES.
  URL: https://rocm.github.io/ROCmInstall.html
  Options: HIP_CXX, HIP_ARCH, HIP_OPT, HIP_LIB.
@@ -670,11 +649,12 @@ The specific libraries and their options are:
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
  Versions: OCCA >= 1.0.9.

- libCEED (optional), used when MFEM_USE_CEED = YES.
+- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
+  or later version, specifically, git-hash 3d05795 or later.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED > 0.6, git-hash bdfed75.
+  Versions: libCEED >= 0.6.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -817,7 +797,6 @@ MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
-MFEM_USE_AMGX
 MFEM_USE_GINKGO
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
@@ -244,10 +244,6 @@ IF (DEFINED TPL_ENABLE_PETSC)
  SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_SLEPC)
-  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_MPFR)
  SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()
@@ -38,7 +38,6 @@ set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
-set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
@@ -104,9 +104,6 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC

-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE

@@ -126,9 +123,6 @@
 // Requires a CUDA compiler (nvcc).
 #cmakedefine MFEM_USE_CUDA

-// Enable MFEM functionality based on the AMGX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the RAJA library
 #cmakedefine MFEM_USE_RAJA

@@ -38,19 +38,7 @@ if(NOT ADIOS2_FOUND)
  endif()

  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  
-  # adios2 version 2.5.0 
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
-  
-  # adios2 version 2.6.0 and onwards
-  if(NOT ADIOS2_LIBRARY)
-  	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
-  	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
-  	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
-  	if(MFEM_USE_MPI)
-  	  add_definitions(-DADIOS2_USE_MPI)
-  	endif()
-  endif() 
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 

  include(FindPackageHandleStandardArgs)
  find_package_handle_standard_args(ADIOS2
@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - SLEPC_FOUND
-#   - SLEPC_INCLUDE_DIRS
-#   - SLEPC_LIBRARIES
-
-set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
-  "Additional packages required by SLEPc")
-
-include(MfemCmakeUtilities)
-mfem_find_package(SLEPc SLEPC SLEPC_DIR
-  "include" "slepceps.h"
-  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
-  "Paths to headers required by SLEPc."
-  "Libraries required by SLEPc."
-  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
-  CHECK_BUILD SLEPC_VERSION_OK TRUE
-"
-#include \"petsc.h\"
-#include \"slepceps.h\"
-int main()
-{
-  PetscErrorCode ierr;
-  int argc = 0;
-  char** argv = NULL;
-  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
-  EPS eps;
-  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
-  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
-  ierr = SlepcFinalize(); CHKERRQ(ierr);
-  return 0;
-}
-"
-  )
@@ -25,6 +25,5 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  ADD_COMPONENT NVector_ParHyp
    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
-  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
@@ -128,15 +128,7 @@ function(add_mfem_miniapp MFEM_EXE_NAME)
  if (MFEM_USE_CUDA)
    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
      PROPERTY LANGUAGE CUDA)
-    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
-      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
-    else()
-      set(LIST_)
-      foreach(item IN LISTS EXTRA_OPTIONS_LIST)
-        list(APPEND LIST_ "-Xcompiler=${item}")
-      endforeach()
-      set(EXTRA_OPTIONS_LIST ${LIST_})
-    endif()
+    list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
  endif()

  # Actually add the executable
@@ -739,7 +731,7 @@ function(mfem_export_mk_files)
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
-      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2)
  foreach(var ${CONFIG_MK_BOOL_VARS})
@@ -48,9 +48,6 @@
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_SLEPC
-#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -118,9 +118,6 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC

-// Enable functionality based on the SLEPc library
-// #define MFEM_USE_SLEPC
-
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR

@@ -137,9 +134,6 @@
 // Requires a CUDA compiler (nvcc).
 // #define MFEM_USE_CUDA

-// Enable MFEM functionality based on the AMGX library.
-// #define MFEM_USE_AMGX
-
 // Build the AMD GPU/HIP-enabled version of the MFEM library.
 // Requires a HIP compiler (hipcc).
 // #define MFEM_USE_HIP
@@ -37,7 +37,6 @@ MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
-MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
@@ -45,7 +44,6 @@ MFEM_USE_PUMI          = @MFEM_USE_PUMI@
 MFEM_USE_HIOP          = @MFEM_USE_HIOP@
 MFEM_USE_GSLIB         = @MFEM_USE_GSLIB@
 MFEM_USE_CUDA          = @MFEM_USE_CUDA@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_HIP           = @MFEM_USE_HIP@
 MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
@@ -39,7 +39,6 @@ option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
-option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -50,7 +49,7 @@ option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
-option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
+option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" ON)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")
@@ -83,18 +82,11 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
-# and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
@@ -163,10 +155,6 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
    "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")

-set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
-    "Path to the SLEPc main directory.")
-set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
-
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
@@ -187,7 +175,6 @@ set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING

 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AMGX")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")

@@ -133,13 +133,12 @@ MFEM_USE_PUMI          = NO
 MFEM_USE_HIOP          = NO
 MFEM_USE_GSLIB         = NO
 MFEM_USE_CUDA          = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_HIP           = NO
 MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
-MFEM_USE_SIMD          = NO
+MFEM_USE_SIMD          = YES
 MFEM_USE_ADIOS2        = NO

 # Compile and link options for zlib.
@@ -155,10 +154,6 @@ LIBUNWIND_LIB = $(if $(NOTMAC),-lunwind -ldl,)
 HYPRE_DIR = @MFEM_DIR@/../hypre/src/hypre
 HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
-ifeq (YES,$(MFEM_USE_CUDA))
-   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
-endif

 # METIS library configuration
 ifeq ($(MFEM_USE_SUPERLU)$(MFEM_USE_STRUMPACK),NONO)
@@ -194,12 +189,10 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecparhyp and nvecparallel remember to build with MPI_ENABLED=ON
-# and modify cmake variables for hypre for sundials
 SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
 SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
 SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
- -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+ -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
@@ -377,11 +370,6 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

-# AMGX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -L$(CUDA_HOME)/lib64 -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx -Xlinker=-rpath=$(CUDA_HOME)/lib64
-
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -78,14 +78,6 @@ groups_parallel=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -115,14 +107,6 @@ groups_all=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -396,15 +380,10 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   # Strip leading and trailing spaces from $1 and store the result in cmd_line
-   shopt -s extglob
-   local cmd_line="${1##+( )}"
-   cmd_line="${cmd_line%%+( )}"
-   shopt -u extglob
-   eval local cmd=(${cmd_line})
+   local cmd=("$@")
   local res=""
   echo $sep
-   echo "<${group}>" "${cmd_line}"
+   echo "<${group}>" "${cmd[@]}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
      timed_run "${cmd[@]}"
@@ -416,15 +395,15 @@ function go()
   else
      res="${red}FAILED${none}"
   fi
-   printf "[${res}] <${group}> ${cmd_line}\n"
+   printf "[${res}] <${group}> ${cmd[*]}\n"
   if [ "${timing}" == "yes" ]; then
      printf "Run time: %s\n" "${timer}"
      timer=(${timer})
      timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
+      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
      summary=("${summary[@]}" "$line")
   else
-      summary=("${summary[@]}" "[${res}] ${cmd_line}")
+      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
   fi
   echo $sep
 }
@@ -459,7 +438,7 @@ function go_group()
      fi
      for run in "${runs[@]}"; do
         if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
+         eval go \${run_prefix} \${run} \${run_suffix} $output
      done
   done
   ${make} clean-exec
@@ -525,7 +504,7 @@ function echo_run()
 {
   echo "   $@"
   { echo "   $@"; echo "$sep";
-     eval "$@"
+     "$@"
     echo "$sep"; } >> "$echo_log" 2>&1
 }

@@ -545,28 +524,6 @@ function build_all()
   echo_run ${make} config ${mfem_config} || exit 1
   echo_run ${make} ${make_j} || exit 1
   echo_run ${make} ${make_all} ${make_j} || exit 1
-   # Build groups in directories other than the directories built by 'make all':
-   for group_params in "${groups[@]}"; do
-      eval params=(${group_params})
-      group_dir="${params[2]}"
-      case "$group_dir" in
-         (examples*|miniapps*)
-            # Built by 'make all'
-            ;;
-         (*)
-            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
-               echo_run mkdir -p "${group_dir}" || exit 1
-               echo_run cd "${group_dir}" || exit 1
-               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
-            else
-               echo_run cd "${group_dir}" || exit 1
-            fi
-            echo_run ${make} clean || exit 1
-            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
-            echo_run cd "${mfem_build_dir}" || exit 1
-            ;;
-      esac
-   done
 }

 # Function that runs all sample runs, given by the array variable "groups".
@@ -1,38 +1,13 @@
 SetFactory("OpenCASCADE");

-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
-
-// Set the geometry order (1, 2, ..., 9)
-order = 3;
-
-// Set the element type (3 - triangles, 4 - quadrilaterals)
-type = 3;
-
-// Number of radial elements
-nrad = 2;
-
-// Number of azimuthal elements on inner arc
-nazm1 = 3;
-
-// Number of azimuthal elements on outer arc
-nazm2 = 5;
-
-// Note: Using type = 4 with nazm1 != nazm2 can lead to mixed meshes
-//       containing both triangles and quadrilaterals.
-
-// Inner and outer radii
 R1 = 1.0;
 R2 = 2.0;

-// Angular size of the sector
-Phi = Pi/3.0;
-
 Point(1) = {0.0, 0, 0, 1.0};
 Point(2) = {R1, 0, 0, 1.0};
 Point(3) = {R2, 0, 0, 1.0};
-Point(4) = {R1*Cos(Phi), R1*Sin(Phi), 0, 1.0};
-Point(5) = {R2*Cos(Phi), R2*Sin(Phi), 0, 1.0};
+Point(4) = {R1*Cos(Pi/3), R1*Sin(Pi/3), 0, 1.0};
+Point(5) = {R2*Cos(Pi/3), R2*Sin(Pi/3), 0, 1.0};
 Line(1) = {2, 3};
 Line(2) = {4, 5};
 Circle(3) = {2, 1, 4};
@@ -40,23 +15,13 @@ Circle(4) = {3, 1, 5};
 Curve Loop(5) = {1, 4, -2, -3};
 Plane Surface(1) = {5};

-Transfinite Curve{1} = nrad+1;
-Transfinite Curve{2} = nrad+1;
-Transfinite Curve{3} = nazm1+1;
-Transfinite Curve{4} = nazm2+1;
-
-If (nazm1 == nazm2)
-   Transfinite Surface{1};
-EndIf
-
-If (type == 4)
-   Recombine Surface {1};
-EndIf
+Transfinite Curve{1} = 7;
+Transfinite Curve{2} = 7;
+Transfinite Curve{3} = 4;
+Transfinite Curve{4} = 10;

 // Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Phi};
-EndIf
+Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Pi/3};

 // Tag surfaces and volumes with positive integers
 Physical Curve(1) = {3};
@@ -65,22 +30,8 @@ Physical Curve(3) = {1};
 Physical Curve(4) = {2};
 Physical Surface(1) = {1};

-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
-
 // Generate 2D mesh
 Mesh 2;
-SetOrder order;
 Mesh.MshFileVersion = 2.2;

-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-annulus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("annulus-sector-t%01g-o%01g.msh", type, order);
-EndIf
+Save "periodic-annulus-sector.msh";
@@ -2,191 +2,184 @@ $MeshFormat
 2.2 0 8
 $EndMeshFormat
 $Nodes
-136
+55
 1 1 0 0
 2 2 0 0
 3 0.5000000000000001 0.8660254037844386 0
 4 1 1.732050807568877 0
-5 1.5 0 0
-6 1.166666666666667 0 0
-7 1.333333333333333 0 0
+5 1.166666666666667 0 0
+6 1.333333333333333 0 0
+7 1.5 0 0
 8 1.666666666666667 0 0
 9 1.833333333333333 0 0
-10 0.7500000000000002 1.299038105676658 0
-11 0.5833333333333335 1.010362971081845 0
-12 0.6666666666666667 1.154700538379251 0
+10 0.5833333333333335 1.010362971081845 0
+11 0.6666666666666667 1.154700538379251 0
+12 0.7500000000000002 1.299038105676658 0
 13 0.8333333333333335 1.443375672974064 0
 14 0.9166666666666669 1.587713240271471 0
 15 0.9396926207859085 0.3420201433256683 0
 16 0.7660444431189786 0.6427876096865386 0
-17 0.993238357741943 0.1160929141252301 0
-18 0.9730448705798238 0.2306158707424401 0
-19 0.8936326403234125 0.4487991802004617 0
-20 0.8354878114129367 0.5495089780708056 0
-21 0.6862416378687343 0.7273736415730481 0
-22 0.597158591702787 0.8021231927550432 0
-23 1.956295201467611 0.4158233816355181 0
-24 1.827090915285202 0.8134732861515996 0
-25 1.618033988749896 1.175570504584944 0
-26 1.338261212717719 1.486289650954786 0
-27 1.995128100519648 0.1395129474882505 0
-28 1.980536137483141 0.278346201920131 0
-29 1.922523391876638 0.551274711633998 0
-30 1.879385241571817 0.6840402866513373 0
-31 1.765895185717855 0.9389431255717802 0
-32 1.696096192312853 1.059838528466408 0
-33 1.532088886237958 1.285575219373077 0
-34 1.438679600677305 1.389316740917992 0
-35 1.231322950651319 1.576021507213442 0
-36 1.118385806941496 1.658075145110082 0
-37 1.162276263405681 0.6710405135499813 0
-38 1.248615852873337 1.079531485311822 0
-39 1.559209616901855 0.5415673055003691 0
-40 1.478306597054007 0.8535007117539289 0
-41 0.9210953433941653 0.9653302893212266 0
-42 1.296548225291847 0.3150268220262836 0
-43 1.055002035226811 1.358510675893086 0
-44 1.704005774249187 0.2344032256041583 0
-45 0.6403651144647218 0.8991270322967013 0
-46 0.7807302289294435 0.9322286608089638 0
-47 0.864063562262777 1.07656622810637 0
-48 0.8070317811313885 1.187802166891514 0
-49 0.7236984477980553 1.043464599594108 0
-50 1.432182741763949 0.1050089406754279 0
-51 1.364365483527898 0.2100178813508558 0
-52 1.197698816861231 0.2100178813508558 0
-53 1.098849408430616 0.1050089406754279 0
-54 1.265516075097282 0.1050089406754278 0
-55 0.8177280765440409 0.7503018362314346 0
-56 0.869411709969103 0.8578160627763305 0
-57 0.7348318576552288 0.8316090412164392 0
-58 1.177596357123201 0.3240245957927452 0
-59 1.058644488954555 0.3330223695592067 0
-60 1.087610484537871 0.2205785356313179 0
-61 1.267619707955123 0.7318605796179638 0
-62 1.372963152504565 0.7926806456859463 0
-63 1.40174301566045 0.9288443029398934 0
-64 1.325179434266893 1.004187894125858 0
-65 1.219835989717452 0.9433678280578752 0
-66 1.191056126561566 0.8072041708039283 0
-67 1.296399571111008 0.8680242368719107 0
-68 1.532241943619239 0.645545107584889 0
-69 1.505274270336623 0.7495229096694089 0
-70 1.294587381237739 0.6278827775334439 0
-71 1.426898499069797 0.5847250415169065 0
-72 1.399930825787181 0.6887028436014264 0
-73 1.139442349713613 1.041464419981624 0
-74 1.030268846553889 1.003397354651425 0
-75 1.001488983398004 0.8672336973974781 0
-76 1.081882623401843 0.7691371054737297 0
-77 1.110662486557728 0.9053007627276766 0
-78 1.207033584034403 0.5523692830420821 0
-79 1.251790904663125 0.4336980525341829 0
-80 1.384102022495183 0.3905403165176455 0
-81 1.471655819698519 0.4660538110090073 0
-82 1.339344701866461 0.5092115470255447 0
-83 1.73779714915742 0.7228379592678562 0
-84 1.648503383029637 0.6322026323841127 0
-85 1.691571478423774 0.4996526642120855 0
-86 1.823933339945692 0.4577380229238018 0
-87 1.787039416783436 0.5922941012619014 0
-88 1.308379426102925 1.350703595740465 0
-89 1.278497639488131 1.215117540526144 0
-90 1.371755231498856 1.111544491736196 0
-91 1.494894610124376 1.14355749816057 0
-92 1.4064614465962 1.25147448186763 0
-93 1.013887168325833 0.451693600067106 0
-94 1.088081715865757 0.5613670568085436 0
-95 1.03019898997678 0.6616228789288338 0
-96 0.8981217165478794 0.6522052443076862 0
-97 0.9637989050473432 0.5564495572737495 0
-98 1.432367408277627 0.2881522898855752 0
-99 1.568186591263407 0.2612777577448668 0
-100 1.655740388466743 0.3367912522362286 0
-101 1.607475002684299 0.4391792788682989 0
-102 1.519921205480963 0.3636657843769371 0
-103 1.184077913657828 1.17252454883891 0
-104 1.119539974442319 1.265517612365998 0
-105 1.010366471282595 1.2274505470358 0
-106 0.9657309073383804 1.096390418178513 0
-107 1.074904410498104 1.134457483508712 0
-108 1.901335258083062 0.07813440853471942 0
-109 1.802670516166124 0.1562688170694388 0
-110 1.636003849499458 0.156268817069439 0
-111 1.568001924749729 0.07813440853471942 0
-112 1.734668591416396 0.07813440853471944 0
-113 0.8516673450756037 1.318862295748801 0
-114 0.9533346901512071 1.338686485820944 0
-115 1.03666802348454 1.48302405311835 0
-116 1.01833401174227 1.607537430343614 0
-117 0.9350006784089369 1.463199863046207 0
-118 1.710829475874804 0.8268157613523761 0
-119 1.594568036464405 0.8401582365531526 0
-120 1.621535709747021 0.7361804344686326 0
-121 1.52488239428597 0.9608573093642675 0
-122 1.571458191517933 1.068213906974606 0
-123 1.448318812892413 1.036200900550232 0
-124 0.908699126206992 1.207626356963657 0
-125 1.500184666513678 0.1831433492101474 0
-126 1.646765991694905 0.9507607885973723 0
-127 0.9498053499729417 0.7597194708525821 0
-128 1.132839036494479 0.4426958263006443 0
-129 1.149421761057113 1.40110366758032 0
-130 1.243841486887416 1.443696659267553 0
-131 1.134903597606542 1.530869109405537 0
-132 1.872198725728137 0.3553499962917315 0
-133 1.788102249988661 0.2948766109479449 0
-134 1.893223337417325 0.2174207916708467 0
-135 1.213959700272622 1.308110604053232 0
-136 1.739836864206217 0.3972646375800152 0
+17 1.986476715483886 0.2321858282504602 0
+18 1.946089741159648 0.4612317414848793 0
+19 1.879385241571817 0.6840402866513365 0
+20 1.787265280646825 0.8975983604009234 0
+21 1.670975622825874 1.09901795614161 0
+22 1.532088886237958 1.285575219373077 0
+23 1.372483275737469 1.454747283146095 0
+24 1.194317183405575 1.604246385510085 0
+25 1.425989114816062 0.1915326920916892 0
+26 0.8788667344146573 1.13917645290495 0
+27 1.630372059110754 0.7154531062316609 0
+28 1.436395769298814 1.053728612482506 0
+29 1.081023776188756 0.6241293681829633 0
+30 1.168737372335971 1.428012728596308 0
+31 1.821063986059922 0.298149890497067 0
+32 1.234707097211386 0.3469796339295647 0
+33 1.377747393186519 0.6200150626754309 0
+34 1.457047681210906 0.3890895843559762 0
+35 0.917846726184522 0.8957978954532204 0
+36 1.218335619030348 0.9017812086952638 0
+37 1.066623110765233 1.061857005744772 0
+38 1.587029716281926 0.1355955181472859 0
+39 1.744445799211916 0.1441515753740107 0
+40 1.25 0.1443375672974065 0
+41 1.453660070628011 0.8435769396609902 0
+42 1.741367044061892 0.499612708014486 0
+43 1.30550638526547 1.257610469847477 0
+44 1.118213276932792 0.1666674689105279 0
+45 0.9109440214958271 1.306610291787315 0
+46 0.9970618258753989 1.438658589955562 0
+47 0.7499999999999998 1.010362971081845 0
+48 0.7034449005273667 0.8850673702175776 0
+49 1.605449512513618 0.9269067082200894 0
+50 1.561654019115059 0.5298592532912715 0
+51 1.229782222487711 1.096820457143683 0
+52 1.617066998712459 0.3090202662210922 0
+53 1.079645953234324 1.246963713711438 0
+54 1.877063966817811 0.1348974588243076 0
+55 1.055356609656722 1.558136350380461 0
 $EndNodes
 $Elements
-38
-1 26 2 3 1 1 5 6 7
-2 26 2 3 1 5 2 8 9
-3 26 2 4 2 3 10 11 12
-4 26 2 4 2 10 4 13 14
-5 26 2 1 3 1 15 17 18
-6 26 2 1 3 15 16 19 20
-7 26 2 1 3 16 3 21 22
-8 26 2 2 4 2 23 27 28
-9 26 2 2 4 23 24 29 30
-10 26 2 2 4 24 25 31 32
-11 26 2 2 4 25 26 33 34
-12 26 2 2 4 26 4 35 36
-13 21 2 1 1 3 41 10 45 46 47 48 12 11 49
-14 21 2 1 1 5 42 1 50 51 52 53 6 7 54
-15 21 2 1 1 16 41 3 55 56 46 45 22 21 57
-16 21 2 1 1 1 42 15 53 52 58 59 18 17 60
-17 21 2 1 1 37 40 38 61 62 63 64 65 66 67
-18 21 2 1 1 39 40 37 68 69 62 61 70 71 72
-19 21 2 1 1 38 41 37 73 74 75 76 66 65 77
-20 21 2 1 1 37 42 39 78 79 80 81 71 70 82
-21 21 2 1 1 24 39 23 83 84 85 86 29 30 87
-22 21 2 1 1 26 38 25 88 89 90 91 33 34 92
-23 21 2 1 1 15 37 16 93 94 95 96 20 19 97
-24 21 2 1 1 42 44 39 98 99 100 101 81 80 102
-25 21 2 1 1 38 43 41 103 104 105 106 74 73 107
-26 21 2 1 1 2 44 5 108 109 110 111 8 9 112
-27 21 2 1 1 10 43 4 113 114 115 116 14 13 117
-28 21 2 1 1 24 40 39 118 119 69 68 84 83 120
-29 21 2 1 1 38 40 25 64 63 121 122 91 90 123
-30 21 2 1 1 41 43 10 106 105 114 113 48 47 124
-31 21 2 1 1 5 44 42 111 110 99 98 51 50 125
-32 21 2 1 1 25 40 24 122 121 119 118 31 32 126
-33 21 2 1 1 37 41 16 76 75 56 55 96 95 127
-34 21 2 1 1 15 42 37 59 58 79 78 94 93 128
-35 21 2 1 1 4 43 26 116 115 129 130 35 36 131
-36 21 2 1 1 23 44 2 132 133 109 108 27 28 134
-37 21 2 1 1 26 43 38 130 129 104 103 89 88 135
-38 21 2 1 1 39 44 23 101 100 133 132 86 85 136
+108
+1 1 2 3 1 1 5
+2 1 2 3 1 5 6
+3 1 2 3 1 6 7
+4 1 2 3 1 7 8
+5 1 2 3 1 8 9
+6 1 2 3 1 9 2
+7 1 2 4 2 3 10
+8 1 2 4 2 10 11
+9 1 2 4 2 11 12
+10 1 2 4 2 12 13
+11 1 2 4 2 13 14
+12 1 2 4 2 14 4
+13 1 2 1 3 1 15
+14 1 2 1 3 15 16
+15 1 2 1 3 16 3
+16 1 2 2 4 2 17
+17 1 2 2 4 17 18
+18 1 2 2 4 18 19
+19 1 2 2 4 19 20
+20 1 2 2 4 20 21
+21 1 2 2 4 21 22
+22 1 2 2 4 22 23
+23 1 2 2 4 23 24
+24 1 2 2 4 24 4
+25 2 2 1 1 32 40 25
+26 2 2 1 1 25 34 32
+27 2 2 1 1 33 41 36
+28 2 2 1 1 38 52 25
+29 2 2 1 1 33 36 29
+30 2 2 1 1 26 47 35
+31 2 2 1 1 35 37 26
+32 2 2 1 1 25 52 34
+33 2 2 1 1 32 44 40
+34 2 2 1 1 15 32 29
+35 2 2 1 1 15 29 16
+36 2 2 1 1 36 41 28
+37 2 2 1 1 32 33 29
+38 2 2 1 1 50 52 42
+39 2 2 1 1 32 34 33
+40 2 2 1 1 42 52 31
+41 2 2 1 1 43 53 51
+42 2 2 1 1 27 41 33
+43 2 2 1 1 26 53 45
+44 2 2 1 1 18 31 17
+45 2 2 1 1 29 35 16
+46 2 2 1 1 29 36 35
+47 2 2 1 1 24 30 23
+48 2 2 1 1 30 53 43
+49 2 2 1 1 17 54 2
+50 2 2 1 1 4 55 24
+51 2 2 1 1 28 51 36
+52 2 2 1 1 47 48 35
+53 2 2 1 1 36 37 35
+54 2 2 1 1 37 53 26
+55 2 2 1 1 22 28 21
+56 2 2 1 1 20 27 19
+57 2 2 1 1 33 50 27
+58 2 2 1 1 15 44 32
+59 2 2 1 1 18 42 31
+60 2 2 1 1 30 43 23
+61 2 2 1 1 35 48 16
+62 2 2 1 1 31 54 17
+63 2 2 1 1 9 39 8
+64 2 2 1 1 8 38 7
+65 2 2 1 1 7 25 6
+66 2 2 1 1 22 43 28
+67 2 2 1 1 23 43 22
+68 2 2 1 1 39 54 31
+69 2 2 1 1 19 42 18
+70 2 2 1 1 24 55 30
+71 2 2 1 1 27 42 19
+72 2 2 1 1 13 46 14
+73 2 2 1 1 51 53 37
+74 2 2 1 1 39 52 38
+75 2 2 1 1 6 40 5
+76 2 2 1 1 34 52 50
+77 2 2 1 1 12 45 13
+78 2 2 1 1 30 55 46
+79 2 2 1 1 10 47 11
+80 2 2 1 1 8 39 38
+81 2 2 1 1 28 49 21
+82 2 2 1 1 7 38 25
+83 2 2 1 1 41 49 28
+84 2 2 1 1 20 49 27
+85 2 2 1 1 11 26 12
+86 2 2 1 1 27 49 41
+87 2 2 1 1 31 52 39
+88 2 2 1 1 25 40 6
+89 2 2 1 1 2 54 9
+90 2 2 1 1 14 55 4
+91 2 2 1 1 45 53 46
+92 2 2 1 1 45 46 13
+93 2 2 1 1 5 44 1
+94 2 2 1 1 21 49 20
+95 2 2 1 1 46 53 30
+96 2 2 1 1 3 48 10
+97 2 2 1 1 34 50 33
+98 2 2 1 1 36 51 37
+99 2 2 1 1 26 45 12
+100 2 2 1 1 11 47 26
+101 2 2 1 1 27 50 42
+102 2 2 1 1 40 44 5
+103 2 2 1 1 43 51 28
+104 2 2 1 1 10 48 47
+105 2 2 1 1 9 54 39
+106 2 2 1 1 46 55 14
+107 2 2 1 1 1 44 15
+108 2 2 1 1 16 48 3
 $EndElements
 $Periodic
 1
 1 1 2
 Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
-3
+7
+9 14
+6 11
+8 13
 5 10
-1 3
+7 12
 2 4
+1 3
 $EndPeriodic
@@ -1,141 +1,25 @@
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
+SetFactory("OpenCASCADE");

-// Set the geometry order (1, 2, ..., 10 for tetrahedra or 9 for other types)
-order = 3;
+R = 1.5;
+r = 0.5;

-// Set the element type (4 - tetrahedra, 6 - wedges, 8 - hexahedra)
-type = 8;
+Torus(1) = {0,0,0, R, r, Pi/3};

-// Minor and major radii
-R1 = 1.0;
-R2 = 2.0;
+pts() = PointsOf{ Volume{1}; };

-// Side length of interior square
-A1 = 0.8;
-
-// Angular size of the sector
-Phi = Pi/3.0;
-
-// Number of azimuthal elements
-nazm = 3;
-
-// Number of elements around a quarter of the circle
-narc = 2;
-
-// Number of elements between surface and interior square
-nshl = 1;
-
-lc = 0.5;
-a1 = A1 / Sqrt(2.0);
-
-Point(1) = {R2+R1, 0, 0, lc};
-Point(2) = {R2, 0, R1, lc};
-Point(3) = {R2-R1, 0, 0, lc};
-Point(4) = {R2, 0, -R1, lc};
-Point(5) = {R2, 0, 0, lc};
-Point(6) = {R2+a1, 0, 0, lc};
-Point(7) = {R2, 0, a1, lc};
-Point(8) = {R2-a1, 0, 0, lc};
-Point(9) = {R2, 0, -a1, lc};
-
-Circle(1) = {1,5,2};
-Circle(2) = {2,5,3};
-Circle(3) = {3,5,4};
-Circle(4) = {4,5,1};
-
-Line(5) = {6,1};
-Line(6) = {7,2};
-Line(7) = {8,3};
-Line(8) = {9,4};
-
-Line(9) = {6, 7};
-Line(10) = {7, 8};
-Line(11) = {8, 9};
-Line(12) = {9, 6};
-
-Line Loop(101) = {1, -6, -9, 5};
-Line Loop(102) = {2, -7, -10, 6};
-Line Loop(103) = {3, -8, -11, 7};
-Line Loop(104) = {4, -5, -12, 8};
-Line Loop(105) = {9, 10, 11, 12};
-
-Plane Surface(201) = {101};
-Plane Surface(202) = {102};
-Plane Surface(203) = {103};
-Plane Surface(204) = {104};
-Plane Surface(205) = {105};
-
-Transfinite Curve{1} = narc+1;
-Transfinite Curve{2} = narc+1;
-Transfinite Curve{3} = narc+1;
-Transfinite Curve{4} = narc+1;
-
-Transfinite Curve{5} = nshl+1;
-Transfinite Curve{6} = nshl+1;
-Transfinite Curve{7} = nshl+1;
-Transfinite Curve{8} = nshl+1;
-
-Transfinite Curve{9} = narc+1;
-Transfinite Curve{10} = narc+1;
-Transfinite Curve{11} = narc+1;
-Transfinite Curve{12} = narc+1;
-
-If (type == 8)
-   Recombine Surface {201};
-   Recombine Surface {202};
-   Recombine Surface {203};
-   Recombine Surface {204};
-   Recombine Surface {205};
-
-   Transfinite Surface {201} = {1,2,7,6};
-   Transfinite Surface {202} = {2,3,8,7};
-   Transfinite Surface {203} = {3,4,9,8};
-   Transfinite Surface {204} = {4,1,6,9};
-   Transfinite Surface {205} = {6,7,8,9};
-EndIf
-
-If (type == 4)
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm};
-}
-Else
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm}; Recombine;
-}
-EndIf
+Characteristic Length{ pts() } = 0.25;

 // Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Surface{227} = {201} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{249} = {202} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{271} = {203} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{293} = {204} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{315} = {205} Rotate{{0,0,1}, {0,0,0}, Phi};
-EndIf
+Periodic Surface{3} = {2} Rotate{{0,0,1}, {0,0,0}, Pi/3};

 // Tag surfaces and volumes with positive integers
-Physical Surface(1) = {201,202,203,204,205};
-Physical Surface(2) = {227,249,271,293,315};
-Physical Surface(3) = {214,236,258,280};
-Physical Volume(1) = {1,2,3,4,5};
-
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
+Physical Surface(1) = {1};
+Physical Surface(2) = {2};
+Physical Surface(3) = {3};
+Physical Volume(1) = {1};

 // Generate 3D mesh
 Mesh 3;
-SetOrder order;
+
 Mesh.MshFileVersion = 2.2;
-
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-torus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("torus-sector-t%01g-o%01g.msh", type, order);
-EndIf
+Save "periodic-torus-sector.msh";
@@ -1,118 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-20
-1 3 0 1 6 5
-1 3 1 2 7 6
-1 3 2 3 8 7
-1 3 3 4 9 8
-1 3 5 6 11 10
-1 2 6 7 11
-1 2 7 12 11
-1 2 7 8 13
-1 2 7 13 12
-1 3 8 9 14 13
-1 3 10 11 16 15
-1 2 11 12 17
-1 2 11 17 16
-1 2 12 13 17
-1 2 13 18 17
-1 3 13 14 19 18
-1 3 15 16 21 20
-1 3 16 17 22 21
-1 3 17 18 23 22
-1 3 18 19 24 23
-
-boundary
-16
-2 1 0 1
-2 1 1 2
-2 1 2 3
-2 1 3 4
-2 1 21 20
-2 1 22 21
-2 1 23 22
-2 1 24 23
-1 1 5 0
-1 1 10 5
-1 1 15 10
-1 1 20 15
-1 1 4 9
-1 1 9 14
-1 1 14 19
-1 1 19 24
-
-vertices
-25
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 0
-
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0
-0
-0
-0
-0.25
-0.25
-0.25
-0.25
-0.25
-0.5
-0.5
-0.5
-0.5
-0.5
-0.75
-0.75
-0.75
-0.75
-0.75
-1
-1
-1
-1
-1
@@ -770,7 +770,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/examples/pumi \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
-                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
@@ -88,8 +88,8 @@ namespace mfem {
 * - <a class="el" href="ex24p_8cpp_source.html">Example 24p</a>: parallel mixed finite element spaces and interpolators
 * - <a class="el" href="ex25_8cpp_source.html">Example 25</a>: simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
 * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
- * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
+ * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM 
+ * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM 
 *
 * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
@@ -101,9 +101,6 @@ namespace mfem {
 *   and
 *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
 *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
- * - CVODES adjoint examples:
- *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
- *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
 *
 * <H4>PETSc Examples</H4>
 * - Variants of Examples
@@ -143,13 +140,11 @@ namespace mfem {
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
 * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
 * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
 * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
 * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
 * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
 * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
 * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
 * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
 * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
@@ -162,7 +157,7 @@ namespace mfem {
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
 * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions betwen meshes
+ * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -19,7 +19,7 @@ html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
 	@doxygen $(DOXYGEN_CONF)
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
-	@cat warnings.log 1>&2
+	@cat warnings.log
 	@# Generate the log of undocumented methods
 	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null

@@ -91,7 +91,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
    add_test(NAME ${TEST_NAME}_ser
      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+    add_test(NAME ${TEST_NAME}_np=4
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,22 +101,13 @@ endforeach()

 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+  add_test(NAME ex11p_strumpack_np=4
    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
    ${MPIEXEC_PREFLAGS}
    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
    ${MPIEXEC_POSTFLAGS})
 endif()

-# If SuperLU_DIST is enabled, add a test run that uses it.
-if (MFEM_USE_SUPERLU)
-  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-    ${MPIEXEC_POSTFLAGS})
-endif()
-
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
@@ -1,38 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "preconditioner": {
-            "print_grid_stats": 1,
-            "print_vis_data": 0,
-            "solver": "AMG",
-            "smoother": {
-                "scope": "jacobi",
-                "solver": "BLOCK_JACOBI",
-                "relaxation_factor": 0.7,
-                "monitor_residual": 0,
-                "print_solve_stats": 0
-            },
-            "print_solve_stats": 0,
-            "presweeps": 1,
-            "interpolator": "D2",
-            "max_row_sum" : 0.9,
-            "strength_threshold" : 0.25,
-            "max_iters": 1,
-            "monitor_residual": 0,
-            "store_res_history": 0,
-            "scope": "amg",
-            "max_levels": 100,
-            "cycle": "V",
-            "postsweeps": 1
-        },
-        "solver": "PCG",
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "max_iters": 100,
-        "monitor_residual": 1,
-        "convergence": "RELATIVE_MAX",
-        "scope": "main",
-        "tolerance": 1e-12,
-        "norm": "L2"
-    }
-}
@@ -1,271 +0,0 @@
-//                                MFEM Example 1
-//
-// Compile with: make ex1
-//
-// Sample runs:  ex1 -m ../../data/square-disc.mesh
-//               ex1 -m ../../data/star.mesh
-//               ex1 -m ../../data/star-mixed.mesh
-//               ex1 -m ../../data/escher.mesh
-//               ex1 -m ../../data/fichera.mesh
-//               ex1 -m ../../data/fichera-mixed.mesh
-//               ex1 -m ../../data/toroid-wedge.mesh
-//               ex1 -m ../../data/periodic-annulus-sector.msh
-//               ex1 -m ../../data/periodic-torus-sector.msh
-//               ex1 -m ../../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../../data/star-surf.mesh
-//               ex1 -m ../../data/square-disc-surf.mesh
-//               ex1 -m ../../data/inline-segment.mesh
-//               ex1 -m ../../data/amr-quad.mesh
-//               ex1 -m ../../data/amr-hex.mesh
-//               ex1 -m ../../data/fichera-amr.mesh
-//               ex1 -m ../../data/mobius-strip.mesh
-//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               ex1 -pa -d cuda
-//               ex1 -pa -d raja-cuda
-//               ex1 -pa -d occa-cuda
-//               ex1 -pa -d raja-omp
-//               ex1 -pa -d occa-omp
-//               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
-//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cpu
-//               ex1 -m ../../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
-//
-// AmgX sample runs:
-//               ./ex1 --amgx-file multi_gs.json --amgx-solver
-//               ./ex1 --amgx-file precon.json --amgx-preconditioner
-//               ./ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
-//               ./ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_solver = true;
-   const char* amgx_json_file = ""; // jason file for amgx
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_solver, "--amgx-solver", "--amgx-solver",
-                  "--amgx-preconditioner",
-                  "--amgx-preconditioner",
-                  "Configure AMGX as solver or preconditioner.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   MFEM_VERIFY(!pa && strcmp(amgx_json_file,"") != 0,
-               "An AmgX json file is needed for this example \n");
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
-      AmgXSolver amgx;
-
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-      amgx.InitSerial();
-      amgx.SetOperator(*A.As<SparseMatrix>());
-
-      if (amgx_solver)
-      {
-         amgx.Mult(B,X);
-      }
-      else
-      {
-         PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
-      }
-
-   }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}
@@ -1,322 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh -o -1 -sc
-//
-// Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -pa -d occa-cuda
-//               mpirun -np 4 ex1p -pa -d raja-omp
-//               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -m ../../data/beam-tet.mesh -pa -d ceed-cpu
-//
-// AmgX sample runs:
-//
-//               mpirun -n 40 ./ex1p --amgx-file amg_pcg.json
-//               lrun -n 4 ./ex1p --amgx-file amg_pcg.json --amgx-mpi-gpu-exclusive
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_mpi_teams = true;
-   const char* amgx_json_file = ""; // jason file for amgx
-   int ndevices = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_mpi_teams, "--amgx-mpi-teams", "--amgx-mpi-teams",
-                  "--amgx-mpi-gpu-exclusive", "--amgx-mpi-gpu-exclusive",
-                  "Create MPI teams when using AMGX.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&ndevices, "-nd","--nd","Number of GPU devices.");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-
-      MFEM_VERIFY(!pa == true && strcmp(amgx_json_file,"") != 0,
-                  "An AmgX json file is needed for this example \n");
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B.
-   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * With partial assembly, use Jacobi smoothing, for now.
-   Solver *prec = NULL;
-   if (!pa)
-   {
-
-      AmgXSolver amgx;
-
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-
-      if (amgx_mpi_teams)
-      {
-         //Forms MPI teams to load balance between mpi ranks and gpus
-         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
-      }
-      else
-      {
-         //Assumes MPI == number of devices
-         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
-      }
-
-      amgx.SetOperator(*A.As<HypreParMatrix>());
-
-      amgx.Mult(B, X);
-
-   }
-   else
-   {
-
-      if (UsesTensorBasis(fespace))
-      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-
-   }
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,72 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_AMGX),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with AMGX)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial AMGX example
-PARALLEL_NAME := Parallel AMGX example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not build)
-
-clean: clean-build
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
@@ -1,24 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 10000,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0000001,
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "monitor_residual": 1,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}
@@ -1,21 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 2,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}
@@ -34,8 +34,7 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -71,7 +70,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool amgx = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -83,8 +81,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
-                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -106,8 +102,8 @@ int main(int argc, char *argv[])
   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -115,10 +111,10 @@ int main(int argc, char *argv[])
   //    elements.
   {
      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

@@ -126,99 +122,71 @@ int main(int argc, char *argv[])
   //    Lagrange finite elements of the specified order. If order < 1, we
   //    instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (mesh.GetNodes())
+   else if (mesh->GetNodes())
   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = mesh->GetNodes()->OwnFEC();
      cout << "Using isoparametric FEs: " << fec->Name() << endl;
   }
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   FiniteElementSpace fespace(&mesh, fec);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
+        << fespace->GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
+   if (mesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
+   LinearForm *b = new LinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero,
   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
+   GridFunction x(fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   BilinearForm *a = new BilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
   //     conditions, applying conforming constraints for non-conforming AMR,
   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   cout << "Size of linear system: " << A->Height() << endl;

   // 11. Solve the linear system A X = B.
-   if (pa)
-   {
-      // Jacobi preconditioning in partial assembly mode
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   else if (amgx)
-   {
-#if defined(MFEM_USE_AMGX)
-      bool amgx_verbose = false;
-      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
-      amgx.SetOperator(*A.As<SparseMatrix>());
-      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
-#else
-      mfem_error("MFEM not configured with AMGX \n");
-#endif
-   }
-   else
+   if (!pa)
   {
 #ifndef MFEM_USE_SUITESPARSE
      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
@@ -232,15 +200,27 @@ int main(int argc, char *argv[])
      umf_solver.Mult(B, X);
 #endif
   }
+   else // Jacobi preconditioning in partial assembly mode
+   {
+      if (UsesTensorBasis(*fespace))
+      {
+         OperatorJacobiSmoother M(*a, ess_tdof_list);
+         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
+      }
+      else
+      {
+         CG(*A, B, X, 1, 400, 1e-12, 0.0);
+      }
+   }

   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 13. Save the refined mesh and the solution. This output can be viewed later
   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);
@@ -252,14 +232,15 @@ int main(int argc, char *argv[])
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
   }

   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;

   return 0;
 }
@@ -88,6 +88,8 @@ private:
   Vector funval2;
   Vector nor;
   Vector fluxN;
+   IntegrationPoint eip1;
+   IntegrationPoint eip2;

 public:
   FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -422,16 +424,19 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
   {
      const IntegrationPoint &ip = ir->IntPoint(i);

-      Tr.SetAllIntPoints(&ip); // set face and element int. points
+      Tr.Loc1.Transform(ip, eip1);
+      Tr.Loc2.Transform(ip, eip2);

      // Calculate basis functions on both elements at the face
-      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+      el1.CalcShape(eip1, shape1);
+      el2.CalcShape(eip2, shape2);

      // Interpolate elfun at the point
      elfun1_mat.MultTranspose(shape1, funval1);
      elfun2_mat.MultTranspose(shape2, funval2);

+      Tr.SetIntPoint(&ip);
+
      // Get the normal vector and the flux on the face
      CalcOrtho(Tr.Jacobian(), nor);
      const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);
@@ -32,8 +32,7 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -73,7 +72,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
-   bool amgx = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -85,8 +83,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx, "-amgx", "--amgx-precon", "-no-amgx",
-                  "--no-amgx-precon", "Use AmgX V-cycle as preconditioner for CG.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -115,8 +111,8 @@ int main(int argc, char *argv[])
   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -124,23 +120,23 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
   {
      int par_ref_levels = 2;
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh.UniformRefinement();
+         pmesh->UniformRefinement();
      }
   }

@@ -148,16 +144,13 @@ int main(int argc, char *argv[])
   //    use continuous Lagrange finite elements of the specified order. If
   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (pmesh.GetNodes())
+   else if (pmesh->GetNodes())
   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = pmesh->GetNodes()->OwnFEC();
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -166,10 +159,9 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -180,67 +172,56 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
+   if (pmesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 9. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
+   ParLinearForm *b = new ParLinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 10. Define the solution vector x as a parallel finite element grid function
   //     corresponding to fespace. Initialize x with initial guess of zero,
   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
+   ParGridFunction x(fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * If AmgX is available solve using amg preconditioner.
   //     * With partial assembly, use Jacobi smoothing, for now.
   Solver *prec = NULL;
   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
      }
   }
-   else if (amgx)
-   {
-#if defined(MFEM_USE_AMGX)
-      bool amgx_verbose = false;
-      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
-                            amgx_verbose);
-#else
-      mfem_error("MFEM not configured with AMGX \n");
-#endif
-   }
   else
   {
      prec = new HypreBoomerAMG;
@@ -256,7 +237,7 @@ int main(int argc, char *argv[])

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
@@ -267,7 +248,7 @@ int main(int argc, char *argv[])

      ofstream mesh_ofs(mesh_name.str().c_str());
      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
+      pmesh->Print(mesh_ofs);

      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
@@ -282,14 +263,16 @@ int main(int argc, char *argv[])
      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
+      sol_sock << "solution\n" << *pmesh << x << flush;
   }

   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
   MPI_Finalize();

   return 0;
@@ -1,503 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../data/star.mesh
-//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
-//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
-//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-double u_exact(const Vector &x);
-double f_exact(const Vector &x);
-
-// #define FORM_DEFINITE
-#define USE_GMRES
-
-#define USE_CSL
-
-#define K2 250.0
-
-int dim;
-double kappa;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool visualization = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-
-   kappa = 2.0 * M_PI;
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-
-   {
-      double minsize = pmesh->GetElementSize(0);
-      double maxsize = minsize;
-      for (int i=1; i<pmesh->GetNE(); ++i)
-      {
-         const double size_i = pmesh->GetElementSize(i);
-         minsize = std::min(minsize, size_i);
-         maxsize = std::max(maxsize, size_i);
-      }
-
-      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
-           << endl;
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm *b = new ParLinearForm(fespace);
-
-   //ConstantCoefficient bcoef(1.0);
-   FunctionCoefficient bcoef(f_exact);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   ConstantCoefficient neg(-K2);
-   ConstantCoefficient pos(K2);
-   b->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   // 10. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddDomainIntegrator(new MassIntegrator(neg));
-
-#ifdef FORM_DEFINITE
-   ParBilinearForm *adef = new ParBilinearForm(fespace);
-   adef->AddDomainIntegrator(new DiffusionIntegrator(one));
-   adef->AddDomainIntegrator(new MassIntegrator(pos));
-
-   if (static_cond) { adef->EnableStaticCondensation(); }
-   adef->Assemble();
-
-   ParGridFunction xdef(fespace);
-   xdef = 0.0;
-
-   ParLinearForm *bdef = new ParLinearForm(fespace);
-   bdef->AddDomainIntegrator(new DomainLFIntegrator(bcoef));
-   bdef->Assemble();
-
-   HypreParMatrix Adef;
-   Vector Bdef, Xdef;
-   adef->FormLinearSystem(ess_tdof_list, xdef, *bdef, Adef, Xdef, Bdef);
-#endif
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   //A.Print("helmholtz");
-
-   // 12. Define and apply a parallel PCG solver for AX=B with the BoomerAMG
-   //     preconditioner from hypre.
-#ifdef FORM_DEFINITE
-   HypreSolver *amg = new HypreBoomerAMG(Adef);
-#else
-   HypreSolver *amg = new HypreBoomerAMG(A);
-#endif
-
-   const bool fullDirect = true;
-
-   if (fullDirect)
-   {
-#ifdef USE_CSL
-      Vector Bdef, Xdef;
-
-      ParBilinearForm *Mform =  new ParBilinearForm(fespace);
-      Mform->AddDomainIntegrator(new MassIntegrator(pos));
-      Mform->Assemble();
-
-      HypreParMatrix Mmat, Smat, Mcopy;
-      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
-      Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
-                              Bdef);  // There must be a better way than creating two identical matrices.
-
-      ParBilinearForm *Sform =  new ParBilinearForm(fespace);
-      Sform->AddDomainIntegrator(new DiffusionIntegrator(one));
-      Sform->Assemble();
-
-      Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
-
-      const double beta1 = 1.0;
-      const double beta2 = 1.0;
-
-      Mmat *= -beta1;
-
-      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
-
-      Mcopy *= beta2;
-
-      ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
-
-      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
-
-      Array<int> block_trueOffsets(3); // number of variables + 1
-      block_trueOffsets[0] = 0;
-      block_trueOffsets[1] = fespace->TrueVSize();
-      block_trueOffsets[2] = fespace->TrueVSize();
-      block_trueOffsets.PartialSum();
-
-      // Note that B is of true size.
-      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
-                  trueRhs(block_trueOffsets);
-
-      trueRhs.GetBlock(0) = B;
-      trueRhs.GetBlock(1) = 0.0;
-
-      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      //Solver * precond = strumpack;
-
-      // strumpack->Mult(B, X);
-
-      BlockOperator blockDiagA(block_trueOffsets);
-      for (int i=0; i<2; ++i)
-      {
-         blockDiagA.SetDiagonalBlock(i, &A);
-      }
-
-      ProductOperator prod(&blockDiagA, strumpack, false, false);
-      //GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
-      BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
-
-      gmres->SetOperator(prod);
-      gmres->SetRelTol(1e-8);
-      gmres->SetMaxIter(10000);
-      gmres->SetPrintLevel(1);
-
-      gmres->Mult(trueRhs, trueY);
-      strumpack->Mult(trueY, trueX);
-
-      X = trueX.GetBlock(0);
-      double xim2 = trueX.GetBlock(1).Norml2();
-      xim2 *= xim2;
-      double sumxim2 = 0.0;
-
-      MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
-
-      if (myid == 0)
-      {
-         cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
-              sqrt(sumxim2) << endl;
-      }
-
-      delete gmres;
-      delete strumpack;
-      delete Arow;
-#else
-      Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      //Solver * precond = strumpack;
-
-      strumpack->Mult(B, X);
-
-      delete strumpack;
-      delete Arow;
-#endif
-   }
-   else
-   {
-#ifdef USE_GMRES
-      HypreGMRES *gmres = new HypreGMRES(A);
-      gmres->SetTol(1e-12);
-      gmres->SetMaxIter(1000);
-      gmres->SetPrintLevel(10);
-      gmres->SetPreconditioner(*amg);
-      gmres->Mult(B, X);
-      delete gmres;
-#else
-      HyprePCG *pcg = new HyprePCG(A);
-      pcg->SetTol(1e-12);
-      pcg->SetMaxIter(100);
-      pcg->SetPrintLevel(2);
-      pcg->SetPreconditioner(*amg);
-      pcg->Mult(B, X);
-#endif
-   }
-
-   /*
-   HYPRE_ParCSRMatrix* amgP = amg->Get_Restriction();
-   HypreParMatrix P0(amgP[0], false);
-   HypreParMatrix P1(amgP[1], false);
-   HypreParMatrix P2(amgP[2], false);
-   //HypreParMatrix P3(amgP[3], false);
-
-   P0.Print("P0");
-   */
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // Compute and print the L^2 norm of the error.
-   {
-      FunctionCoefficient uex(u_exact);
-
-      double err = x.ComputeL2Error(uex);
-      double xnrm = x.ComputeL2Error(zero);
-      ParGridFunction zerogf(fespace);
-      zerogf = 0.0;
-      double normE = zerogf.ComputeL2Error(uex);
-      if (myid == 0)
-      {
-         cout << "|| E_h - E ||_{L^2} = " << err << endl;
-         cout << "|| E_h ||_{L^2} = " << xnrm << endl;
-         cout << "|| E ||_{L^2} = " << normE << endl;
-      }
-   }
-
-   // 14. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 15. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 16. Free the used memory.
-   //delete pcg;
-   delete amg;
-   delete a;
-   delete b;
-   delete fespace;
-   if (order > 0) { delete fec; }
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-
-double u_exact(const Vector & x)
-{
-   double xi(x(0));
-   double yi(x(1));
-   double zi(1.0);
-
-   if (x.Size() == 3)
-   {
-      zi = x(2);
-   }
-
-   return sin(kappa*xi)*sin(kappa*yi)*sin(kappa*zi);
-}
-
-double f_exact(const Vector &x)
-{
-   double xi(x(0));
-   double yi(x(1));
-   double zi(1.0);
-
-   if (x.Size() == 3)
-   {
-      zi = x(2);
-   }
-
-   const double s = 1.0;
-
-   return ((3.0*kappa*kappa) - (s*K2)) * sin(kappa*xi)*sin(kappa*yi)*sin(
-             kappa*zi) / s;
-}
@@ -6,20 +6,13 @@
 //               ex22 -m ../data/inline-tri.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
 //               ex22 -m ../data/inline-tet.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -83,8 +76,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -115,10 +106,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -148,18 +135,13 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase resolution. In this example we do
+   // 3. Refine the mesh to increase resolution. In this example we do
   //    'ref_levels' of uniform refinement where the user specifies
   //    the number of levels with the '-r' option.
   for (int l = 0; l < ref_levels; l++)
@@ -167,7 +149,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
   //    order.
   if (dim == 1 && prob != 0 )
@@ -189,7 +171,7 @@ int main(int argc, char *argv[])
   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
        << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined based on the type
   //    of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -201,12 +183,12 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
   ComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 8. Define the solution vector u as a complex finite element grid function
+   // 7. Define the solution vector u as a complex finite element grid function
   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
   //    the exact solution if it is known.
   ComplexGridFunction u(fespace);
@@ -228,6 +210,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -280,7 +263,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 9. Set up the sesquilinear form a(.,.) on the finite element space
+   // 8. Set up the sesquilinear form a(.,.) on the finite element space
   //    corresponding to the damped harmonic oscillator operator of the
   //    appropriate type:
   //
@@ -299,7 +282,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -323,7 +305,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 9a. Set up the bilinear form for the preconditioner corresponding to the
+   // 8a. Set up the bilinear form for the preconditioner corresponding to the
   //     appropriate operator
   //
   //      0) A scalar H1 field
@@ -336,8 +318,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   BilinearForm *pcOp = new BilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
   switch (prob)
   {
      case 0:
@@ -358,9 +338,9 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 10. Assemble the form and the corresponding linear system, applying any
-   //     necessary transformations such as: assembly, eliminating boundary
-   //     conditions, conforming constraints for non-conforming AMR, etc.
+   // 9. Assemble the form and the corresponding linear system, applying any
+   //    necessary transformations such as: assembly, eliminating boundary
+   //    conditions, conforming constraints for non-conforming AMR, etc.
   a->Assemble();
   pcOp->Assemble();

@@ -368,17 +348,28 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;

-   cout << "Size of linear system: " << A->Width() << endl << endl;
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

-   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
+   {
+      ComplexSparseMatrix * Asp =
+         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
+
+      cout << "Size of linear system: "
+           << 2 * Asp->real().Width() << endl << endl;
+   }
+
+   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
   //     preconditioner based on the appropriate sparse smoother.
   {
      Array<int> blockOffsets;
      blockOffsets.SetSize(3);
      blockOffsets[0] = 0;
-      blockOffsets[1] = A->Height() / 2;
-      blockOffsets[2] = A->Height() / 2;
+      blockOffsets[1] = PCOp.Ptr()->Height();
+      blockOffsets[2] = PCOp.Ptr()->Height();
      blockOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockOffsets);
@@ -386,31 +377,22 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      double s = 1.0;
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
+         case 0:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+         case 1:
+            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
+            s = -1.0;
+            break;
+         case 2:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+
+         default: break; // This should be unreachable
      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 1:
-               pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 2:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            default:
-               break; // This should be unreachable
-         }
-      }
-      double s = (prob != 1) ? 1.0 : -1.0;
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
                                s:-s);
@@ -428,7 +410,7 @@ int main(int argc, char *argv[])
      gmres.Mult(B, U);
   }

-   // 12. Recover the solution as a finite element grid function and compute the
+   // 11. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a->RecoverFEMSolution(U, b, u);

@@ -460,7 +442,7 @@ int main(int argc, char *argv[])
      cout << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -475,7 +457,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -534,7 +516,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -7,19 +7,12 @@
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
 //               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -48,6 +41,7 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.

+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -90,8 +84,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -124,10 +116,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -164,24 +152,19 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ser_ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -191,7 +174,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
   //    the specified order.
   if (dim == 1 && prob != 0 )
@@ -219,7 +202,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    based on the type of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -231,14 +214,14 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
   ParComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 10. Define the solution vector u as a parallel complex finite element grid
-   //     function corresponding to fespace. Initialize u with initial guess of
-   //     1+0i or the exact solution if it is known.
+   // 9. Define the solution vector u as a parallel complex finite element grid
+   //    function corresponding to fespace. Initialize u with initial guess of
+   //    1+0i or the exact solution if it is known.
   ParComplexGridFunction u(fespace);
   ParComplexGridFunction * u_exact = NULL;
   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -258,6 +241,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -312,7 +296,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
   //     space corresponding to the damped harmonic oscillator operator of the
   //     appropriate type:
   //
@@ -331,7 +315,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -355,7 +338,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 11a. Set up the parallel bilinear form for the preconditioner
+   // 10a. Set up the parallel bilinear form for the preconditioner
   //      corresponding to the appropriate operator
   //
   //      0) A scalar H1 field
@@ -368,7 +351,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   ParBilinearForm *pcOp = new ParBilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -389,7 +371,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
@@ -400,22 +382,30 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;
+
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

   if (myid == 0)
   {
+      ComplexHypreParMatrix * Ahyp =
+         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
+
      cout << "Size of linear system: "
-           << 2 * fespace->GlobalTrueVSize() << endl << endl;
+           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
   }

-   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
   //     diagonal preconditioner based on the appropriate multigrid
   //     preconditioner from hypre.
   {
      Array<int> blockTrueOffsets;
      blockTrueOffsets.SetSize(3);
      blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = A->Height() / 2;
-      blockTrueOffsets[2] = A->Height() / 2;
+      blockTrueOffsets[1] = PCOp.Ptr()->Height();
+      blockTrueOffsets[2] = PCOp.Ptr()->Height();
      blockTrueOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
@@ -423,34 +413,25 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
-      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-               break;
-            case 1:
+         case 0:
+            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+            break;
+         case 1:
+            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
+            break;
+         case 2:
+            if (dim == 2 )
+            {
               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               break;
-            case 2:
-               if (dim == 2 )
-               {
-                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               else
-               {
-                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               break;
-            default: break; // This should be unreachable
-         }
+            }
+            else
+            {
+               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
+            }
+            break;
+         default: break; // This should be unreachable
      }
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
@@ -468,7 +449,7 @@ int main(int argc, char *argv[])
      fgmres.SetPrintLevel(1);
      fgmres.Mult(B, U);
   }
-   // 14. Recover the parallel grid function corresponding to U. This is the
+   // 13. Recover the parallel grid function corresponding to U. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(U, b, u);

@@ -503,7 +484,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -523,7 +504,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -588,7 +569,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -7,7 +7,6 @@
 //               ex24 -m ../data/beam-tet.mesh
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               ex24 -m ../data/escher.mesh
 //               ex24 -m ../data/escher.mesh -o 2
 //               ex24 -m ../data/fichera.mesh
@@ -25,13 +24,12 @@
 //               ex24 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces, with two variants:
 //
 //               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
+//               2) (div v, q) for v in H(div) tested against q in L_2
 //
-//               Using different approaches, we project the gradient, curl, or
+//               Using different approaches, we project the gradient or
 //               divergence to the appropriate space.
 //
 //               We recommend viewing examples 1, 3, and 5 before viewing this
@@ -47,11 +45,8 @@ using namespace mfem;
 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
 double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -70,7 +65,7 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
+                  "Choose between 0: H(Curl) or 1: H(Div)");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -88,7 +83,6 @@ int main(int argc, char *argv[])
      return 1;
   }
   args.PrintOptions(cout);
-   kappa = freq * M_PI;

   // 2. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -125,15 +119,10 @@ int main(int argc, char *argv[])
      trial_fec = new H1_FECollection(order, dim);
      test_fec = new ND_FECollection(order, dim);
   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
   else
   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
+      trial_fec = new RT_FECollection(order - 1, dim);
+      test_fec = new L2_FECollection(order - 1, dim);
   }

   FiniteElementSpace trial_fes(mesh, trial_fec);
@@ -147,12 +136,6 @@ int main(int argc, char *argv[])
      cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
      cout << "Number of H1 finite element unknowns: " << trial_size << endl;
   }
-   else if (prob == 1)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-      cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-           endl;
-   }
   else
   {
      cout << "Number of Raviart-Thomas finite element unknowns: "
@@ -167,18 +150,12 @@ int main(int argc, char *argv[])
   GridFunction x(&test_fes);
   FunctionCoefficient p_coef(p_exact);
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
   FunctionCoefficient divgradp_coef(div_gradp_exact);

   if (prob == 0)
   {
      gftrial.ProjectCoefficient(p_coef);
   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
   else
   {
      gftrial.ProjectCoefficient(gradp_coef);
@@ -202,11 +179,6 @@ int main(int argc, char *argv[])
      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
   else
   {
      a.AddDomainIntegrator(new MassIntegrator(one));
@@ -272,10 +244,6 @@ int main(int argc, char *argv[])
   {
      dlo.AddDomainInterpolator(new GradientInterpolator());
   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
   else
   {
      dlo.AddDomainInterpolator(new DivergenceInterpolator());
@@ -290,10 +258,6 @@ int main(int argc, char *argv[])
   {
      exact_proj.ProjectCoefficient(gradp_coef);
   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
   else
   {
      exact_proj.ProjectCoefficient(divgradp_coef);
@@ -312,21 +276,8 @@ int main(int argc, char *argv[])
      cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
           "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
      cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-           " ||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-           "||_{L_2} = " << errProj << '\n' << endl;
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in H(div): "
-           "|| E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
           "||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
+      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
           "||_{L_2} = " << errProj << '\n' << endl;
   }
   else
@@ -344,7 +295,7 @@ int main(int argc, char *argv[])

      cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
           "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v "
+      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
           "||_{L_2} = " << errInterp << '\n' << endl;
      cout << " Projection f_h of exact div v in L_2: || f_h - div v "
           "||_{L_2} = " << errProj << '\n' << endl;
@@ -420,33 +371,3 @@ double div_gradp_exact(const Vector &x)

   return 0.0;
 }
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -6,8 +6,7 @@
 //               mpirun -np 4 ex24p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 2
+//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -p 1 -pa
 //               mpirun -np 4 ex24p -m ../data/escher.mesh
 //               mpirun -np 4 ex24p -m ../data/escher.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/fichera.mesh
@@ -25,13 +24,12 @@
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces, with two variants:
 //
 //               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
+//               2) (div v, q) for v in H(div) tested against q in L_2
 //
-//               Using different approaches, we project the gradient, curl, or
+//               Using different approaches, we project the gradient or
 //               divergence to the appropriate space.
 //
 //               We recommend viewing examples 1, 3, and 5 before viewing this
@@ -47,11 +45,8 @@ using namespace mfem;
 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
 double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -76,7 +71,7 @@ int main(int argc, char *argv[])
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
+                  "Choose between 0: H(Curl) or 1: H(Div)");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -101,7 +96,6 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   kappa = freq * M_PI;

   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -153,15 +147,10 @@ int main(int argc, char *argv[])
      trial_fec = new H1_FECollection(order, dim);
      test_fec = new ND_FECollection(order, dim);
   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
   else
   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
+      trial_fec = new RT_FECollection(order - 1, dim);
+      test_fec = new L2_FECollection(order - 1, dim);
   }

   ParFiniteElementSpace trial_fes(pmesh, trial_fec);
@@ -177,12 +166,6 @@ int main(int argc, char *argv[])
         cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
         cout << "Number of H1 finite element unknowns: " << trial_size << endl;
      }
-      else if (prob == 1)
-      {
-         cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-         cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-              endl;
-      }
      else
      {
         cout << "Number of Raviart-Thomas finite element unknowns: "
@@ -198,18 +181,12 @@ int main(int argc, char *argv[])
   ParGridFunction x(&test_fes);
   FunctionCoefficient p_coef(p_exact);
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
   FunctionCoefficient divgradp_coef(div_gradp_exact);

   if (prob == 0)
   {
      gftrial.ProjectCoefficient(p_coef);
   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
   else
   {
      gftrial.ProjectCoefficient(gradp_coef);
@@ -233,11 +210,6 @@ int main(int argc, char *argv[])
      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
   else
   {
      a.AddDomainIntegrator(new MassIntegrator(one));
@@ -321,10 +293,6 @@ int main(int argc, char *argv[])
   {
      dlo.AddDomainInterpolator(new GradientInterpolator());
   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
   else
   {
      dlo.AddDomainInterpolator(new DivergenceInterpolator());
@@ -339,10 +307,6 @@ int main(int argc, char *argv[])
   {
      exact_proj.ProjectCoefficient(gradp_coef);
   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
   else
   {
      exact_proj.ProjectCoefficient(divgradp_coef);
@@ -360,27 +324,11 @@ int main(int argc, char *argv[])

      if (myid == 0)
      {
-         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl)"
-              ": || E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad"
-              " p ||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-              "||_{L_2} = " << errProj << '\n' << endl;
-      }
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in "
-              "H(div): || E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
+         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
+              "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
+         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
              "||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
+         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
              "||_{L_2} = " << errProj << '\n' << endl;
      }
   }
@@ -402,7 +350,7 @@ int main(int argc, char *argv[])
         cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
              "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
         cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
-              " ||_{L_2} = " << errInterp << '\n' << endl;
+              "||_{L_2} = " << errInterp << '\n' << endl;
         cout << " Projection f_h of exact div v in L_2: || f_h - div v "
              "||_{L_2} = " << errProj << '\n' << endl;
      }
@@ -488,33 +436,3 @@ double div_gradp_exact(const Vector &x)

   return 0.0;
 }
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -82,27 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -368,19 +365,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -392,22 +389,27 @@ int main(int argc, char *argv[])
   //     applying any necessary transformations such as: assembly, eliminating
   //     boundary conditions, applying conforming constraints for
   //     non-conforming AMR, etc.
-   a.Assemble(0);
+   a.Assemble();

-   OperatorPtr A;
+   OperatorHandle Ah;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

-   // 13. Solve using a direct or an iterative solver
+   // 13. Transform to monolithic SparseMatrix
+   SparseMatrix *A = Ah.As<ComplexSparseMatrix>()->GetSystemMatrix();
+
+   cout << "Size of linear system: " << A->Height() << endl;
+
+   // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
   {
-      ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
-      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      csolver.SetPrintLevel(1);
-      csolver.Mult(B, X);
+      UMFPackSolver  solver(*A);
+      solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      solver.Mult(B, X);
   }
 #else
-   // 13a. Set up the Bilinear form a(.,.) for the preconditioner
+
+   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
   //    Domain:   1/mu (Curl E, Curl F) + omega^2 * epsilon (E,F)
@@ -422,23 +424,23 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

      prec.Assemble();

-      OperatorPtr PCOpAh;
+      OperatorHandle PCOpAh;
      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);

-      // 13b. Define and apply a GMRES solver for AU=B with a block diagonal
+      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
      //      preconditioner based on the Gauss-Seidel sparse smoother.
      Array<int> offsets(3);
      offsets[0] = 0;
@@ -465,15 +467,17 @@ int main(int argc, char *argv[])
   }
 #endif

-   // 14. Recover the solution as a finite element grid function and compute the
+   // 15. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a.RecoverFEMSolution(X, b, x);

   // If exact is known compute the error
   if (exact_known)
   {
+      ComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -502,7 +506,7 @@ int main(int argc, char *argv[])
           << sqrt(L2Error_Re*L2Error_Re + L2Error_Im*L2Error_Im) << "\n\n";
   }

-   // 15. Save the refined mesh and the solution. This output can be viewed
+   // 16. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("ex25.mesh");
@@ -517,7 +521,7 @@ int main(int argc, char *argv[])
      x.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      // Define visualization keys for GLVis (see GLVis documentation)
@@ -568,7 +572,8 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 18. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -766,7 +771,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -777,13 +782,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -794,13 +800,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -811,13 +818,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -831,18 +839,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -855,18 +864,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -879,13 +889,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -82,27 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -396,19 +393,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -422,15 +419,21 @@ int main(int argc, char *argv[])
   //     constraints for non-conforming AMR, etc.
   a.Assemble();

-   OperatorPtr Ah;
+   OperatorHandle Ah;
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

-   // 15. Solve using a direct or an iterative solver
+   // 15. Transform to monolithic HypreParMatrix
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A->GetGlobalNumRows() << endl;
+   }
+
+   // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
   {
-      // Transform to monolithic HypreParMatrix
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
      SuperLURowLocMatrix SA(*A);
      SuperLUSolver superlu(MPI_COMM_WORLD);
      superlu.SetPrintStatistics(false);
@@ -438,9 +441,9 @@ int main(int argc, char *argv[])
      superlu.SetColumnPermutation(superlu::PARMETIS);
      superlu.SetOperator(SA);
      superlu.Mult(B, X);
-      delete A;
   }
 #else
+
   // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
@@ -456,20 +459,20 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

      prec.Assemble();

-      OperatorPtr PCOpAh;
+      OperatorHandle PCOpAh;
      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);

      // 16b. Define and apply a parallel GMRES solver for AU=B with a block
@@ -493,7 +496,7 @@ int main(int argc, char *argv[])
      gmres.SetMaxIter(2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
-      gmres.SetOperator(*Ah);
+      gmres.SetOperator(*A);
      gmres.SetPreconditioner(BlockAMS);
      gmres.Mult(B, X);
   }
@@ -506,8 +509,10 @@ int main(int argc, char *argv[])
   // If exact is known compute the error
   if (exact_known)
   {
+      ParComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -624,6 +629,7 @@ int main(int argc, char *argv[])
   }

   // 20. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -822,7 +828,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -833,13 +839,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -850,13 +857,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -867,13 +875,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -887,18 +896,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -911,18 +921,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -935,13 +946,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -60,7 +60,6 @@ int main(int argc, char *argv[])
   bool static_cond = false;
   bool visualization = 1;
   bool amg_elast = 0;
-   bool reorder_space = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -76,8 +75,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
-                  "Use byNODES ordering of vector space instead of byVDIM");
   args.Parse();
   if (!args.Good())
   {
@@ -159,14 +156,7 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order, dim);
-      if (reorder_space)
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
-      }
-      else
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
-      }
+      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
   }
   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
@@ -259,7 +249,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      amg->SetSystemsOptions(dim, reorder_space);
+      amg->SetSystemsOptions(dim);
   }
   HyprePCG *pcg = new HyprePCG(A);
   pcg->SetTol(1e-8);
@@ -158,9 +158,9 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   Array<int> ess_bdr(pmesh->bdr_attributes.Max());
   if (pmesh->bdr_attributes.Size())
   {
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }
@@ -207,36 +207,18 @@ int main(int argc, char *argv[])
   // 13. Solve the system AX=B using PCG with the AMS preconditioner from hypre
   //     (in the full assembly case) or CG with Jacobi preconditioner (in the
   //     partial assembly case).
-   if (pa) // matrix-free auxiliary space solver with PA
+
+   if (pa) // Jacobi preconditioning in partial assembly mode
   {
-      StopWatch sw;
-      sw.Clear();
-      sw.Start();
-
-      MatrixFreeAMS *ams = new MatrixFreeAMS(*a, *A, *fespace, muinv, sigma, NULL,
-                                             ess_bdr); //, 2, 2);
-
-      //OperatorJacobiSmoother jacobi(*a, ess_tdof_list);
+      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);

      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-12);
      cg.SetMaxIter(1000);
      cg.SetPrintLevel(1);
      cg.SetOperator(*A);
-      cg.SetPreconditioner(*ams);
-      //cg.SetPreconditioner(jacobi);
-      //cg.iterative_mode = false;
-      sw.Stop();
-      cout << myid << ": MatrixFreeAMS-CG setup time " << sw.RealTime() << endl;
-
-      sw.Clear();
-      sw.Start();
-
+      cg.SetPreconditioner(Jacobi);
      cg.Mult(B, X);
-
-      sw.Stop();
-      cout << myid << ": MatrixFreeAMS-CG solve time " << sw.RealTime() << endl;
-      ams->PrintTimings(myid);
   }
   else
   {
@@ -1,425 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-#define TEST_AIRY
-
-#ifdef TEST_AIRY
-#include "gsl_sf_airy.h"
-
-#define XSHIFT 0.0 // 0.25
-#endif
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-void E_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-int dim;
-
-//#define K2_VALUE 11.9
-//#define K2_VALUE 2.1
-//#define K2_VALUE 1500.9
-#define K2_VALUE 10981.41589009910  // 104.792251097584^2 or 5 GHz
-
-void test_Airy_epsilon(const Vector &x, Vector &e)
-{
-   e(0) = 1.0;
-   e(1) = 1.0;
-   e(2) = (4.0 * (x(0) + XSHIFT)) - 1.0;
-
-   e *= -K2_VALUE;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   //const char *mesh_file = "../data/beam-tet.mesh";
-   const char *mesh_file = "../data/inline-tetHalf.mesh";
-   //const char *mesh_file = "inline-tetSlab.mesh";
-   //const char *mesh_file = "../data/inline-hexHalf.mesh";
-   //const char *mesh_file = "../data/inline-tet.mesh";
-   int order = 2;
-   bool static_cond = false;
-   bool visualization = 1;
-#ifdef MFEM_USE_STRUMPACK
-   bool use_strumpack = false;
-#endif
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
-                  "-no-strumpack", "--no-strumpack-solver",
-                  "Use STRUMPACK's double complex linear solver.");
-#endif
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-      cout << "Using k2 " << K2_VALUE << endl;
-   }
-   kappa = freq * M_PI;
-
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
-      ref_levels = 0;
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   long globalNE = pmesh->GetGlobalNE();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-      cout << "Number of mesh elements: " << globalNE << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_exact);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   ParGridFunction solproj(fespace);
-   VectorFunctionCoefficient E(sdim, E_exact);
-   x.ProjectCoefficient(E);
-
-   solproj = x;
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(-K2_VALUE);
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-#ifdef TEST_AIRY
-   VectorFunctionCoefficient epsilon(3, test_Airy_epsilon);
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
-#else
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-#endif
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   StopWatch chrono;
-   chrono.Clear();
-   chrono.Start();
-
-#ifdef MFEM_USE_STRUMPACK
-   if (use_strumpack)
-   {
-      Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      //Solver * precond = strumpack;
-
-      strumpack->Mult(B, X);
-      //X = 0.0;
-
-      {
-         // Check residual
-         Vector res(X.Size());
-         Vector ssol(X.Size());
-         ssol = X;
-
-         const double Bnrm = B.Norml2();
-         const double Bnrm2 = Bnrm*Bnrm;
-
-         A.Mult(ssol, res);
-         res -= B;
-
-         const double Rnrm = res.Norml2();
-         const double Rnrm2 = Rnrm*Rnrm;
-
-         double sumBnrm2 = 0.0;
-         double sumRnrm2 = 0.0;
-         MPI_Allreduce(&Bnrm2, &sumBnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-         MPI_Allreduce(&Rnrm2, &sumRnrm2, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-
-         if (myid == 0)
-         {
-            cout << myid << ": STRUMPACK residual norm " << sqrt(sumRnrm2) << ", B norm " <<
-                 sqrt(sumBnrm2) << endl;
-         }
-      }
-
-      delete strumpack;
-      delete Arow;
-   }
-   else
-#endif
-   {
-      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-      //     preconditioner from hypre.
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      HypreSolver *ams = new HypreAMS(A, prec_fespace);
-      HyprePCG *pcg = new HyprePCG(A);
-      pcg->SetTol(1e-12);
-      pcg->SetMaxIter(500);
-      pcg->SetPrintLevel(2);
-      pcg->SetPreconditioner(*ams);
-      pcg->Mult(B, X);
-
-      delete pcg;
-      delete ams;
-   }
-
-   chrono.Stop();
-   if (myid == 0)
-   {
-      cout << "Solver time " << chrono.RealTime() << endl;
-   }
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      ParGridFunction x0(fespace);
-      x0 = 0.0;
-
-      double solnrm = x0.ComputeL2Error(E);
-
-      if (myid == 0)
-      {
-         cout << "\n|| E_h - E ||_{L^2} = " << err << ", relative error " << err / solnrm
-              << endl;
-      }
-   }
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-      //solproj.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-#ifdef TEST_AIRY
-      const double y = (4.0 * (x(0) + XSHIFT)) - 1.0;
-      const double k = sqrt(K2_VALUE);
-      const double beta = pow(0.25 * k, 2.0/3.0);
-
-      E(0) = 0.0;
-      E(1) = 0.0;
-      E(2) = gsl_sf_airy_Ai(-beta * y, GSL_PREC_DOUBLE);
-#else
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-#endif
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-#ifdef TEST_AIRY
-      f = 0.0;
-#else
-      f(0) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (-K2_VALUE + kappa * kappa) * sin(kappa * x(0));
-#endif
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -1,848 +0,0 @@
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#include "gsl_sf_airy.h"
-
-#include "multigrid.hpp"
-
-using namespace std;
-using namespace mfem;
-
-
-//#define DIRECT_SOLVER
-
-// Define exact solution
-void E_exact(const Vector & x, Vector & E);
-void H_exact(const Vector & x, Vector & H);
-void f_exact_H(const Vector & x, Vector & f_H);
-void get_maxwell_solution(const Vector & x, double E[], double curlE[],
-                          double curl2E[]);
-void epsilon_func(const Vector &x, Vector &M);
-void epsilon2_func(const Vector &x, Vector &M);
-void epsilon_func_mat(const Vector &x, DenseMatrix &M);
-
-int dim;
-double omega;
-int sol = 3;
-
-
-class FOSLSSolver : public Solver
-{
-public:
-   FOSLSSolver(ParFiniteElementSpace *fespace_, std::vector<HypreParMatrix*>& P,
-               const int myid_)
-      : Solver(2 * fespace_->GetTrueVSize()), M_inv(MPI_COMM_WORLD),
-        fespace(fespace_),
-        n(fespace_->GetTrueVSize()), nfull(fespace_->GetVSize()), LSpcg(MPI_COMM_WORLD),
-        myid(myid_)
-   {
-      z.SetSize(n);
-      Minv_x.SetSize(n);
-
-      ParMesh *pmesh = fespace->GetParMesh();
-      int dim = pmesh->Dimension();
-      int sdim = pmesh->SpaceDimension();
-
-      VectorFunctionCoefficient epsilon(dim, epsilon_func);
-      VectorFunctionCoefficient epsilonT(epsilon);  // transpose of epsilon
-      VectorFunctionCoefficient epsilon2(dim,epsilon2_func);
-      ConstantCoefficient pos(omega);
-      ConstantCoefficient sigma(omega*omega);
-      ScalarVectorProductCoefficient coeff(pos,epsilon);
-      ScalarVectorProductCoefficient coeffT(pos,epsilonT);
-      ScalarVectorProductCoefficient coeff2(sigma,epsilon2);
-
-      bM = new ParBilinearForm(fespace);
-      bM->AddDomainIntegrator(new VectorFEMassIntegrator());
-      bM->Assemble();
-      bM->Finalize();
-
-      bM_eps = new ParBilinearForm(fespace);
-      bM_eps->AddDomainIntegrator(new VectorFEMassIntegrator(epsilonT));
-      bM_eps->Assemble();
-      bM_eps->Finalize();
-
-      bM_curl = new ParMixedBilinearForm(fespace,fespace);
-      bM_curl->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator());
-      bM_curl->Assemble();
-      bM_curl->Finalize();
-
-      Array<int> ess_tdof_list;  // empty
-
-      bM->FormSystemMatrix(ess_tdof_list, M);
-      bM_eps->FormSystemMatrix(ess_tdof_list, M_eps);
-      //bM_curl->FormColSystemMatrix(ess_tdof_list, M_curl);
-      {
-         OperatorPtr M_curl_ptr;
-         bM_curl->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, M_curl_ptr);
-         M_curl = M_curl_ptr.As<HypreParMatrix>();
-      }
-
-      M_inv.SetAbsTol(1.0e-12);
-      M_inv.SetRelTol(1.0e-12);
-      M_inv.SetMaxIter(100);
-      M_inv.SetOperator(M);
-      M_inv.SetPrintLevel(0);
-
-      block_trueOffsets.SetSize(5);
-
-      block_trueOffsets[0] = 0;
-      block_trueOffsets[1] = n;
-      block_trueOffsets[2] = n;
-      block_trueOffsets[3] = n;
-      block_trueOffsets[4] = n;
-      block_trueOffsets.PartialSum();
-
-      trueRhs = new BlockVector(block_trueOffsets);
-      trueSol = new BlockVector(block_trueOffsets);
-
-      //    _           _    _  _       _  _
-      //   |             |  |    |     |    |
-      //   |  A00   A01  |  | E  |     |F_E |
-      //   |             |  |    |  =  |    |
-      //   |  A10   A11  |  | H  |     |F_G |
-      //   |_           _|  |_  _|     |_  _|
-      //
-      // A00 = (curl E, curl F) + \omega^2 (E,F)
-      // A01 = - \omega *( (curl E, F) + (E,curl F)
-      // A10 = - \omega *( (curl H, G) + (H,curl G)
-      // A11 = (curl H, curl G) + \omega^2 (H,G)
-
-      ParBilinearForm *a_EE = new ParBilinearForm(fespace);
-      a_EE->AddDomainIntegrator(new CurlCurlIntegrator());
-      a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
-      a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-      a_EE->Assemble();
-      a_EE->Finalize();
-      HypreParMatrix *A_EE = new HypreParMatrix;
-      a_EE->FormSystemMatrix(ess_tdof_list, *A_EE);
-
-      ParBilinearForm *a_HH = new ParBilinearForm(fespace);
-      a_HH->AddDomainIntegrator(new CurlCurlIntegrator());
-      a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
-      a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-      a_HH->Assemble();
-      a_HH->Finalize();
-      HypreParMatrix *A_HH = new HypreParMatrix;
-      a_HH->FormSystemMatrix(ess_tdof_list, *A_HH);
-
-      ParBilinearForm *a_tang = new ParBilinearForm(fespace);
-      a_tang->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(1.0));
-      a_tang->Assemble();
-      a_tang->Finalize();
-      OperatorHandle A_tang_ptr;
-      a_tang->FormSystemMatrix(ess_tdof_list, A_tang_ptr);
-      HypreParMatrix *A_tang = A_tang_ptr.As<HypreParMatrix>();
-
-      // (k curl u, eps v) + (k u, curl v)
-      ParMixedBilinearForm *a_mix1 = new ParMixedBilinearForm(fespace,fespace);
-      a_mix1->AddDomainIntegrator(new MixedVectorCurlIntegrator(coeffT));
-      a_mix1->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(pos));
-      a_mix1->Assemble();
-      a_mix1->Finalize();
-      HypreParMatrix *A_mix1 = NULL; // new HypreParMatrix;
-      //a_mix1->FormColSystemMatrix(ess_tdof_list, *A_mix1);
-      {
-         OperatorPtr A_mix1_ptr;
-         a_mix1->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix1_ptr);
-         A_mix1 = A_mix1_ptr.As<HypreParMatrix>();
-      }
-
-      // (k curl u, v) + (k eps u, curl v)
-      ParMixedBilinearForm *a_mix2 = new ParMixedBilinearForm(fespace,fespace);
-      a_mix2->AddDomainIntegrator(new MixedVectorCurlIntegrator(pos));
-      a_mix2->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
-      a_mix2->Assemble();
-      a_mix2->Finalize();
-      HypreParMatrix *A_mix2 = NULL; // new HypreParMatrix;
-      //a_mix2->FormColSystemMatrix(ess_tdof_list, *A_mix2);
-      {
-         OperatorPtr A_mix2_ptr;
-         a_mix2->FormRectangularSystemMatrix(ess_tdof_list, ess_tdof_list, A_mix2_ptr);
-         A_mix2 = A_mix2_ptr.As<HypreParMatrix>();
-      }
-
-      BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
-      const int numBlocks = 4;
-
-      LS_Maxwellop->SetBlock(0, 0, A_EE);
-      LS_Maxwellop->SetBlock(1, 0, A_mix2, -1.0); // no bc
-      LS_Maxwellop->SetBlock(3, 0, A_tang, -1.0);
-      LS_Maxwellop->SetBlock(0, 1, A_mix1, -1.0);  // no bc
-      LS_Maxwellop->SetBlock(1, 1, A_HH);
-      LS_Maxwellop->SetBlock(2, 1, A_tang, -1.0);  // other rotation
-      LS_Maxwellop->SetBlock(1, 2, A_tang);
-      LS_Maxwellop->SetBlock(2, 2, A_EE);
-      LS_Maxwellop->SetBlock(3, 2, A_mix2, -1.0);  // no bc
-      LS_Maxwellop->SetBlock(0, 3, A_tang);  // other rotation
-      LS_Maxwellop->SetBlock(2, 3, A_mix1, -1.0);  // no bc
-      LS_Maxwellop->SetBlock(3, 3, A_HH);
-
-      // Set up the preconditioner
-      Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
-      Array2D<double> blockAcoef(numBlocks, numBlocks);
-
-      for (int i=0; i<numBlocks; ++i)
-      {
-         for (int j=0; j<numBlocks; ++j)
-         {
-            if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
-            {
-               blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
-               blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
-            }
-            else
-            {
-               blockA(i,j) = NULL;
-               blockAcoef(i,j) = 1.0;
-            }
-         }
-      }
-
-      LSpcg.SetAbsTol(1.0e-12);
-      LSpcg.SetRelTol(1.0e-8);
-      LSpcg.SetMaxIter(2000);
-      LSpcg.SetOperator(*LS_Maxwellop);
-      LSpcg.SetPrintLevel(1);
-
-      BlockMGSolver * precMG = NULL;
-
-#ifdef DIRECT_SOLVER
-      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-      Array2D<SparseMatrix*> Asp;
-      Asp.SetSize(numBlocks,numBlocks);
-
-      {
-         int nprocs, rank;
-         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-         MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-         std::vector<int> allnumrows(nprocs);
-         const int blockNumRows = n;
-         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
-                       MPI_COMM_WORLD);
-
-         for (int b=0; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b].resize(nprocs);
-            all_block_num_loc_rows[b].resize(nprocs);
-
-            for (int j=0; j<numBlocks; ++j)
-            {
-               Asp(b,j) = NULL;
-            }
-         }
-
-         blockProcOffsets[0][0] = 0;
-         for (int i=0; i<nprocs-1; ++i)
-         {
-            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-         }
-
-         for (int i=0; i<nprocs; ++i)
-         {
-            for (int b=0; b<numBlocks; ++b)
-            {
-               all_block_num_loc_rows[b][i] = allnumrows[i];
-            }
-
-            for (int b=1; b<numBlocks; ++b)
-            {
-               blockProcOffsets[b][i] = blockProcOffsets[0][i];
-            }
-         }
-      }
-
-      LSH = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, block_trueOffsets, blockA,
-                                           Asp,
-                                           blockAcoef, blockProcOffsets, all_block_num_loc_rows);
-
-      invLSH = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*LSH), MPI_COMM_WORLD);
-#else
-      precMG = new BlockMGSolver(LS_Maxwellop->Height(), LS_Maxwellop->Width(),
-                                 blockA, blockAcoef, P);
-      precMG->SetTheta(0.5);
-      LSpcg.SetPreconditioner(*precMG);
-#endif
-   }
-
-   void SetOperator(const Operator &op) { }
-
-   void Mult(const Vector &x, Vector &y) const
-   {
-      // Solve (curl E, curl v) - k^2 (eps E, v) + ik <pi(u), pi(v)> = (x, v), with no BC,
-      // where x is complex, using FOSLS. This is the Galerkin discretization of
-      // curl curl u - k^2 eps u = x, with ik n x u x n - n x curl u = 0 on the boundary.
-
-      MFEM_VERIFY(x.Size() == 2*n, "");
-
-      (*trueRhs) = 0.0;
-
-      for (int i=0; i<n; ++i)
-      {
-         z[i] = x[i];   // Set z = x_Re
-      }
-
-      M_inv.Mult(z, Minv_x);
-      M_eps.Mult(Minv_x, z);
-
-      trueRhs->GetBlock(0) -= z;
-
-      M_curl->Mult(Minv_x, z);
-      z *= 1.0 / omega;
-
-      trueRhs->GetBlock(1) = z;
-
-      for (int i=0; i<n; ++i)
-      {
-         z[i] = x[n + i];   // Set z = x_Im
-      }
-
-      M_inv.Mult(z, Minv_x);
-      M_eps.Mult(Minv_x, z);
-
-      trueRhs->GetBlock(2) -= z;
-
-      M_curl->Mult(Minv_x, z);
-      z *= 1.0 / omega;
-
-      trueRhs->GetBlock(3) += z;
-
-#ifdef DIRECT_SOLVER
-      invLSH->Mult(*trueRhs, *trueSol);
-#else
-      LSpcg.Mult(*trueRhs, *trueSol);
-#endif
-
-      for (int i=0; i<n; ++i)
-      {
-         y[i] = trueSol->GetBlock(0)[i];   // Set y_Re = E_Re
-      }
-
-      for (int i=0; i<n; ++i)
-      {
-         y[n + i] = trueSol->GetBlock(2)[i];   // Set y_Im = E_Im
-      }
-   }
-
-   void ComplexFOSLSTest()
-   {
-      Array<int> block_offsets;
-
-      block_offsets.SetSize(5);
-
-      block_offsets[0] = 0;
-      block_offsets[1] = nfull;
-      block_offsets[2] = nfull;
-      block_offsets[3] = nfull;
-      block_offsets[4] = nfull;
-      block_offsets.PartialSum();
-
-      BlockVector rhs(block_offsets);
-      BlockVector rhsIm(block_offsets);
-
-      rhs = 0.0;
-      rhsIm = 0.0;
-
-      const double ci = 0.0; // 3.3;
-
-      // Exact complex solution: E = Er + i Ei = Epw + ci i Epw, where Epw is E_exact.
-
-      // Set up the linear form with the real part Fr only.
-      const int sdim = 3;
-      VectorFunctionCoefficient Eex(sdim, E_exact);
-      VectorFunctionCoefficient Hex(sdim, H_exact);
-      ConstantCoefficient negOne(-1.0);
-      VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
-      VectorFunctionCoefficient mHex(sdim, H_exact, &negOne);
-
-      ConstantCoefficient neg(-omega);
-      ConstantCoefficient pos(omega);
-      VectorFunctionCoefficient f_H(3,f_exact_H);  // f / omega
-      ScalarVectorProductCoefficient sf_H(neg,f_H);
-      ScalarVectorProductCoefficient spf_H(pos,f_H);
-      ScalarVectorProductCoefficient mf_H(negOne,f_H);
-
-      VectorFunctionCoefficient epsilon(3, epsilon_func);
-      VectorFunctionCoefficient epsilonT(3, epsilon_func);  // transpose of epsilon
-      MatrixFunctionCoefficient epsilonTmat(3,
-                                            epsilon_func_mat);  // transpose of epsilon
-
-      MatVecCoefficient epsT_spf_H(epsilonTmat, spf_H);
-      MatVecCoefficient epsT_sf_H(epsilonTmat, sf_H);
-
-      ParLinearForm *b_E = new ParLinearForm;
-      b_E->Update(fespace, rhs.GetBlock(0), 0);
-      b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
-                                  epsT_sf_H)); // (k^{-1} Fr, -k eps Qr)
-      b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                    Eex));  // <n x E_Re x n, n x Q_Re x n>
-      b_E->Assemble();
-
-      ParLinearForm *b_H = new ParLinearForm;
-      b_H->Update(fespace, rhs.GetBlock(1), 0);
-      b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
-                                  f_H));  // (k^{-1} Fr, curl Rr)
-      b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                    Hex));  // <n x H_Re, n x R_Re>
-      b_H->Assemble();
-
-      ParLinearForm *b_E_Im = new ParLinearForm;
-      b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
-      b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                       mHex));  // <n x H_Re, n x Q_Im x n>
-      b_E_Im->Assemble();
-
-      ParLinearForm *b_H_Im = new ParLinearForm;
-      b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
-      b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                       mEex));  // -<n x E_Re x n, n x R_Im>
-      b_H_Im->Assemble();
-
-      // Add the imaginary part Fi.
-
-      ParLinearForm *b_Ei = new ParLinearForm;
-      b_Ei->Update(fespace, rhsIm.GetBlock(0), 0);
-      b_Ei->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                     Hex));  // -<n x H_Im, n x Q_Re x n>
-      b_Ei->Assemble();
-
-      ParLinearForm *b_Hi = new ParLinearForm;
-      b_Hi->Update(fespace, rhsIm.GetBlock(1), 0);
-      b_Hi->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                     Eex));  // <n x E_Im x n, n x R_Re>
-      b_Hi->Assemble();
-
-      ParLinearForm *b_Ei_Im = new ParLinearForm;
-      b_Ei_Im->Update(fespace, rhsIm.GetBlock(2), 0);
-      b_Ei_Im->AddDomainIntegrator(new VectorFEDomainLFIntegrator(
-                                      epsT_sf_H)); // -(k^{-1} Fi, k eps Qi)
-      b_Ei_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                        Eex));  // <n x E_Im x n, n x Q_Im x n>
-      b_Ei_Im->Assemble();
-
-      ParLinearForm *b_Hi_Im = new ParLinearForm;
-      b_Hi_Im->Update(fespace, rhsIm.GetBlock(3), 0);
-      b_Hi_Im->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(
-                                      f_H)); // (k^{-1} Fi, curl Ri)
-      b_Hi_Im->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                        Hex));  // <n x H_Im, n x R_Im>
-      b_Hi_Im->Assemble();
-
-      rhsIm *= ci;
-      rhs += rhsIm;
-
-      for (int i=0; i<4; ++i)
-      {
-         fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(i),
-                                                         trueRhs->GetBlock(i));
-      }
-
-#ifdef DIRECT_SOLVER
-      invLSH->Mult(*trueRhs, *trueSol);
-#else
-      LSpcg.Mult(*trueRhs, *trueSol);
-#endif
-
-      // Check error
-      ParGridFunction E_gf(fespace);
-      int order = 2;
-      int order_quad = std::max(2, 2*order+1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i=0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      ParMesh *pmesh = fespace->GetParMesh();
-
-      // Check error of real part
-
-      E_gf.SetFromTrueDofs(trueSol->GetBlock(0));
-      double Error_E = E_gf.ComputeL2Error(Eex, irs);
-      double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
-
-      cout << myid << ": real error " << Error_E << " relative to " << norm_E << endl;
-
-      {
-         ostringstream mesh_name, sol_name;
-         mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-         sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-         ofstream mesh_ofs(mesh_name.str().c_str());
-         mesh_ofs.precision(8);
-         pmesh->Print(mesh_ofs);
-
-         ofstream sol_ofs(sol_name.str().c_str());
-         sol_ofs.precision(8);
-         E_gf.Save(sol_ofs);
-      }
-
-      // Check error of imaginary part
-      E_gf.SetFromTrueDofs(trueSol->GetBlock(2));
-      const double iml2 = E_gf.Norml2();
-      E_gf *= ci == 0.0 ? 0.0 : (1.0 / ci);
-      Error_E = E_gf.ComputeL2Error(Eex, irs);
-
-      cout << myid << ": imag error " << Error_E << " relative to " << norm_E <<
-           ", l2 norm " << iml2 << endl;
-   }
-
-private:
-
-   BlockVector *trueRhs, *trueSol;
-
-   Array<int> block_trueOffsets;
-
-   ParBilinearForm *bM, *bM_eps;
-   ParMixedBilinearForm *bM_curl;
-
-   HypreParMatrix M, M_eps;
-   HypreParMatrix *M_curl;
-
-   CGSolver M_inv;
-
-   const int n;
-   const int nfull;
-   const int myid;
-
-   mutable Vector z, Minv_x;
-
-   CGSolver LSpcg;
-
-   STRUMPACKSolver *invLSH;
-   HypreParMatrix *LSH;
-
-   ParFiniteElementSpace *fespace;
-};
-
-int main(int argc, char *argv[])
-{
-   StopWatch chrono;
-
-   // 1. Initialize MPI
-   int num_procs, myid;
-   MPI_Init(&argc, &argv); // Initialize MPI
-   MPI_Comm_size(MPI_COMM_WORLD,
-                 &num_procs); //total number of processors available
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
-   // 1. Parse command-line options.
-   // geometry file
-   const char *mesh_file = "../data/inline-hex.mesh";
-   // finite element order of approximation
-   int order = 1;
-   // static condensation flag
-   bool static_cond = false;
-   // visualization flag
-   bool visualization = 1;
-   // number of wavelengths
-   double k = 1.0;
-   // number of mg levels
-   int ref_levels = 1;
-   // number of initial ref
-   int initref = 1;
-
-   // optional command line inputs
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&k, "-k", "--wavelengths",
-                  "Number of wavelengths.");
-   args.AddOption(&ref_levels, "-ref", "--ref_levels",
-                  "Number of Refinements.");
-   args.AddOption(&initref, "-initref", "--initref",
-                  "Number of initial refinements.");
-   args.AddOption(&sol, "-sol", "--exact",
-                  "Exact solution flag - "
-                  " 1:sinusoidal, 2: point source, 3: plane wave");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   // check if the inputs are correct
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // Angular frequency
-   //omega = 2.0*k*M_PI;
-   omega = k;
-
-   // 2. Read the mesh from the given mesh file.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-
-   if (sol == 4)
-   {
-      mesh->EnsureNodes();
-      GridFunction *nodes = mesh->GetNodes();
-      (*nodes) *= 0.5;
-   }
-
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 3. Executing uniform h-refinement
-   for (int i = 0; i < initref; i++ )
-   {
-      mesh->UniformRefinement();
-   }
-
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-
-   // 4. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
-   std::vector<ParMesh * > ParMeshes(ref_levels+1);
-   std::vector<HypreParMatrix*> P(ref_levels);
-
-   for (int i = 0; i < ref_levels; i++)
-   {
-      ParMeshes[i] =new ParMesh(*pmesh);
-      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
-      pmesh->UniformRefinement();
-      // Update fespace
-      fespace->Update();
-      OperatorHandle Tr(Operator::Hypre_ParCSR);
-      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
-      Tr.SetOperatorOwner(false);
-      Tr.Get(P[i]);
-   }
-   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
-
-   FOSLSSolver fosls(fespace, P, myid);
-
-   fosls.ComplexFOSLSTest();
-
-   for (auto p: ParMeshes) { delete p; }
-   for (auto p: fespaces) { delete p; }
-   for (auto p: P) { delete p; }
-   ParMeshes.clear();
-   fespaces.clear();
-   P.clear();
-   delete fec;
-   delete fespace;
-   delete pmesh;
-
-   MPI_Finalize();
-   return 0;
-}
-
-//define exact solution
-void E_exact(const Vector &x, Vector &E)
-{
-   double curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-}
-
-void H_exact(const Vector &x, Vector &H)
-{
-   double E[3], curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
-}
-
-
-void f_exact_H(const Vector &x, Vector &f)
-{
-   // curl H - omega E = f
-   // = curl (curl E / omega) - omega E
-   f = 0.0;
-   if (sol !=4)
-   {
-      double E[3], curlE[3], curl2E[3];
-      get_maxwell_solution(x, E, curlE, curl2E);
-      f(0) = curl2E[0] / omega - omega * E[0];
-      f(1) = curl2E[1] / omega - omega * E[1];
-      f(2) = curl2E[2] / omega - omega * E[2];
-   }
-}
-
-void get_maxwell_solution(const Vector &X, double E[], double curlE[],
-                          double curl2E[])
-{
-   double x = X[0];
-   double y = X[1];
-   double z = X[2];
-
-   if (sol ==-1)
-   {
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
-      E[2] = x * y * (1.0 - x) * (1.0 - y);
-
-      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
-      curlE[1] = -2.0*(y-1.0)*y*(x-z);
-      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
-
-      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
-      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
-      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
-   }
-   else if (sol == 0) // polynomial
-   {
-      // Polynomial vanishing on the boundary
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
-      E[2] = (1.0 - x) * x * (1.0 - y) * y;
-      //
-      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
-      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
-      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
-
-      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
-      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
-      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
-   }
-   else if (sol == 1) // sinusoidal
-   {
-      E[0] = sin(omega * y);
-      E[1] = sin(omega * z);
-      E[2] = sin(omega * x);
-
-      curlE[0] = -omega * cos(omega * z);
-      curlE[1] = -omega * cos(omega * x);
-      curlE[2] = -omega * cos(omega * y);
-
-      curl2E[0] = omega * omega * E[0];
-      curl2E[1] = omega * omega * E[1];
-      curl2E[2] = omega * omega * E[2];
-   }
-   else if (sol == 2) // point source
-   {
-      // shift to avoid singularity
-      double x0 = x + 0.1;
-      double x1 = y + 0.1;
-      double x2 = z + 0.1;
-      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
-
-      E[0] = cos(omega * r);
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      double r_x = x0 / r;
-      double r_y = x1 / r;
-      double r_z = x2 / r;
-      double r_xy = -(r_x / r) * r_y;
-      double r_xz = -(r_x / r) * r_z;
-      double r_yx = r_xy;
-      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
-      double r_zx = r_xz;
-      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
-
-      curlE[0] = 0.0;
-      curlE[1] = -omega * r_z * sin(omega * r);
-      curlE[2] =  omega * r_y * sin(omega * r);
-
-      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
-                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
-      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
-                               omega * r));
-      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
-                               omega * r));
-   }
-   else if (sol == 3) // plane wave
-   {
-      double coeff = omega / sqrt(3.0);
-      E[0] = cos(coeff * (x + y + z));
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = -coeff * sin(coeff * (x + y + z));
-      curlE[2] = coeff * sin(coeff * (x + y + z));
-
-      curl2E[0] = 2.0 * coeff * coeff * E[0];
-      curl2E[1] = -coeff * coeff * E[0];
-      curl2E[2] = -coeff * coeff * E[0];
-   }
-   else if (sol == -1)
-   {
-      E[0] = cos(omega * y);
-      E[1] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = 0.0;
-      curlE[2] = -omega * sin(omega * y);
-
-      curl2E[0] = omega*omega * cos(omega*y);
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-   else if (sol == 4) // Airy function
-   {
-      E[0] = 0;
-      E[1] = 0;
-      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
-      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
-      //E[2] = boost::math::airy_ai(b);
-      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
-
-      curlE[0] = 0.0;
-      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
-                                                                     GSL_PREC_DOUBLE);
-      curlE[2] = 0.0;
-
-      // not used
-      curl2E[0] = 0.0;
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-}
-
-void epsilon_func(const Vector &x, Vector &M)
-{
-   M.SetSize(3);
-
-   M = 1.0;
-   if (sol == 4)
-   {
-      M[2] = 4.0*x(0)-1.0;
-   }
-}
-
-void epsilon2_func(const Vector &x, Vector &M)
-{
-   M.SetSize(3);
-
-   M = 1.0;
-   if (sol == 4)
-   {
-      M[2] = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
-   }
-}
-
-void epsilon_func_mat(const Vector &x, DenseMatrix &M)
-{
-   M.SetSize(3);
-
-   M = 0.0;
-   M(0,0) = 1.0;
-   M(1,1) = 1.0;
-   if (sol != 4)
-   {
-      M(2,2) = 1.0;
-   }
-   else
-   {
-      M(2,2) = 4.0*x(0)-1.0;
-   }
-}
@@ -1,764 +0,0 @@
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#include "gsl_sf_airy.h"
-
-#include "multigrid.hpp"
-
-using namespace std;
-using namespace mfem;
-
-#define COMPLEX_IMPEDANCE  // Whether to use impedance BC and solve a complex system
-
-// Define exact solution
-void E_exact(const Vector & x, Vector & E);
-void H_exact(const Vector & x, Vector & H);
-void f_exact_H(const Vector & x, Vector & f_H);
-void get_maxwell_solution(const Vector & x, double E[], double curlE[],
-                          double curl2E[]);
-void epsilon_func(const Vector &x, DenseMatrix &M);
-void epsilon2_func(const Vector &x, DenseMatrix &M);
-
-int dim;
-double omega;
-int sol = 4;
-
-int main(int argc, char *argv[])
-{
-   StopWatch chrono;
-
-   // 1. Initialize MPI
-   int num_procs, myid;
-   MPI_Init(&argc, &argv); // Initialize MPI
-   MPI_Comm_size(MPI_COMM_WORLD,
-                 &num_procs); //total number of processors available
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid); // Determine process identifier
-   // 1. Parse command-line options.
-   // geometry file
-   const char *mesh_file = "../data/inline-hex.mesh";
-   // finite element order of approximation
-   int order = 1;
-   // static condensation flag
-   bool static_cond = false;
-   // visualization flag
-   bool visualization = 1;
-   // number of wavelengths
-   double k = 1.0;
-   // number of mg levels
-   int ref_levels = 1;
-   // number of initial ref
-   int initref = 1;
-
-   // optional command line inputs
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&k, "-k", "--wavelengths",
-                  "Number of wavelengths.");
-   args.AddOption(&ref_levels, "-ref", "--ref_levels",
-                  "Number of Refinements.");
-   args.AddOption(&initref, "-initref", "--initref",
-                  "Number of initial refinements.");
-   args.AddOption(&sol, "-sol", "--exact",
-                  "Exact solution flag - "
-                  " 1:sinusoidal, 2: point source, 3: plane wave");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   // check if the inputs are correct
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // Angular frequency
-   //omega = 2.0*k*M_PI;
-   omega = k;
-
-   // 2. Read the mesh from the given mesh file.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-
-   if (sol == 4)
-   {
-      mesh->EnsureNodes();
-      GridFunction *nodes = mesh->GetNodes();
-      (*nodes) *= 0.5;
-   }
-
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 3. Executing uniform h-refinement
-   for (int i = 0; i < initref; i++ )
-   {
-      mesh->UniformRefinement();
-   }
-
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-
-   // 4. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   std::vector<ParFiniteElementSpace * > fespaces(ref_levels+1);
-   std::vector<ParMesh * > ParMeshes(ref_levels+1);
-   std::vector<HypreParMatrix*> P(ref_levels);
-
-   for (int i = 0; i < ref_levels; i++)
-   {
-      ParMeshes[i] =new ParMesh(*pmesh);
-      fespaces[i] = new ParFiniteElementSpace(*fespace, *ParMeshes[i]);
-      pmesh->UniformRefinement();
-      // Update fespace
-      fespace->Update();
-      OperatorHandle Tr(Operator::Hypre_ParCSR);
-      fespace->GetTrueTransferOperator(*fespaces[i], Tr);
-      Tr.SetOperatorOwner(false);
-      Tr.Get(P[i]);
-   }
-   fespaces[ref_levels] = new ParFiniteElementSpace(*fespace);
-
-   Array<int> ess_tdof_listE;
-   Array<int> ess_tdof_listH;
-   Array<int> ess_bdrE(pmesh->bdr_attributes.Max());
-   Array<int> ess_bdrH(pmesh->bdr_attributes.Max());
-#ifdef COMPLEX_IMPEDANCE
-   ess_bdrE = 0;
-   ess_bdrH = 0;  // Neumann
-#else
-   ess_bdrE = 1;
-   ess_bdrH = 0;  // Neumann
-#endif
-   fespace->GetEssentialTrueDofs(ess_bdrE, ess_tdof_listE);
-   fespace->GetEssentialTrueDofs(ess_bdrH, ess_tdof_listH);
-
-#ifdef COMPLEX_IMPEDANCE
-   Array<int> bdr_attr(pmesh->bdr_attributes.Max());
-   bdr_attr = 1;
-   ConstantCoefficient impedance(omega);
-   RestrictedCoefficient imp_rest(impedance, bdr_attr);
-   Array<int> block_offsets(5);
-   Array<int> block_trueOffsets(5);
-#else
-   Array<int> block_offsets(3);
-   Array<int> block_trueOffsets(3);
-#endif
-   block_offsets[0] = 0;
-   block_offsets[1] = fespace->GetVSize();
-   block_offsets[2] = fespace->GetVSize();
-#ifdef COMPLEX_IMPEDANCE
-   block_offsets[3] = fespace->GetVSize();
-   block_offsets[4] = fespace->GetVSize();
-#endif
-   block_offsets.PartialSum();
-
-   block_trueOffsets[0] = 0;
-   block_trueOffsets[1] = fespace->TrueVSize();
-   block_trueOffsets[2] = fespace->TrueVSize();
-#ifdef COMPLEX_IMPEDANCE
-   block_trueOffsets[3] = fespace->TrueVSize();
-   block_trueOffsets[4] = fespace->TrueVSize();
-#endif
-   block_trueOffsets.PartialSum();
-
-   //    _           _    _  _       _  _
-   //   |             |  |    |     |    |
-   //   |  A00   A01  |  | E  |     |F_E |
-   //   |             |  |    |  =  |    |
-   //   |  A10   A11  |  | H  |     |F_G |
-   //   |_           _|  |_  _|     |_  _|
-   //
-   // A00 = (curl E, curl F) + \omega^2 (E,F)
-   // A01 = - \omega *( (curl E, F) + (E,curl F)
-   // A10 = - \omega *( (curl H, G) + (H,curl G)
-   // A11 = (curl H, curl G) + \omega^2 (H,G)
-
-   BlockVector x(block_offsets), rhs(block_offsets);
-   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);
-
-   x = 0.0;
-   rhs = 0.0;
-   trueX = 0.0;
-   trueRhs = 0.0;
-
-   VectorFunctionCoefficient Eex(sdim, E_exact);
-   ConstantCoefficient negOne(-1.0);
-   VectorFunctionCoefficient mEex(sdim, E_exact, &negOne);
-
-   ParGridFunction * E_gf = new ParGridFunction;
-   ParGridFunction * Exact_gf = new ParGridFunction(fespace);
-   E_gf->MakeRef(fespace, x.GetBlock(0));
-   E_gf->ProjectCoefficient(Eex);
-   Exact_gf->ProjectCoefficient(Eex);
-
-   VectorFunctionCoefficient Hex(sdim, H_exact);
-   ParGridFunction * H_gf = new ParGridFunction;
-   H_gf->MakeRef(fespace, x.GetBlock(1));
-   H_gf->ProjectCoefficient(Hex);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient sigma(pow(omega, 2));
-   ConstantCoefficient neg(-abs(omega));
-   ConstantCoefficient pos(abs(omega));
-
-   MatrixFunctionCoefficient epsilon(dim,epsilon_func);
-   TransposeMatrixCoefficient epsilonT(epsilon);  // transpose of epsilon
-   MatrixFunctionCoefficient epsilon2(dim,epsilon2_func);
-   ScalarMatrixProductCoefficient coeff(neg,epsilon);
-   ScalarMatrixProductCoefficient coeff2(sigma,epsilon2);
-
-   // 6. Set up the linear form
-   VectorFunctionCoefficient f_H(sdim,f_exact_H);
-   ScalarVectorProductCoefficient sf_H(neg,f_H);
-
-   MatVecCoefficient epsT_sf_H(epsilonT, sf_H);
-
-   ParLinearForm *b_E = new ParLinearForm;
-   b_E->Update(fespace, rhs.GetBlock(0), 0);
-   b_E->AddDomainIntegrator(new VectorFEDomainLFIntegrator(epsT_sf_H));
-#ifdef COMPLEX_IMPEDANCE
-   b_E->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                 Eex));  // <g_Im, n x Q x n> = <n x E_Re x n, n x Q x n>
-
-   ParLinearForm *b_E_Im = new ParLinearForm;
-   b_E_Im->Update(fespace, rhs.GetBlock(2), 0);
-   b_E_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                    Hex));  // -<g_Re, n x Q x n> = <n x H_Re, n x Q x n>
-   b_E_Im->Assemble();
-#endif
-   b_E->Assemble();
-
-   ParLinearForm *b_H = new ParLinearForm;
-   b_H->Update(fespace, rhs.GetBlock(1), 0);
-   b_H->AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
-#ifdef COMPLEX_IMPEDANCE
-   b_H->AddBoundaryIntegrator(new VectorFEDomainLFIntegrator(
-                                 Hex));  // -<g_Re, n x R> = <n x H_Re, n x R>
-
-   ParLinearForm *b_H_Im = new ParLinearForm;
-   b_H_Im->Update(fespace, rhs.GetBlock(3), 0);
-   b_H_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentLFIntegrator(
-                                    mEex));  // -<g_Im, n x R> = -<n x E_Re x n, n x R>
-   b_H_Im->Assemble();
-#endif
-   b_H->Assemble();
-
-#ifdef COMPLEX_IMPEDANCE
-   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(2),
-                                                   trueRhs.GetBlock(2));
-   fespace->GetProlongationMatrix()->MultTranspose(rhs.GetBlock(3),
-                                                   trueRhs.GetBlock(3));
-#endif
-
-   // 7. Bilinear form a(.,.) on the finite element space
-   ParBilinearForm *a_EE = new ParBilinearForm(fespace);
-   a_EE->AddDomainIntegrator(new CurlCurlIntegrator(one));
-   a_EE->AddDomainIntegrator(new VectorFEMassIntegrator(coeff2));
-
-#ifdef COMPLEX_IMPEDANCE
-   a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-   //a_EE->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
-   //a_EE->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
-   //a_EE->AddBdrFaceIntegrator(new BoundaryMassIntegrator(imp_rest));
-#endif
-
-   a_EE->Assemble();
-   a_EE->Finalize();
-   HypreParMatrix *A_EE = new HypreParMatrix;
-   a_EE->FormLinearSystem(ess_tdof_listE, x.GetBlock(0), rhs.GetBlock(0), *A_EE,
-                          trueX.GetBlock(0), trueRhs.GetBlock(0));
-
-   ParBilinearForm *a_HH = new ParBilinearForm(fespace);
-   a_HH->AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
-   a_HH->AddDomainIntegrator(new VectorFEMassIntegrator(sigma));
-
-#ifdef COMPLEX_IMPEDANCE
-   a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator());
-   //a_HH->AddBoundaryIntegrator(new VectorFEMassIntegrator(imp_rest));
-   //a_HH->AddBoundaryIntegrator(new BoundaryMassIntegrator(imp_rest));
-#endif
-
-   a_HH->Assemble();
-   a_HH->Finalize();
-   HypreParMatrix *A_HH = new HypreParMatrix;
-   a_HH->FormLinearSystem(ess_tdof_listH, x.GetBlock(1), rhs.GetBlock(1), *A_HH,
-                          trueX.GetBlock(1), trueRhs.GetBlock(1));
-
-   ParMixedBilinearForm *a_HE = new ParMixedBilinearForm(fespace,fespace);
-   a_HE->AddDomainIntegrator(new MixedVectorCurlIntegrator(neg));
-   a_HE->AddDomainIntegrator(new MixedVectorWeakCurlIntegrator(coeff));
-   a_HE->Assemble();
-   a_HE->Finalize();
-   HypreParMatrix *A_HE = new HypreParMatrix;
-   a_HE->FormColLinearSystem(ess_tdof_listE,x.GetBlock(0),rhs.GetBlock(1),*A_HE,
-                             trueX.GetBlock(0),trueRhs.GetBlock(1));
-
-   HypreParMatrix *A_EH = A_HE->Transpose();
-
-#ifdef COMPLEX_IMPEDANCE
-   ParBilinearForm *a_EH_Im = new ParBilinearForm(fespace);
-   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(imp_rest));
-   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(impedance));
-   //a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator(omega));
-   a_EH_Im->AddBoundaryIntegrator(new VectorFEBoundaryTangentIntegrator());
-   a_EH_Im->Assemble();
-   a_EH_Im->Finalize();
-
-   OperatorHandle A_EH_Im_ptr;
-   a_EH_Im->FormSystemMatrix(ess_tdof_listE,
-                             A_EH_Im_ptr);  // empty ess_tdof_list for impedance
-
-   HypreParMatrix *A_EH_Im = A_EH_Im_ptr.As<HypreParMatrix>();
-
-   HypreParMatrix *A_HE_Im = A_EH_Im->Transpose();
-#endif
-
-   BlockOperator *LS_Maxwellop = new BlockOperator(block_trueOffsets);
-#ifdef COMPLEX_IMPEDANCE
-   const int numBlocks = 4;
-#else
-   const int numBlocks = 2;
-#endif
-
-   LS_Maxwellop->SetBlock(0, 0, A_EE);
-   LS_Maxwellop->SetBlock(0, 1, A_EH);
-   LS_Maxwellop->SetBlock(1, 0, A_HE);
-   LS_Maxwellop->SetBlock(1, 1, A_HH);
-
-#ifdef COMPLEX_IMPEDANCE
-   LS_Maxwellop->SetBlock(0, 3, A_EH_Im);
-   LS_Maxwellop->SetBlock(1, 2, A_HE_Im, -1.0);
-
-   LS_Maxwellop->SetBlock(2, 1, A_EH_Im, -1.0);
-   LS_Maxwellop->SetBlock(3, 0, A_HE_Im);
-
-   LS_Maxwellop->SetBlock(2, 2, A_EE);
-   LS_Maxwellop->SetBlock(2, 3, A_EH);
-   LS_Maxwellop->SetBlock(3, 2, A_HE);
-   LS_Maxwellop->SetBlock(3, 3, A_HH);
-#endif
-
-   if (myid == 0)
-   {
-      cout << "Size of fine grid system: "
-           << 2.0 * A_EE->GetGlobalNumRows() << " x " << 2.0* A_EE->GetGlobalNumCols() <<
-           endl;
-   }
-
-   // Set up the preconditioner
-   Array2D<HypreParMatrix*> blockA(numBlocks, numBlocks);
-   Array2D<double> blockAcoef(numBlocks, numBlocks);
-
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (LS_Maxwellop->IsZeroBlock(i,j) == 0)
-         {
-            blockA(i,j) = static_cast<HypreParMatrix *>(&LS_Maxwellop->GetBlock(i,j));
-            blockAcoef(i,j) = LS_Maxwellop->GetBlockCoef(i,j);
-         }
-         else
-         {
-            blockA(i,j) = NULL;
-            blockAcoef(i,j) = 1.0;
-         }
-      }
-   }
-
-   // // double nnz = A_HH->NNZ();
-   // // double ndof = A_HH->GetGlobalNumRows();
-   // // double est_mem_b = nnz*12.0 + (ndof+1.0)*4;
-   // // double gb = est_mem_b*4.0/pow(1024.0,3);
-
-   // // mfem::out << "Estimated memory taken by the global matrix: " <<  gb << endl;
-
-   int maxit(2000);
-   double rtol(1.e-8);
-   double atol(1.e-12);
-
-   // trueX = 0.0;
-   CGSolver pcg(MPI_COMM_WORLD);
-   pcg.SetAbsTol(atol);
-   pcg.SetRelTol(rtol);
-   pcg.SetMaxIter(maxit);
-   pcg.SetOperator(*LS_Maxwellop);
-   pcg.SetPrintLevel(1);
-   chrono.Clear();
-   chrono.Start();
-   BlockMGSolver * precMG = new BlockMGSolver(LS_Maxwellop->Height(),
-                                              LS_Maxwellop->Width(), blockA, blockAcoef, P);
-   //precMG->SetTheta(0.5);
-   // // int lv_coarse = min(ref_levels,ref_levels-1);
-   // // int levels = ref_levels - lv_coarse;
-   // // BlkParSchwarzSmoother * precAS = new BlkParSchwarzSmoother(fespaces[lv_coarse]->GetParMesh(),levels,fespaces[ref_levels],LS_Maxwellop);
-   chrono.Stop();
-   if (myid == 0)
-   {
-      cout << "MG Setup time: " << chrono.RealTime() << endl;
-   }
-
-   // Randomize H RHS
-   //trueRhs = 1.0;
-   /*
-   for (int i=block_trueOffsets[1]; i<block_trueOffsets[2]; ++i)
-     trueRhs[i] = i % 53;
-   */
-
-   chrono.Clear();
-   chrono.Start();
-   pcg.SetPreconditioner(*precMG);
-   // // pcg.SetPreconditioner(*precAS);
-   pcg.Mult(trueRhs, trueX);
-   chrono.Stop();
-   delete precMG;
-   // // delete precAS;
-
-   // // trueX = 0.0;
-   // // invA->Mult(trueRhs,trueX);
-
-   if (myid == 0)
-   {
-      cout << "MG Solution time time: " << chrono.RealTime() << endl;
-   }
-   // // cin.get();
-   // //  if(myid == 0)
-   // //    cout << "MG prec Solution time: " << chrono.RealTime() << endl;
-
-
-   // // chrono.Clear();
-   // // chrono.Start();
-   // // Block_AMSSolver * precAMS = new Block_AMSSolver(block_trueOffsets,fespaces);
-   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::SCHWARZ);
-   // // precAMS->SetSmootherType(Block_AMS::BlkSmootherType::HYPRE);
-   // // precAMS->SetOperator(LS_Maxwellop);
-   // // precAMS->SetTheta(1.0/5.0);
-   // // // 0-Smoother, 1-Grad, 2,3,4-Pix,Piy,Piz
-   // // precAMS->SetCycleType("023414320");
-   // // precAMS->SetNumberofCycles(1);
-   // // chrono.Stop();
-   // // if(myid == 0)
-   // //    cout << "BlkAMS Setup time: " << chrono.RealTime() << endl;
-
-   // // // resolve with block AMS
-   // // trueX = 0;
-   // // chrono.Clear();
-   // // chrono.Start();
-   // // pcg.SetPreconditioner(*precAMS);
-   // // pcg.Mult(trueRhs, trueX);
-   // // chrono.Stop();
-   // // delete precAMS;
-
-   // // if(myid == 0)
-   // //    cout << "BlockAMS Solution time: " << chrono.RealTime() << endl;
-
-
-   a_EE->RecoverFEMSolution(trueX.GetBlock(0), *b_E, *E_gf);
-   a_HH->RecoverFEMSolution(trueX.GetBlock(1), *b_H, *H_gf);
-
-
-   int order_quad = max(2, 2*order+1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   double Error_E = E_gf->ComputeL2Error(Eex, irs);
-   double norm_E = ComputeGlobalLpNorm(2, Eex, *pmesh, irs);
-
-   double Error_H = H_gf->ComputeL2Error(Hex, irs);
-   double norm_H = ComputeGlobalLpNorm(2, Hex , *pmesh, irs);
-
-   if (myid == 0)
-   {
-      cout << "|| E_h - E || = " << Error_E  << "\n";
-      cout << "|| E_h - E ||/||E|| = " << Error_E/norm_E  << "\n";
-      cout << "|| H_h - H || = " << Error_H  << "\n";
-      cout << "|| H_h - H ||/||H|| = " << Error_H/norm_H  << "\n";
-      cout << "Total error = " << setprecision(15) << sqrt(Error_H*Error_H
-                                                           +Error_E*Error_E) << "\n";
-   }
-
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      E_gf->Save(sol_ofs);
-   }
-
-   //ParGridFunction ExactE(fespace);
-   /*
-   if (visualization)
-     {
-       // 8. Connect to GLVis.
-       char vishost[] = "localhost";
-       int  visport   = 19916;
-       // socketstream E_sock(vishost, visport);
-       // E_sock << "parallel " << num_procs << " " << myid << "\n";
-       // E_sock.precision(8);
-       // E_sock << "solution\n" << *pmesh << *E_gf << "window_title 'Electric field'" << endl;
-       socketstream Exact_sock(vishost, visport);
-       Exact_sock << "parallel " << num_procs << " " << myid << "\n";
-       Exact_sock.precision(8);
-       Exact_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Electric field'" << endl;
-
-       // MPI_Barrier(pmesh->GetComm());
-       // socketstream Eex_sock(vishost, visport);
-       // Eex_sock << "parallel " << num_procs << " " << myid << "\n";
-       // Eex_sock.precision(8);
-       // Eex_sock << "solution\n" << *pmesh << *Exact_gf << "window_title 'Exact Electric field'" << endl;
-     }
-   */
-   // delete A_EE;
-   // delete A_HE;
-   // delete A_EH;
-   // delete A_HH;
-   // delete LS_Maxwellop;
-   // delete a_EE;
-   // delete a_HE;
-   // delete a_HH;
-   // delete b_E;
-   // delete b_H;
-   delete E_gf;
-   delete Exact_gf;
-   for (auto p: ParMeshes) { delete p; }
-   for (auto p: fespaces) { delete p; }
-   for (auto p: P) { delete p; }
-   ParMeshes.clear();
-   fespaces.clear();
-   P.clear();
-   delete fec;
-   delete fespace;
-   delete pmesh;
-
-   // cout << "Freed memory: " << endl;
-   // cin.get();
-   MPI_Finalize();
-   return 0;
-}
-
-//define exact solution
-void E_exact(const Vector &x, Vector &E)
-{
-   double curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-}
-
-void H_exact(const Vector &x, Vector &H)
-{
-   double E[3], curlE[3], curl2E[3];
-   get_maxwell_solution(x, E, curlE, curl2E);
-   for (int i = 0; i<3; i++) { H(i) = curlE[i]/omega; }
-}
-
-
-void f_exact_H(const Vector &x, Vector &f)
-{
-   // curl H - omega E = f
-   // = curl (curl E / omega) - omega E
-   f = 0.0;
-   if (sol !=4)
-   {
-      double E[3], curlE[3], curl2E[3];
-      get_maxwell_solution(x, E, curlE, curl2E);
-      f(0) = curl2E[0] / omega - omega * E[0];
-      f(1) = curl2E[1] / omega - omega * E[1];
-      f(2) = curl2E[2] / omega - omega * E[2];
-   }
-}
-
-void get_maxwell_solution(const Vector &X, double E[], double curlE[],
-                          double curl2E[])
-{
-   double x = X[0];
-   double y = X[1];
-   double z = X[2];
-
-
-
-   if (sol ==-1)
-   {
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
-      E[2] = x * y * (1.0 - x) * (1.0 - y);
-
-      curlE[0] = -(x-1.0) * x * (y*(2.0*z-3.0)+1.0);
-      curlE[1] = -2.0*(y-1.0)*y*(x-z);
-      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
-
-      curl2E[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
-      curl2E[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
-      curl2E[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
-   }
-   else if (sol == 0) // polynomial
-   {
-      // Polynomial vanishing on the boundary
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = (1.0 - x) * x * y * (1.0 - z) * z;
-      E[2] = (1.0 - x) * x * (1.0 - y) * y;
-      //
-      curlE[0] = -(-1.0 + x) * x * (1.0 + y * (-3.0 + 2.0 * z));
-      curlE[1] = -2.0 * (-1.0 + y) * y * (x - z);
-      curlE[2] = (1.0 + (-3.0 + 2.0 * x) * y) * (-1.0 + z) * z;
-
-      curl2E[0] = -2.0 * (-1.0 + y) * y + (-3.0 + 2.0 * x) * (-1.0 + z) * z;
-      curl2E[1] = -2.0 * y * (-x + x * x + (-1.0 + z) * z);
-      curl2E[2] = -2.0 * (-1.0 + y) * y + (-1.0 + x) * x * (-3.0 + 2.0 * z);
-   }
-   else if (sol == 1) // sinusoidal
-   {
-      E[0] = sin(omega * y);
-      E[1] = sin(omega * z);
-      E[2] = sin(omega * x);
-
-      curlE[0] = -omega * cos(omega * z);
-      curlE[1] = -omega * cos(omega * x);
-      curlE[2] = -omega * cos(omega * y);
-
-      curl2E[0] = omega * omega * E[0];
-      curl2E[1] = omega * omega * E[1];
-      curl2E[2] = omega * omega * E[2];
-   }
-   else if (sol == 2) // point source
-   {
-      // shift to avoid singularity
-      double x0 = x + 0.1;
-      double x1 = y + 0.1;
-      double x2 = z + 0.1;
-      //
-      double r = sqrt(x0 * x0 + x1 * x1 + x2 * x2);
-
-      E[0] = cos(omega * r);
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      double r_x = x0 / r;
-      double r_y = x1 / r;
-      double r_z = x2 / r;
-      double r_xy = -(r_x / r) * r_y;
-      double r_xz = -(r_x / r) * r_z;
-      double r_yx = r_xy;
-      double r_yy = (1.0 / r) * (1.0 - r_y * r_y);
-      double r_zx = r_xz;
-      double r_zz = (1.0 / r) * (1.0 - r_z * r_z);
-
-      curlE[0] = 0.0;
-      curlE[1] = -omega * r_z * sin(omega * r);
-      curlE[2] =  omega * r_y * sin(omega * r);
-
-      curl2E[0] = omega * ((r_yy + r_zz) * sin(omega * r) +
-                           (omega * r_y * r_y + omega * r_z * r_z) * cos(omega * r));
-      curl2E[1] = -omega * (r_yx * sin(omega * r) + omega * r_y * r_x * cos(
-                               omega * r));
-      curl2E[2] = -omega * (r_zx * sin(omega * r) + omega * r_z * r_x * cos(
-                               omega * r));
-   }
-   else if (sol == 3) // plane wave
-   {
-      double coeff = omega / sqrt(3.0);
-      E[0] = cos(coeff * (x + y + z));
-      E[1] = 0.0;
-      E[2] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = -coeff * sin(coeff * (x + y + z));
-      curlE[2] = coeff * sin(coeff * (x + y + z));
-
-      curl2E[0] = 2.0 * coeff * coeff * E[0];
-      curl2E[1] = -coeff * coeff * E[0];
-      curl2E[2] = -coeff * coeff * E[0];
-   }
-   else if (sol == -1)
-   {
-      E[0] = cos(omega * y);
-      E[1] = 0.0;
-
-      curlE[0] = 0.0;
-      curlE[1] = 0.0;
-      curlE[2] = -omega * sin(omega * y);
-
-      curl2E[0] = omega*omega * cos(omega*y);
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-   else if (sol == 4) // Airy function
-   {
-      E[0] = 0;
-      E[1] = 0;
-      // double b = -pow(omega/4.0,2.0/3.0)*(4.0*x(0)-1.0);
-      double b = -pow(omega/4.0,2.0/3.0)*(4.0*x-1.0);
-      //E[2] = boost::math::airy_ai(b);
-      E[2] = gsl_sf_airy_Ai(b, GSL_PREC_DOUBLE);
-
-      curlE[0] = 0.0;
-      curlE[1] = 4.0 * pow(omega/4.0,2.0/3.0) * gsl_sf_airy_Ai_deriv(b,
-                                                                     GSL_PREC_DOUBLE);
-      curlE[2] = 0.0;
-
-      //  not used
-      curl2E[0] = 0.0;
-      curl2E[1] = 0.0;
-      curl2E[2] = 0.0;
-   }
-}
-
-
-void epsilon_func(const Vector &x, DenseMatrix &M)
-{
-   M.SetSize(3);
-
-   M = 0.0;
-   M(0,0) = 1.0;
-   M(1,1) = 1.0;
-   if (sol != 4)
-   {
-      M(2,2) = 1.0;
-   }
-   else
-   {
-      M(2,2) = 4.0*x(0)-1.0;
-      // M(2,2) = 2.0;
-   }
-}
-
-void epsilon2_func(const Vector &x, DenseMatrix &M)
-{
-   M.SetSize(3);
-
-   M = 0.0;
-   M(0,0) = 1.0;
-   M(1,1) = 1.0;
-   if (sol != 4)
-   {
-      M(2,2) = 1.0;
-   }
-   else
-   {
-      M(2,2) = (4.0*x(0)-1.0) * (4.0*x(0)-1.0);
-      // M(2,2) = 4.0;
-   }
-}
@@ -1,915 +0,0 @@
-//                       MFEM Example 3 - Parallel Version
-//
-// Compile with: make ex3p
-//
-// Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh
-//               mpirun -np 4 ex3p -m ../data/escher.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/fichera.mesh
-//               mpirun -np 4 ex3p -m ../data/fichera-q2.vtk
-//               mpirun -np 4 ex3p -m ../data/fichera-q3.mesh
-//               mpirun -np 4 ex3p -m ../data/square-disc-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/beam-hex-nurbs.mesh
-//               mpirun -np 4 ex3p -m ../data/amr-quad.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/amr-hex.mesh
-//               mpirun -np 4 ex3p -m ../data/star-surf.mesh -o 2
-//               mpirun -np 4 ex3p -m ../data/mobius-strip.mesh -o 2 -f 0.1
-//               mpirun -np 4 ex3p -m ../data/klein-bottle.mesh -o 2 -f 0.1
-//
-// Description:  This example code solves a simple electromagnetic diffusion
-//               problem corresponding to the second order definite Maxwell
-//               equation curl curl E + E = f with boundary condition
-//               E x n = <given tangential field>. Here, we use a given exact
-//               solution E and compute the corresponding r.h.s. f.
-//               We discretize with Nedelec finite elements in 2D or 3D.
-//
-//               The example demonstrates the use of H(curl) finite element
-//               spaces with the curl-curl and the (vector finite element) mass
-//               bilinear form, as well as the computation of discretization
-//               error when the exact solution is known. Static condensation is
-//               also illustrated.
-//
-//               We recommend viewing examples 1-2 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, E, and r.h.s., f. See below for implementation.
-void E_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-int dim;
-
-#define SIGMAVAL -250.0
-//#define FORM_DEFINITE
-//#define SOLVE_A2
-//#define ITER_A2
-
-//#define USE_CSL
-
-//#define USE_HELMHOLTZ
-
-//#define TEST_MULTIPLE_SP
-
-#ifdef USE_HELMHOLTZ
-void GetHelmholtzMatrix(ParMesh *pmesh, const int dir, HypreParMatrix *A)
-{
-   const int order = 1;
-   FiniteElementCollection *fec;
-   fec = new H1_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-
-   Array<int> ess_tdof_list;
-
-   const bool homogeneousBCeverywhere = false;
-   if (homogeneousBCeverywhere)
-   {
-      if (pmesh->bdr_attributes.Size())
-      {
-         Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-         ess_bdr = 1;
-         fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-   }
-   else
-   {
-      // Set boundary conditions, depending on dir.
-      MFEM_VERIFY(dim == 3, "");
-      for (int i=0; i<pmesh->GetNBE(); ++i)
-      {
-         Element *elem = pmesh->GetBdrElement(i);
-         MFEM_VERIFY(elem->GetNVertices() >= 3, "");
-         const int *vertices = elem->GetVertices();
-         double *v[3];
-         for (int j=0; j<3; ++j)
-         {
-            v[j] = pmesh->GetVertex(vertices[j]);
-         }
-
-         double u[3];
-         double w[3];
-         for (int j=0; j<3; ++j)
-         {
-            u[j] = v[1][j] - v[0][j];  // An edge tangent
-            w[j] = v[2][j] - v[1][j];  // Another edge tangent, not parallel to u.
-         }
-
-         double n[3];  // normal vector, taken as the cross product u x v
-         n[0] = (u[1]*w[2]) - (u[2]*w[1]);
-         n[1] = (u[2]*w[0]) - (u[0]*w[2]);
-         n[2] = (u[0]*w[1]) - (u[1]*w[0]);
-
-         double t = sqrt((n[0]*n[0]) + (n[1]*n[1]) + (n[2]*n[2]));
-
-         int d = -1;
-         for (int j=0; j<3; ++j)
-         {
-            n[j] /= t;  // normalize
-            if (fabs(fabs(n[j]) - 1.0) < 1.0e-8)
-            {
-               d = j;
-            }
-         }
-
-         MFEM_VERIFY(d >= 0, "");
-
-         if (d != dir)  // face has essential BC at all DOF's.
-         {
-            elem->SetAttribute(1);
-         }
-         else
-         {
-            elem->SetAttribute(0);
-         }
-      }
-
-      Array<int> ess_bdr(2);
-      ess_bdr = 0;
-      ess_bdr[1] = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient neg(SIGMAVAL);
-
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   a->AddDomainIntegrator(new MassIntegrator(neg));
-
-   ParLinearForm *b = new ParLinearForm(fespace);
-   ConstantCoefficient zero(0.0);
-   b->AddDomainIntegrator(new DomainLFIntegrator(zero));
-   b->Assemble();
-
-   bool static_cond = false;
-
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   ParGridFunction x(fespace);
-   x = 0.0;
-
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, *A, X, B);
-}
-#endif
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../data/beam-tet.mesh";
-   int order = 2;
-   bool static_cond = false;
-   bool visualization = 1;
-#ifdef MFEM_USE_STRUMPACK
-   bool use_strumpack = false;
-#endif
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&use_strumpack, "-strumpack", "--strumpack-solver",
-                  "-no-strumpack", "--no-strumpack-solver",
-                  "Use STRUMPACK's double complex linear solver.");
-#endif
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-   kappa = freq * M_PI;
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   dim = mesh->Dimension();
-   int sdim = mesh->SpaceDimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      //(int)floor(log(100000./mesh->GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh->UniformRefinement();
-      }
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
-   //    meshes need to be reoriented before we can define high-order Nedelec
-   //    spaces on them.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh->UniformRefinement();
-      }
-   }
-   pmesh->ReorientTetMesh();
-
-   {
-      double minsize = pmesh->GetElementSize(0);
-      double maxsize = minsize;
-      for (int i=1; i<pmesh->GetNE(); ++i)
-      {
-         const double size_i = pmesh->GetElementSize(i);
-         minsize = std::min(minsize, size_i);
-         maxsize = std::max(maxsize, size_i);
-      }
-
-      cout << myid << ": Element size range: (" << minsize << ", " << maxsize << ")"
-           << endl;
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use the Nedelec finite elements of the specified order.
-   FiniteElementCollection *fec = new ND_FECollection(order, dim);
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   long globalNE = pmesh->GetGlobalNE();
-   if (myid == 0)
-   {
-      cout << "Number of mesh elements: " << globalNE << endl;
-      cout << "Number of finite element unknowns: " << size << endl;
-      cout << "Root local number of finite element unknowns: " << fespace->TrueVSize()
-           << endl;
-   }
-
-   // 7. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh->bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
-   //    basis functions in the finite element fespace.
-   VectorFunctionCoefficient f(sdim, f_exact);
-   ParLinearForm *b = new ParLinearForm(fespace);
-   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
-   b->Assemble();
-
-   // 9. Define the solution vector x as a parallel finite element grid function
-   //    corresponding to fespace. Initialize x by projecting the exact
-   //    solution. Note that only values from the boundary edges will be used
-   //    when eliminating the non-homogeneous boundary condition to modify the
-   //    r.h.s. vector b.
-   ParGridFunction x(fespace);
-   VectorFunctionCoefficient E(sdim, E_exact);
-   x.ProjectCoefficient(E);
-
-   // 10. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + sigma I, by adding the curl-curl and the
-   //     mass domain integrators.
-   Coefficient *muinv = new ConstantCoefficient(1.0);
-   Coefficient *sigma = new ConstantCoefficient(SIGMAVAL);
-   Coefficient *sigmaAbs = new ConstantCoefficient(fabs(SIGMAVAL));
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
-
-   //cout << myid << ": NBE " << pmesh->GetNBE() << endl;
-
-#ifdef FORM_DEFINITE
-   ParBilinearForm *adef = new ParBilinearForm(fespace);
-   adef->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   adef->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
-
-   if (static_cond) { adef->EnableStaticCondensation(); }
-   adef->Assemble();
-
-   HypreParMatrix Adef;
-   Vector Bdef, Xdef;
-   adef->FormLinearSystem(ess_tdof_list, x, *b, Adef, Xdef, Bdef);
-#endif
-
-#ifdef USE_CSL
-   Vector Bdef, Xdef;
-
-   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
-   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
-   Mform->Assemble();
-
-   // Mform->Finalize();
-
-   HypreParMatrix Mmat, Smat, Mcopy;
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
-                           Bdef);  // There must be a better way than creating two identical matrices.
-
-   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
-   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   Sform->Assemble();
-
-   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
-
-   ParBilinearForm *agrad = new ParBilinearForm(fespace);
-   //agrad->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   agrad->AddDomainIntegrator(new VectorFEMassIntegrator(*muinv));
-
-   if (static_cond) { agrad->EnableStaticCondensation(); }
-   agrad->Assemble();
-   HypreParMatrix Agrad;
-   agrad->FormLinearSystem(ess_tdof_list, x, *b, Agrad, Xdef, Bdef);
-#endif
-
-#ifdef ITER_A2
-   Vector Bdef, Xdef;
-
-   ParBilinearForm *Mform =  new ParBilinearForm(fespace);
-   Mform->AddDomainIntegrator(new VectorFEMassIntegrator(*sigmaAbs));
-   Mform->Assemble();
-
-   Mform->Finalize();
-
-   HypreParMatrix Mmat, Mcopy;
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mmat, Xdef, Bdef);
-   Mform->FormLinearSystem(ess_tdof_list, x, *b, Mcopy, Xdef,
-                           Bdef);  // There must be a better way to implement M^2.
-
-   /*
-   HypreParMatrix *Mmat = Mform->ParallelAssemble();
-   HypreParMatrix *Mcopy = Mform->ParallelAssemble();  // There must be a better way to implement M^2.
-   */
-
-   ParBilinearForm *Sform =  new ParBilinearForm(fespace);
-   Sform->AddDomainIntegrator(new CurlCurlIntegrator(*muinv));
-   Sform->Assemble();
-
-   HypreParMatrix Smat, Scopy;
-   Sform->FormLinearSystem(ess_tdof_list, x, *b, Smat, Xdef, Bdef);
-   Sform->FormLinearSystem(ess_tdof_list, x, *b, Scopy, Xdef, Bdef);
-#endif
-
-   // 11. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   HypreParMatrix A;
-   Vector B, X;
-   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);
-
-#ifdef SOLVE_A2
-   HypreParMatrix Acopy;
-   {
-      Vector Bdum, Xdum;
-      a->FormLinearSystem(ess_tdof_list, x, *b, Acopy, Xdum, Bdum);
-   }
-#endif
-
-   if (myid == 0)
-   {
-      cout << "Size of linear system: " << A.GetGlobalNumRows() << endl;
-   }
-
-   StopWatch chrono;
-   chrono.Clear();
-   chrono.Start();
-
-   //A.Print("maxwell1000_2");
-
-#ifdef MFEM_USE_STRUMPACK
-   if (use_strumpack)
-   {
-      const bool fullDirect = true;
-
-#ifdef USE_CSL
-      const double beta1 = 1.0;
-      const double beta2 = 0.5;
-
-      Mmat *= -beta1;
-
-      // HypreParMatrix *cslRe = Add(1.0, Smat, -beta1, Mmat);
-      HypreParMatrix * cslRe = ParAdd(&Smat, &Mmat);
-
-      Mcopy *= beta2;
-
-      //ComplexHypreParMatrix chpm(cslRe, &Mcopy, false, false);
-      ComplexHypreParMatrix chpm(&A, &Mcopy, false,
-                                 false);  // For the case beta1 = 1.
-
-      HypreParMatrix *cSysMat = chpm.GetSystemMatrix();
-
-      Array<int> block_offsets(3); // number of variables + 1
-      block_offsets[0] = 0;
-      block_offsets[1] = fespace->GetVSize();
-      block_offsets[2] = fespace->GetVSize();
-      block_offsets.PartialSum();
-
-      Array<int> block_trueOffsets(3); // number of variables + 1
-      block_trueOffsets[0] = 0;
-      block_trueOffsets[1] = fespace->TrueVSize();
-      block_trueOffsets[2] = fespace->TrueVSize();
-      block_trueOffsets.PartialSum();
-
-      //cout << myid << ": V size " << fespace->GetVSize() << ", true " << fespace->TrueVSize() << ", global true " << size << ", B size "
-      //<< B.Size() << ", X size " << X.Size() << endl;
-
-      // Note that B is of true size.
-      BlockVector trueY(block_trueOffsets), trueX(block_trueOffsets),
-                  trueRhs(block_trueOffsets);
-
-      trueRhs.GetBlock(0) = B;
-      trueRhs.GetBlock(1) = 0.0;
-
-      Operator * Arow = new STRUMPACKRowLocMatrix(*cSysMat);
-
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-      // strumpack->SetSymmetricPattern(true);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-#endif
-
-      if (fullDirect)
-      {
-#ifdef USE_CSL
-         //Solver * precond = strumpack;
-
-         // strumpack->Mult(B, X);
-
-         BlockOperator blockDiagA(block_trueOffsets);
-         for (int i=0; i<2; ++i)
-         {
-            blockDiagA.SetDiagonalBlock(i, &A);
-         }
-
-         ParFiniteElementSpace *prec_fespace =
-            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-         HypreSolver *amsgrad = new HypreAMS(Agrad, prec_fespace);
-
-#ifdef HYPRE_DYLAN
-         {
-            Vector Xtmp(X);
-            amsgrad->Mult(B,
-                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-         }
-
-         HypreAMSG *amsg = new HypreAMSG((HypreAMS*) amsgrad, argc, argv);
-         BlockOperator blockDiagP(block_trueOffsets);
-         for (int i=0; i<2; ++i)
-         {
-            blockDiagP.SetDiagonalBlock(i, amsg);
-         }
-
-         TripleProductOperator strumpackProj(&blockDiagP, strumpack, &blockDiagP, false,
-                                             false, false);
-         ProductOperator prod(&blockDiagA, &strumpackProj, false, false);
-#else
-         ProductOperator prod(&blockDiagA, strumpack, false, false);
-#endif
-
-         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
-         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
-
-         gmres->SetOperator(prod);
-         gmres->SetRelTol(1e-12);
-         gmres->SetMaxIter(1000);
-         gmres->SetPrintLevel(1);
-
-         gmres->Mult(trueRhs, trueY);
-         strumpack->Mult(trueY, trueX);
-
-         X = trueX.GetBlock(0);
-         double xim2 = trueX.GetBlock(1).Norml2();
-         xim2 *= xim2;
-         double sumxim2 = 0.0;
-
-         MPI_Allreduce(&xim2, &sumxim2, 1, MPI_DOUBLE, MPI_SUM, fespace->GetComm());
-
-         if (myid == 0)
-         {
-            cout << myid << ": norm of Xim " << trueX.GetBlock(1).Norml2() << ", global " <<
-                 sqrt(sumxim2) << endl;
-         }
-
-         delete gmres;
-         delete strumpack;
-         delete Arow;
-#else
-         cout << "Solving with STRUMPACK" << endl;
-
-#ifdef TEST_MULTIPLE_SP
-         const int Ns = 2;
-         std::vector<Operator*> Arows(Ns);
-         std::vector<STRUMPACKSolver*> strumpacks(Ns);
-
-         //Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-         for (int m=0; m<Ns; ++m)
-         {
-            Arows[m] = new STRUMPACKRowLocMatrix(A);
-
-            //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpacks[m] = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpacks[m]->SetPrintFactorStatistics(true);
-            strumpacks[m]->SetPrintSolveStatistics(false);
-            strumpacks[m]->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpacks[m]->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-            // strumpack->SetSymmetricPattern(true);
-            strumpacks[m]->SetOperator(*Arows[m]);
-            strumpacks[m]->SetFromCommandLine();
-            //Solver * precond = strumpack;
-
-            strumpacks[m]->Mult(B, X);
-
-            //delete strumpack;
-            //delete Arow;
-         }
-#else
-         Operator * Arow = new STRUMPACKRowLocMatrix(A);
-
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-         strumpack->SetPrintFactorStatistics(true);
-         strumpack->SetPrintSolveStatistics(false);
-         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         // strumpack->SetMC64Job(strumpack::MC64Job::NONE);
-         // strumpack->SetSymmetricPattern(true);
-         strumpack->SetOperator(*Arow);
-         strumpack->SetFromCommandLine();
-         //Solver * precond = strumpack;
-
-         cout << "Solving with strumpack one time" << endl;
-
-         strumpack->Mult(B, X);
-
-         delete strumpack;
-         delete Arow;
-#endif
-#endif
-      }
-      else
-      {
-         ParFiniteElementSpace *prec_fespace =
-            (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-         HypreSolver *ams = new HypreAMS(A, prec_fespace);
-
-#ifdef HYPRE_DYLAN
-         {
-            Vector Xtmp(X);
-            ams->Mult(B,
-                      Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-         }
-
-         HypreParMatrix H[3];
-#ifdef USE_HELMHOLTZ
-         for (int i=0; i<3; ++i)
-         {
-            GetHelmholtzMatrix(pmesh, i, &(H[i]));
-         }
-#endif
-
-#ifdef USE_CSL
-         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, &trueX, &trueY,
-                                         (HypreAMS*) ams, argc, argv);
-#else
-         HypreIAMS *iams = new HypreIAMS(A, H, strumpack, NULL, NULL, (HypreAMS*) ams,
-                                         argc, argv);
-#endif
-
-         GMRESSolver *gmres = new GMRESSolver(fespace->GetComm());
-         //FGMRESSolver *gmres = new FGMRESSolver(fespace->GetComm());
-         //BiCGSTABSolver *gmres = new BiCGSTABSolver(fespace->GetComm());
-         //MINRESSolver *gmres = new MINRESSolver(fespace->GetComm());
-
-         gmres->SetOperator(A);
-         gmres->SetRelTol(1e-16);
-         gmres->SetMaxIter(1000);
-         gmres->SetPrintLevel(1);
-
-#ifdef SOLVE_A2
-         {
-            StopWatch chronoA2;
-            chronoA2.Clear();
-            chronoA2.Start();
-
-            HypreParMatrix * A2 = ParMult(&A, &Acopy);
-
-            chronoA2.Stop();
-            cout << "A2 setup time " << chronoA2.RealTime() << endl;
-
-            Vector AB(B);
-            A.Mult(B, AB);
-            gmres->SetOperator(*A2);
-
-            HypreSolver *ams2 = new HypreAMS(A, prec_fespace);
-            {
-               Vector Xtmp(X);
-               ams2->Mult(B,
-                          Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-            }
-
-#ifdef ITER_A2
-            // Iteratively solve 0.5 (A^2 + S^2 + M^2) u^{k+1} = 0.5 (SM + MS) u^k + Ab
-
-            StopWatch chronoIterA2;
-            chronoIterA2.Clear();
-            chronoIterA2.Start();
-
-            HypreParMatrix * M2 = ParMult(&Mmat, &Mcopy);
-            HypreParMatrix * S2 = ParMult(&Smat, &Scopy);
-
-            HypreParMatrix * MS = ParMult(&Mmat, &Scopy);
-            HypreParMatrix * SM = ParMult(&Smat, &Mcopy);
-
-            HypreParMatrix * Bmat = ParAdd(SM, MS);
-            (*Bmat) *= 0.5;
-
-            // TODO: there must be a better way to form a sum of three matrices. Of course, we could define an operator that does 3 mat-vecs.
-            //HypreParMatrix * S2M2 = ParAdd(S2, M2);
-            //HypreParMatrix * iterMat = ParAdd(A2, S2M2);
-            HypreParMatrix * iterMat = ParAdd(A2, Bmat);
-
-            chronoIterA2.Stop();
-            cout << "Iter A2 setup time " << chronoIterA2.RealTime() << endl;
-
-            /*
-            HypreSolver *ams3 = new HypreAMS(*iterMat, prec_fespace);
-            {
-              Vector Xtmp(X);
-              ams3->Mult(B, Xtmp);  // Just a hack to get ams to run its setup function. There should be a better way.
-            }
-            */
-
-            /*
-            // GMRES
-            gmres->SetOperator(*iterMat);
-            gmres->SetPreconditioner(*ams2);
-            */
-
-
-            //HypreBoomerAMG *amg = new HypreBoomerAMG(*iterMat);
-            HypreBoomerAMG *amg = new HypreBoomerAMG(*A2);
-
-            // PCG
-            HyprePCG *pcg = new HyprePCG(*iterMat);
-            //HyprePCG *pcg = new HyprePCG(*A2);
-            pcg->SetTol(1e-12);
-            pcg->SetMaxIter(10);
-            pcg->SetPrintLevel(2);
-            pcg->SetPreconditioner(*amg);
-
-            /*
-            // Strumpack linear solver
-            Operator * Arow = new STRUMPACKRowLocMatrix(*iterMat);
-
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpack->SetPrintFactorStatistics(true);
-            strumpack->SetPrintSolveStatistics(false);
-            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->SetOperator(*Arow);
-            strumpack->SetFromCommandLine();
-            */
-
-            Vector iterRHS(AB);
-            Vector iterU(AB);
-            Vector iterU0(AB);
-
-            iterU = 0.0;
-            iterU0 = 0.0;
-
-            bool iterate = true;
-            int numIter = 0;
-            while (iterate)
-            {
-               iterRHS = iterU;
-               iterRHS.Add(-1.0, iterU0);
-
-               cout << "Iteration " << numIter + 1 << ": diff norm " << iterRHS.Norml2() <<
-                    endl;
-
-               iterU0 = iterU;
-
-               Bmat->Mult(iterU0, iterRHS);
-               //iterRHS.Add(2.0, AB);
-               iterRHS.Add(1.0, AB);
-
-               //gmres->Mult(iterRHS, iterU);
-               pcg->Mult(iterRHS, iterU);
-               //strumpack->Mult(iterRHS, iterU);
-
-               numIter++;
-
-               if (numIter > 100)
-               {
-                  iterate = false;
-               }
-            }
-
-            //delete strumpack;
-            //delete Arow;
-
-            delete pcg;
-
-            X = iterU;
-#else
-            //HypreIAMS *iams2 = new HypreIAMS(*A2, (HypreAMS*) ams2, argc, argv);
-            //gmres->SetPreconditioner(*iams2);
-            cout << myid << ": Solving" << endl;
-            gmres->SetPreconditioner(*ams2);
-            gmres->Mult(AB, X);
-            cout << myid << ": Solved" << endl;
-            return 3;
-#endif
-         }
-#else
-         gmres->SetPreconditioner(*iams);
-         gmres->Mult(B, X);
-#endif
-#else
-         HypreGMRES *gmres = new HypreGMRES(A);
-         gmres->SetTol(1e-12);
-         gmres->SetMaxIter(100);
-         gmres->SetPrintLevel(10);
-
-#ifdef FORM_DEFINITE
-         HypreSolver *amsdef = new HypreAMS(Adef, prec_fespace);
-         gmres->SetPreconditioner(*amsdef);
-#else
-         gmres->SetPreconditioner(*ams);
-#endif
-         gmres->Mult(B, X);
-#endif
-
-         delete gmres;
-         //delete iams;
-         //delete ams;
-      }
-   }
-   else
-#endif
-   {
-      // 12. Define and apply a parallel PCG solver for AX=B with the AMS
-      //     preconditioner from hypre.
-      ParFiniteElementSpace *prec_fespace =
-         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      HypreSolver *ams = new HypreAMS(A, prec_fespace);
-      HyprePCG *pcg = new HyprePCG(A);
-      pcg->SetTol(1e-12);
-      pcg->SetMaxIter(500);
-      pcg->SetPrintLevel(2);
-      pcg->SetPreconditioner(*ams);
-      pcg->Mult(B, X);
-
-      delete pcg;
-      delete ams;
-   }
-
-   chrono.Stop();
-   cout << myid << ": Solver time " << chrono.RealTime() << endl;
-
-   // 13. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a->RecoverFEMSolution(X, *b, x);
-
-   // 14. Compute and print the L^2 norm of the error.
-   {
-      double err = x.ComputeL2Error(E);
-      Vector zeroVec(3);
-      zeroVec = 0.0;
-      VectorConstantCoefficient vzero(zeroVec);
-      ParGridFunction zerogf(fespace);
-      zerogf = 0.0;
-      double normE = zerogf.ComputeL2Error(E);
-      double normX = x.ComputeL2Error(vzero);
-      if (myid == 0)
-      {
-         cout << "|| E_h - E ||_{L^2} = " << err << endl;
-         cout << "|| E_h ||_{L^2} = " << normX << endl;
-         cout << "|| E ||_{L^2} = " << normE << endl;
-      }
-   }
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   delete a;
-   delete sigma;
-   delete muinv;
-   delete b;
-   delete fespace;
-   delete fec;
-   delete pmesh;
-
-   MPI_Finalize();
-
-   return 0;
-}
-
-
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (SIGMAVAL + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -11,12 +11,6 @@
 //               ex5 -m ../data/escher.mesh
 //               ex5 -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               ex5 -m ../data/star.mesh -pa -d cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-omp
-//               ex5 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -56,7 +50,6 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -66,8 +59,6 @@ int main(int argc, char *argv[])
                  "Finite element order (polynomial degree).");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -79,18 +70,13 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
@@ -103,7 +89,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use the
+   // 4. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -111,7 +97,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, hdiv_coll);
   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, l2_coll);

-   // 6. Define the BlockStructure of the problem, i.e. define the array of
+   // 5. Define the BlockStructure of the problem, i.e. define the array of
   //    offsets for each variable. The last component of the Array is the sum
   //    of the dimensions of each block.
   Array<int> block_offsets(3); // number of variables + 1
@@ -126,7 +112,7 @@ int main(int argc, char *argv[])
   std::cout << "dim(R+W) = " << block_offsets.Last() << "\n";
   std::cout << "***********************************************************\n";

-   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 6. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -136,28 +122,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
+   // 7. Allocate memory (x, rhs) for the analytical solution and the right hand
   //    side.  Define the GridFunction u,p for the finite element solution and
   //    linear forms fform and gform for the right hand side.  The data
   //    allocated by x and rhs are passed as a reference to the grid functions
   //    (u,p) and the linear forms (fform, gform).
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
+   BlockVector x(block_offsets), rhs(block_offsets);

   LinearForm *fform(new LinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);

   LinearForm *gform(new LinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);

-   // 9. Assemble the finite element matrices for the Darcy operator
+   // 8. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -202,7 +185,7 @@ int main(int argc, char *argv[])
      darcyOp.SetBlock(1,0, &B);
   }

-   // 10. Construct the operators for preconditioner
+   // 9. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
@@ -219,11 +202,10 @@ int main(int argc, char *argv[])
   if (pa)
   {
      mVarf->AssembleDiagonal(Md);
-      auto Md_host = Md.HostRead();
      Vector invMd(mVarf->Height());
      for (int i=0; i<mVarf->Height(); ++i)
      {
-         invMd(i) = 1.0 / Md_host[i];
+         invMd(i) = 1.0 / Md(i);
      }

      Vector BMBt_diag(bVarf->Height());
@@ -264,7 +246,7 @@ int main(int argc, char *argv[])
   darcyPrec.SetDiagonalBlock(0, invM);
   darcyPrec.SetDiagonalBlock(1, invS);

-   // 11. Solve the linear system with MINRES.
+   // 10. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(1000);
   double rtol(1.e-6);
@@ -281,7 +263,6 @@ int main(int argc, char *argv[])
   solver.SetPrintLevel(1);
   x = 0.0;
   solver.Mult(rhs, x);
-   if (device.IsEnabled()) { x.HostRead(); }
   chrono.Stop();

   if (solver.GetConverged())
@@ -292,7 +273,7 @@ int main(int argc, char *argv[])
                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

-   // 12. Create the grid functions u and p. Compute the L2 error norms.
+   // 11. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
   u.MakeRef(R_space, x.GetBlock(0), 0);
   p.MakeRef(W_space, x.GetBlock(1), 0);
@@ -312,7 +293,7 @@ int main(int argc, char *argv[])
   std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
   std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";

-   // 13. Save the mesh and the solution. This output can be viewed later using
+   // 12. Save the mesh and the solution. This output can be viewed later using
   //     GLVis: "glvis -m ex5.mesh -g sol_u.gf" or "glvis -m ex5.mesh -g
   //     sol_p.gf".
   {
@@ -329,13 +310,13 @@ int main(int argc, char *argv[])
      p.Save(p_ofs);
   }

-   // 14. Save data in the VisIt format
+   // 13. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5", mesh);
   visit_dc.RegisterField("velocity", &u);
   visit_dc.RegisterField("pressure", &p);
   visit_dc.Save();

-   // 15. Save data in the ParaView format
+   // 14. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5", mesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -347,7 +328,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",&p);
   paraview_dc.Save();

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -360,7 +341,7 @@ int main(int argc, char *argv[])
      p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete fform;
   delete gform;
   delete invM;
@@ -11,12 +11,6 @@
 //               mpirun -np 4 ex5p -m ../data/escher.mesh
 //               mpirun -np 4 ex5p -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-omp
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -66,7 +60,6 @@ int main(int argc, char *argv[])
   int order = 1;
   bool par_format = false;
   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;
   bool adios2 = false;

@@ -82,8 +75,6 @@ int main(int argc, char *argv[])
                  "Format to use when saving the results for VisIt.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -105,18 +96,13 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 10,000 elements, unless the user specifies it as input.
@@ -132,7 +118,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -145,7 +131,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -165,7 +151,7 @@ int main(int argc, char *argv[])
      std::cout << "***********************************************************\n";
   }

-   // 8. Define the two BlockStructure of the problem.  block_offsets is used
+   // 7. Define the two BlockStructure of the problem.  block_offsets is used
   //    for Vector based on dof (like ParGridFunction or ParLinearForm),
   //    block_trueOffstes is used for Vector based on trueDof (HypreParVector
   //    for the rhs and solution of the linear system).  The offsets computed
@@ -182,7 +168,7 @@ int main(int argc, char *argv[])
   block_trueOffsets[2] = W_space->TrueVSize();
   block_trueOffsets.PartialSum();

-   // 9. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 8. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -192,30 +178,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 10. Define the parallel grid function and parallel linear forms, solution
-   //     vector and rhs.
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
+   // 9. Define the parallel grid function and parallel linear forms, solution
+   //    vector and rhs.
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);

   ParLinearForm *fform(new ParLinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);
   fform->ParallelAssemble(trueRhs.GetBlock(0));
-   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);

   ParLinearForm *gform(new ParLinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);
   gform->ParallelAssemble(trueRhs.GetBlock(1));
-   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);

-   // 11. Assemble the finite element matrices for the Darcy operator
+   // 10. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -268,7 +249,7 @@ int main(int argc, char *argv[])
      darcyOp->SetBlock(1,0, B);
   }

-   // 12. Construct the operators for preconditioner
+   // 11. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
@@ -285,11 +266,10 @@ int main(int argc, char *argv[])
   {
      Md_PA.SetSize(R_space->GetTrueVSize());
      mVarf->AssembleDiagonal(Md_PA);
-      auto Md_host = Md_PA.HostRead();
      Vector invMd(Md_PA.Size());
      for (int i=0; i<Md_PA.Size(); ++i)
      {
-         invMd(i) = 1.0 / Md_host[i];
+         invMd(i) = 1.0 / Md_PA(i);
      }

      Vector BMBt_diag(W_space->GetTrueVSize());
@@ -322,7 +302,7 @@ int main(int argc, char *argv[])
   darcyPr->SetDiagonalBlock(0, invM);
   darcyPr->SetDiagonalBlock(1, invS);

-   // 13. Solve the linear system with MINRES.
+   // 12. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(pa ? 1000 : 500);
   double rtol(1.e-6);
@@ -339,7 +319,6 @@ int main(int argc, char *argv[])
   solver.SetPrintLevel(verbose);
   trueX = 0.0;
   solver.Mult(trueRhs, trueX);
-   if (device.IsEnabled()) { trueX.HostRead(); }
   chrono.Stop();

   if (verbose)
@@ -353,7 +332,7 @@ int main(int argc, char *argv[])
      std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
   }

-   // 14. Extract the parallel grid function corresponding to the finite element
+   // 13. Extract the parallel grid function corresponding to the finite element
   //     approximation X. This is the local solution on each processor. Compute
   //     L2 error norms.
   ParGridFunction *u(new ParGridFunction);
@@ -381,7 +360,7 @@ int main(int argc, char *argv[])
      std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_*".
   {
      ostringstream mesh_name, u_name, p_name;
@@ -402,7 +381,7 @@ int main(int argc, char *argv[])
      p->Save(p_ofs);
   }

-   // 16. Save data in the VisIt format
+   // 15. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5-Parallel", pmesh);
   visit_dc.RegisterField("velocity", u);
   visit_dc.RegisterField("pressure", p);
@@ -411,7 +390,7 @@ int main(int argc, char *argv[])
                      DataCollection::PARALLEL_FORMAT);
   visit_dc.Save();

-   // 17. Save data in the ParaView format
+   // 16. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5P", pmesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -423,7 +402,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",p);
   paraview_dc.Save();

-   // 18. Optionally output a BP (binary pack) file using ADIOS2. This can be
+   // 17. Optionally output a BP (binary pack) file using ADIOS2. This can be
   //     visualized with the ParaView VTX reader.
 #ifdef MFEM_USE_ADIOS2
   if (adios2)
@@ -443,7 +422,7 @@ int main(int argc, char *argv[])
   }
 #endif

-   // 19. Send the solution by socket to a GLVis server.
+   // 18. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -463,7 +442,7 @@ int main(int argc, char *argv[])
             << endl;
   }

-   // 20. Free the used memory.
+   // 19. Free the used memory.
   delete fform;
   delete gform;
   delete u;
@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//             * ex6 -pa -d ceed-cuda
+//               * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -108,11 +108,7 @@ int main(int argc, char *argv[])
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   BilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   LinearForm b(&fespace);

   ConstantCoefficient one(1.0);
@@ -203,10 +199,9 @@ int main(int argc, char *argv[])
         umf_solver.Mult(B, X);
 #endif
      }
-      else // Diagonal preconditioning in partial assembly mode.
+      else // No preconditioning for now in partial assembly mode.
      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 3, 2000, 1e-12, 0.0);
+         CG(*A, B, X, 3, 2000, 1e-12, 0.0);
      }

      // 18. After solving the linear system, reconstruct the solution as a
@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//             * mpirun -np 4 ex6p -pa -d ceed-cuda
+//               * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -129,11 +129,7 @@ int main(int argc, char *argv[])
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   ParBilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   ParLinearForm b(&fespace);

   ConstantCoefficient one(1.0);
@@ -224,26 +220,17 @@ int main(int argc, char *argv[])

      // 17. Solve the linear system A X = B.
      //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-      //     * With partial assembly, use a diagonal preconditioner.
-      Solver *M = NULL;
-      if (pa)
-      {
-         M = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-      else
-      {
-         HypreBoomerAMG *amg = new HypreBoomerAMG;
-         amg->SetPrintLevel(0);
-         M = amg;
-      }
+      //     * With partial assembly, use no preconditioner, for now.
+      HypreBoomerAMG *amg = NULL;
+      if (!pa) { amg = new HypreBoomerAMG; amg->SetPrintLevel(0); }
      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-6);
      cg.SetMaxIter(2000);
      cg.SetPrintLevel(3); // print the first and the last iterations only
-      cg.SetPreconditioner(*M);
+      if (amg) { cg.SetPreconditioner(*amg); }
      cg.SetOperator(*A);
      cg.Mult(B, X);
-      delete M;
+      delete amg;

      // 18. Switch back to the host and extract the parallel grid function
      //     corresponding to the finite element approximation X. This is the
@@ -20,11 +20,8 @@
 // Device sample runs:
 //    ex9 -pa
 //    ex9 -ea
-//    ex9 -fa
 //    ex9 -pa -m ../data/periodic-cube.mesh
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -147,7 +144,6 @@ int main(int argc, char *argv[])
   int order = 3;
   bool pa = false;
   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -174,8 +170,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -284,11 +278,6 @@ int main(int argc, char *argv[])
      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
@@ -448,19 +437,21 @@ int main(int argc, char *argv[])
 FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
+   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+   bool ea = M.GetAssemblyLevel() == AssemblyLevel::ELEMENT;
   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
+   if (pa || ea)
   {
      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
      M_solver.SetOperator(M);
      dg_solver = NULL;
   }
+   else
+   {
+      M_prec = new DSmoother(M.SpMat());
+      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
+      M_solver.SetOperator(M.SpMat());
+   }
   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
@@ -21,11 +21,8 @@
 // Device sample runs:
 //    mpirun -np 4 ex9p -pa
 //    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -167,7 +164,6 @@ int main(int argc, char *argv[])
   int order = 3;
   bool pa = false;
   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -197,8 +193,6 @@ int main(int argc, char *argv[])
                  "--no-partial-assembly", "Enable Partial Assembly.");
   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -335,12 +329,6 @@ int main(int argc, char *argv[])
      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
   m->AddDomainIntegrator(new MassIntegrator);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
@@ -577,21 +565,29 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
+   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
+   bool ea = _M.GetAssemblyLevel()==AssemblyLevel::ELEMENT;
+
+   if (pa || ea)
   {
      M.Reset(&_M, false);
      K.Reset(&_K, false);
   }
+   else
+   {
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
+   }

   M_solver.SetOperator(*M);

   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
+   if (pa || ea)
+   {
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
+      dg_solver = NULL;
+   }
+   else
   {
      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -600,11 +596,6 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,

      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }

   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
@@ -114,11 +114,6 @@ ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-ex11p-test-superlu: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
-test-par-YES: ex11p-test-superlu
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -1,517 +0,0 @@
-#pragma once
-
-#include "mfem.hpp"
-
-using namespace std;
-
-namespace mfem
-{
-
-STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
-{
-   //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
-   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
-   strumpack->SetPrintFactorStatistics(true);
-   strumpack->SetPrintSolveStatistics(false);
-   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-   strumpack->SetOperator(*Arow);
-   strumpack->SetFromCommandLine();
-   return strumpack;
-}
-
-hypre_CSRMatrix* GetHypreParMatrixData(const HypreParMatrix & hypParMat)
-{
-   // First cast the parameter to a hypre_ParCSRMatrix
-   hypre_ParCSRMatrix * parcsr_op =
-      (hypre_ParCSRMatrix *)const_cast<HypreParMatrix&>(hypParMat);
-
-   MFEM_ASSERT(parcsr_op != NULL,"STRUMPACK: const_cast failed in SetOperator");
-
-   // Create the CSRMatrixMPI A_ by borrowing the internal data from a hypre_CSRMatrix.
-   return hypre_MergeDiagAndOffd(parcsr_op);
-}
-
-// Row and column offsets are assumed to be the same, for each process.
-// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
-HypreParMatrix* CreateHypreParMatrixFromBlocks(MPI_Comm comm,
-                                               Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
-                                               Array2D<SparseMatrix*> const& blocksSp,
-                                               Array2D<double> const& coefficient,
-                                               std::vector<std::vector<int> > const& blockProcOffsets,
-                                               std::vector<std::vector<int> > const& all_block_num_loc_rows)
-{
-   const int numBlocks = offsets.Size() - 1;
-   const int num_loc_rows = offsets[numBlocks];
-
-   int nprocs, rank;
-   MPI_Comm_rank(comm, &rank);
-   MPI_Comm_size(comm, &nprocs);
-
-   std::vector<int> all_num_loc_rows(nprocs);
-   std::vector<int> procOffsets(nprocs);
-   std::vector<std::vector<int> > procBlockOffsets(nprocs);
-
-   MPI_Allgather(&num_loc_rows, 1, MPI_INT, all_num_loc_rows.data(), 1, MPI_INT,
-                 comm);
-
-   int first_loc_row = 0;
-   int glob_nrows = 0;
-   procOffsets[0] = 0;
-   for (int i=0; i<nprocs; ++i)
-   {
-      glob_nrows += all_num_loc_rows[i];
-      if (i < rank)
-      {
-         first_loc_row += all_num_loc_rows[i];
-      }
-
-      if (i < nprocs-1)
-      {
-         procOffsets[i+1] = procOffsets[i] + all_num_loc_rows[i];
-      }
-
-      if (numBlocks > 0)
-      {
-         procBlockOffsets[i].resize(numBlocks);
-         procBlockOffsets[i][0] = 0;
-      }
-
-      for (int j=1; j<numBlocks; ++j)
-      {
-         procBlockOffsets[i][j] = procBlockOffsets[i][j-1] + all_block_num_loc_rows[j
-                                                                                    -1][i];
-      }
-   }
-
-   const int glob_ncols = glob_nrows;
-
-   std::vector<int> opI(num_loc_rows+1);
-   std::vector<int> cnt(num_loc_rows);
-
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      opI[i] = 0;
-      cnt[i] = 0;
-   }
-
-   opI[num_loc_rows] = 0;
-
-   Array2D<hypre_CSRMatrix*> csr_blocks(numBlocks, numBlocks);
-
-   // Loop over all blocks, to determine nnz for each row.
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (blocks(i, j) == NULL)
-         {
-            csr_blocks(i, j) = NULL;
-
-            if (blocksSp(i, j) != NULL)
-            {
-               const int nrows = blocksSp(i, j)->Height();
-               for (int k=0; k<nrows; ++k)
-               {
-                  const int rowg = offsets[i] + k;
-                  opI[rowg + 1] += blocksSp(i, j)->GetI()[k+1] - blocksSp(i, j)->GetI()[k];
-               }
-            }
-         }
-         else
-         {
-            MFEM_VERIFY(blocksSp(i, j) == NULL, "");
-
-            csr_blocks(i, j) = GetHypreParMatrixData(*(blocks(i, j)));
-
-            const int nrows = csr_blocks(i, j)->num_rows;
-
-            for (int k=0; k<nrows; ++k)
-            {
-               const int rowg = offsets[i] + k;
-               //(*(leftInjection(i, j)))[k]
-               opI[rowg + 1] += csr_blocks(i, j)->i[k+1] - csr_blocks(i, j)->i[k];
-            }
-         }
-      }
-   }
-
-   // Now opI[i] is nnz for row i-1. Do a partial sum to get offsets.
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      opI[i+1] += opI[i];
-   }
-
-   const int nnz = opI[num_loc_rows];
-
-   std::vector<HYPRE_Int> opJ(nnz);
-   std::vector<double> data(nnz);
-
-   // Loop over all blocks, to set matrix data.
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (csr_blocks(i, j) != NULL || blocksSp(i, j) != NULL)
-         {
-            const bool useCSR = (csr_blocks(i, j) != NULL);
-
-            const int nrows = useCSR ? csr_blocks(i, j)->num_rows : blocksSp(i,
-                                                                             j)->Height();
-            const double coef = coefficient(i, j);
-
-            int *Iarray = useCSR ? csr_blocks(i, j)->i : blocksSp(i, j)->GetI();
-
-            //const bool failure = (nrows != offsets[i+1] - offsets[i]);
-
-            MFEM_VERIFY(nrows == offsets[i+1] - offsets[i], "");
-
-            for (int k=0; k<nrows; ++k)
-            {
-               const int rowg = offsets[i] + k;  // process-local row
-               const int nnz_k = Iarray[k+1] - Iarray[k];
-               const int osk = Iarray[k];
-
-               for (int l=0; l<nnz_k; ++l)
-               {
-                  // Find the column process offset for the block.
-                  const int bcol = useCSR ? csr_blocks(i, j)->j[osk + l] : blocksSp(i,
-                                                                                    j)->GetJ()[osk + l];
-                  int bcolproc = 0;
-
-                  for (int p=1; p<nprocs; ++p)
-                  {
-                     if (blockProcOffsets[j][p] > bcol)
-                     {
-                        bcolproc = p-1;
-                        break;
-                     }
-                  }
-
-                  if (blockProcOffsets[j][nprocs - 1] <= bcol)
-                  {
-                     bcolproc = nprocs - 1;
-                  }
-
-                  const int colg = procOffsets[bcolproc] + procBlockOffsets[bcolproc][j] +
-                                   (bcol - blockProcOffsets[j][bcolproc]);
-
-                  if (colg < 0)
-                  {
-                     cout << "BUG, negative global column index" << endl;
-                  }
-
-                  opJ[opI[rowg] + cnt[rowg]] = colg;
-                  data[opI[rowg] + cnt[rowg]] = useCSR ? coef * csr_blocks(i,
-                                                                           j)->data[osk + l] : coef * blocksSp(i, j)->GetData()[osk + l];
-                  cnt[rowg]++;
-               }
-            }
-         }
-      }
-   }
-
-   bool cntCheck = true;
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      if (cnt[i] != opI[i+1] - opI[i])
-      {
-         cntCheck = false;
-      }
-   }
-
-   MFEM_VERIFY(cntCheck, "");
-
-   for (int i=0; i<numBlocks; ++i)
-   {
-      for (int j=0; j<numBlocks; ++j)
-      {
-         if (csr_blocks(i, j) != NULL)
-         {
-            hypre_CSRMatrixDestroy(csr_blocks(i, j));
-         }
-      }
-   }
-
-   std::vector<HYPRE_Int> rowStarts2(2);
-   rowStarts2[0] = first_loc_row;
-   rowStarts2[1] = first_loc_row + all_num_loc_rows[rank];
-
-   if (nnz > 0)
-   {
-      HYPRE_Int minJ = opJ[0];
-      HYPRE_Int maxJ = opJ[0];
-      for (int i=0; i<nnz; ++i)
-      {
-         minJ = std::min(minJ, opJ[i]);
-         maxJ = std::max(maxJ, opJ[i]);
-
-         if (opJ[i] >= glob_ncols)
-         {
-            cout << "Column indices out of range" << endl;
-         }
-      }
-   }
-
-   HypreParMatrix *hmat = new HypreParMatrix(comm, num_loc_rows, glob_nrows,
-                                             glob_ncols, (int*) opI.data(), (HYPRE_Int*) opJ.data(), (double*) data.data(),
-                                             (HYPRE_Int*) rowStarts2.data(), (HYPRE_Int*) rowStarts2.data());
-
-   return hmat;
-}
-
-class BlockMGSolver : public Solver
-{
-private:
-   /// The linear system matrix
-   Array2D<HypreParMatrix *>&
-   Af;  // TODO: remove this, as it is used only in the constructor
-   Array2D<double>&
-   Acoef;  // TODO: remove this, as it is used only in the constructor
-   vector<Array<int>> Aoffsets;
-   vector<Array<int>> Poffsets_i;
-   vector<Array<int>> Poffsets_j;
-   std::vector<Array2D<HypreParMatrix *>> A;
-   std::vector<HypreParMatrix *>& P;
-   std::vector<BlockOperator *> BlkP;
-   std::vector<BlockOperator *> BlkA;
-   std::vector<BlockOperator *> S;
-   HypreParMatrix * Ac;
-   int numGrids, numBlocks;
-   STRUMPACKSolver *invAc = nullptr;
-   double theta = 0.5;
-
-public:
-   BlockMGSolver(const int height, const int width, Array2D<HypreParMatrix *>& Af_,
-                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *>& P_);
-
-   virtual void SetOperator(const Operator &op) {}
-
-   virtual void SetTheta(const double a) { theta = a; }
-
-   virtual void Mult(const Vector &r, Vector &z) const;
-   virtual ~BlockMGSolver();
-};
-
-BlockMGSolver::BlockMGSolver(const int height, const int width,
-                             Array2D<HypreParMatrix *>& Af_, Array2D<double>& Acoef_,
-                             std::vector<HypreParMatrix *>& P_)
-   : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
-{
-   numBlocks = Af.NumRows();
-   MFEM_VERIFY(Af.NumCols() == numBlocks, "");
-   numGrids = P.size();
-   BlkP.resize(numGrids);
-   BlkA.resize(numGrids+1);
-   S.resize(numGrids);
-   A.resize(numGrids + 1);
-   A[numGrids] = Af;
-   Aoffsets.resize(numGrids+1);
-   Poffsets_i.resize(numGrids);
-   Poffsets_j.resize(numGrids);
-   // Construct Bilinear form Matrices on each level
-   for (int k = numGrids ; k > 0; k--)
-   {
-      A[k - 1].SetSize(numBlocks,numBlocks);
-      Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         Aoffsets[k][i+1] = A[k](i,i)->Height();
-      }
-
-      Aoffsets[k].PartialSum();
-      BlkA[k] = new BlockOperator(Aoffsets[k]);
-      S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
-
-      for (int i=0; i<numBlocks; i++)
-      {
-         for (int j=0; j<numBlocks; j++)
-         {
-            if (A[k](i,j) == NULL)
-            {
-               A[k - 1](i,j) = NULL;
-            }
-            else
-            {
-               A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
-               BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
-            }
-         }
-
-         HypreSmoother *S_i = new HypreSmoother;
-         S_i->SetType(HypreSmoother::Jacobi);
-         S_i->SetOperator(*(A[k](i,i)));
-
-         S[k - 1]->SetBlock(i,i,S_i);
-      }
-
-      Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
-      Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         Poffsets_i[k-1][i+1] = P[k-1]->Height();
-         Poffsets_j[k-1][i+1] = P[k-1]->Width();
-      }
-      Poffsets_i[k-1].PartialSum();
-      Poffsets_j[k-1].PartialSum();
-
-      BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
-      for (int i=0; i<numBlocks; i++)
-      {
-         BlkP[k-1]->SetBlock(i,i,P[k-1]);
-      }
-   }
-   // Set up coarse solve operator
-   // Convert the coarse grid blockmatrix to a HypreParMatrix
-   Array<int> offsets(numBlocks+1);
-   offsets[0]=0;
-   for (int i=0; i<numBlocks; i++)
-   {
-      offsets[i+1]=A[0](i,i)->Height();
-   }
-
-   offsets.PartialSum();
-
-   BlkA[0] = new BlockOperator(offsets);
-
-   Array2D<SparseMatrix*> Asp;
-   //Array2D<double> Acoef;
-   Asp.SetSize(numBlocks,numBlocks);
-   //Acoef.SetSize(numBlocks,numBlocks);
-   for (int i=0; i<numBlocks; i++)
-   {
-      for (int j=0; j<numBlocks; j++)
-      {
-         if (A[0](i,j) != NULL)
-         {
-            BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
-         }
-
-         Asp(i,j) = NULL;
-         //Acoef(i,j) = 1.0;
-      }
-   }
-
-   // Convert to HypreParMatrix
-   HypreParMatrix * Ac;
-
-   std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-   std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-
-   {
-      int nprocs, rank;
-      MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-      MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-      std::vector<int> allnumrows(nprocs);
-      const int blockNumRows = A[0](0,0)->Height();
-      MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT,
-                    MPI_COMM_WORLD);
-
-      for (int b=0; b<numBlocks; ++b)
-      {
-         blockProcOffsets[b].resize(nprocs);
-         all_block_num_loc_rows[b].resize(nprocs);
-      }
-
-      blockProcOffsets[0][0] = 0;
-      for (int i=0; i<nprocs-1; ++i)
-      {
-         blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-      }
-
-      for (int i=0; i<nprocs; ++i)
-      {
-         for (int b=0; b<numBlocks; ++b)
-         {
-            all_block_num_loc_rows[b][i] = allnumrows[i];
-         }
-
-         for (int b=1; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b][i] = blockProcOffsets[0][i];
-         }
-      }
-   }
-
-   Ac = CreateHypreParMatrixFromBlocks(MPI_COMM_WORLD, offsets, A[0], Asp,
-                                       Acoef, blockProcOffsets, all_block_num_loc_rows);
-
-   invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), MPI_COMM_WORLD);
-
-   delete Ac;
-}
-
-void BlockMGSolver::Mult(const Vector &r, Vector &z) const
-{
-   // Residual vectors
-   std::vector<Vector> rv(numGrids + 1);
-   // correction vectors
-   std::vector<Vector> zv(numGrids + 1);
-   // allocation
-   for (int i = 0; i <= numGrids ; i++)
-   {
-      int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
-      rv[i].SetSize(n);
-      zv[i].SetSize(n);
-   }
-   // Initial residual
-   rv[numGrids] = r;
-
-   // smooth and update residuals down to the coarsest level
-   for (int i = numGrids; i > 0 ; i--)
-   {
-      // Pre smooth
-      S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
-      // compute residual
-      int n = BlkA[i]->Width();
-      Vector w(n);
-      BlkA[i]->Mult(zv[i], w);
-      rv[i] -= w;
-      // Restrict
-      BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
-   }
-
-   // Coarse grid Solve
-   invAc->Mult(rv[0], zv[0]);
-   //
-   for (int i = 1; i <= numGrids ; i++)
-   {
-      // Prolong correction
-      Vector u(BlkP[i - 1]->Height());
-      BlkP[i - 1]->Mult(zv[i - 1], u);
-      // Update correction
-      zv[i] += u;
-      // Update residual
-      Vector v(BlkA[i]->Height());
-      BlkA[i]->Mult(u, v); rv[i] -= v;
-      // Post smooth
-      S[i - 1]->Mult(rv[i], v); v *= theta;
-      // Update correction
-      zv[i] += v;
-   }
-   z = zv[numGrids];
-}
-
-BlockMGSolver::~BlockMGSolver()
-{
-   for (int i = numGrids - 1; i >= 0 ; i--)
-   {
-      delete S[i];
-      delete BlkP[i];
-      delete BlkA[i];
-      for (int j=0; j<numBlocks; j++)
-      {
-         for (int k=0; k<numBlocks; k++)
-         {
-            delete A[i](j,k);
-         }
-      }
-      A[i].DeleteAll();
-   }
-   delete BlkA[numGrids];
-   delete invAc;
-   A.clear();
-}
-
-} // namespace mfem
@@ -1,869 +0,0 @@
-#ifndef BGMULTIGRID
-#define BGMULTIGRID
-
-#include "mfem.hpp"
-
-using namespace std;
-
-//namespace mfem {
-namespace blockgmg
-{
-
-// Row and column offsets are assumed to be the same, for each process.
-// Array offsets stores process-local offsets with respect to the blocks. Process offsets are not included.
-HypreParMatrix* CreateHypreParMatrixFromBlocks2(MPI_Comm comm,
-                                                Array<int> const& offsets, Array2D<HypreParMatrix*> const& blocks,
-                                                Array2D<SparseMatrix*> const& blocksSp,
-                                                Array2D<double> const& coefficient,
-                                                std::vector<std::vector<int> > const& blockProcOffsets,
-                                                std::vector<std::vector<int> > const& all_block_num_loc_rows);
-
-class BlockMGSolver : public Solver
-{
-private:
-   /// The linear system matrix
-   Array2D<HypreParMatrix *>&
-   Af;  // TODO: remove this, as it is used only in the constructor
-   Array2D<double>&
-   Acoef;  // TODO: remove this, as it is used only in the constructor
-   vector<Array<int>> Aoffsets;
-   vector<Array<int>> Poffsets_i;
-   vector<Array<int>> Poffsets_j;
-   std::vector<Array2D<HypreParMatrix *>> A;
-   std::vector<HypreParMatrix *> const& P;
-   std::vector<BlockOperator *> BlkP;
-   std::vector<BlockOperator *> BlkA;
-   std::vector<BlockOperator *> S;
-   HypreParMatrix * Ac;
-   SparseMatrix AcSp;
-   int numGrids, numBlocks;
-   //STRUMPACKSolver *invAc = nullptr;
-   Operator *invAc = nullptr;
-   double theta = 0.5;
-
-public:
-   BlockMGSolver(MPI_Comm comm, const int height, const int width,
-                 Array2D<HypreParMatrix *>& Af_,
-                 Array2D<double>& Acoef_, std::vector<HypreParMatrix *> const& P_)
-      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_)
-   {
-      numBlocks = Af.NumRows();
-      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
-      numGrids = P.size();
-      BlkP.resize(numGrids);
-      BlkA.resize(numGrids+1);
-      S.resize(numGrids);
-      A.resize(numGrids + 1);
-      A[numGrids] = Af;
-      Aoffsets.resize(numGrids+1);
-      Poffsets_i.resize(numGrids);
-      Poffsets_j.resize(numGrids);
-      // Construct Bilinear form Matrices on each level
-      for (int k = numGrids ; k > 0; k--)
-      {
-         A[k - 1].SetSize(numBlocks,numBlocks);
-         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Aoffsets[k][i+1] = A[k](i,i)->Height();
-         }
-
-         Aoffsets[k].PartialSum();
-         BlkA[k] = new BlockOperator(Aoffsets[k]);
-         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
-
-         for (int i=0; i<numBlocks; i++)
-         {
-            for (int j=0; j<numBlocks; j++)
-            {
-               if (A[k](i,j) == NULL)
-               {
-                  A[k - 1](i,j) = NULL;
-               }
-               else
-               {
-                  A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
-                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
-               }
-            }
-
-            HypreSmoother *S_i = new HypreSmoother;
-            S_i->SetType(HypreSmoother::Jacobi);
-            S_i->SetOperator(*(A[k](i,i)));
-
-            S[k - 1]->SetBlock(i,i,S_i);
-         }
-
-         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
-         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Poffsets_i[k-1][i+1] = P[k-1]->Height();
-            Poffsets_j[k-1][i+1] = P[k-1]->Width();
-         }
-         Poffsets_i[k-1].PartialSum();
-         Poffsets_j[k-1].PartialSum();
-
-         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
-         for (int i=0; i<numBlocks; i++)
-         {
-            BlkP[k-1]->SetBlock(i,i,P[k-1]);
-         }
-      }
-      // Set up coarse solve operator
-      // Convert the coarse grid blockmatrix to a HypreParMatrix
-      Array<int> offsets(numBlocks+1);
-      offsets[0]=0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         offsets[i+1]=A[0](i,i)->Height();
-      }
-
-      offsets.PartialSum();
-
-      BlkA[0] = new BlockOperator(offsets);
-
-      Array2D<SparseMatrix*> Asp;
-      //Array2D<double> Acoef;
-      Asp.SetSize(numBlocks,numBlocks);
-      //Acoef.SetSize(numBlocks,numBlocks);
-      for (int i=0; i<numBlocks; i++)
-      {
-         for (int j=0; j<numBlocks; j++)
-         {
-            if (A[0](i,j) != NULL)
-            {
-               BlkA[0]->SetBlock(i, j, A[0](i,j), Acoef(i,j));
-            }
-
-            Asp(i,j) = NULL;
-            //Acoef(i,j) = 1.0;
-         }
-      }
-
-      // Convert to HypreParMatrix
-      //HypreParMatrix * Ac;
-
-      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-
-      {
-         int nprocs, rank;
-         MPI_Comm_rank(comm, &rank);
-         MPI_Comm_size(comm, &nprocs);
-
-         std::vector<int> allnumrows(nprocs);
-         const int blockNumRows = A[0](0,
-                                       0)->Height();    // TODO: Not valid if blocks are of different size
-         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
-
-         for (int b=0; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b].resize(nprocs);
-            all_block_num_loc_rows[b].resize(nprocs);
-         }
-
-         blockProcOffsets[0][0] = 0;
-         for (int i=0; i<nprocs-1; ++i)
-         {
-            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-         }
-
-         for (int i=0; i<nprocs; ++i)
-         {
-            for (int b=0; b<numBlocks; ++b)
-            {
-               all_block_num_loc_rows[b][i] = allnumrows[i];
-            }
-
-            for (int b=1; b<numBlocks; ++b)
-            {
-               blockProcOffsets[b][i] = blockProcOffsets[0][i];
-            }
-         }
-      }
-
-      Ac = CreateHypreParMatrixFromBlocks2(comm, offsets, A[0], Asp,
-                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
-
-#ifdef MFEM_USE_STRUMPACK
-      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
-      delete Ac;
-#else
-      Ac->GetDiag(AcSp);  // AcSp does not own the data
-      UMFPackSolver *umf_solver = new UMFPackSolver();
-      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver->SetOperator(AcSp);
-      invAc = umf_solver;
-#endif
-   }
-
-   virtual void SetOperator(const Operator &op) {}
-
-   virtual void SetTheta(const double a) { theta = a; }
-
-   virtual void Mult(const Vector &r, Vector &z) const
-   {
-      // Residual vectors
-      std::vector<Vector> rv(numGrids + 1);
-      // correction vectors
-      std::vector<Vector> zv(numGrids + 1);
-      // allocation
-      for (int i = 0; i <= numGrids ; i++)
-      {
-         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
-         rv[i].SetSize(n);
-         zv[i].SetSize(n);
-      }
-      // Initial residual
-      rv[numGrids] = r;
-
-      // smooth and update residuals down to the coarsest level
-      for (int i = numGrids; i > 0 ; i--)
-      {
-         // Pre smooth
-         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
-         // compute residual
-         int n = BlkA[i]->Width();
-         Vector w(n);
-         BlkA[i]->Mult(zv[i], w);
-         rv[i] -= w;
-         // Restrict
-         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
-      }
-
-      // Coarse grid Solve
-      invAc->Mult(rv[0], zv[0]);
-      //
-      for (int i = 1; i <= numGrids ; i++)
-      {
-         // Prolong correction
-         Vector u(BlkP[i - 1]->Height());
-         BlkP[i - 1]->Mult(zv[i - 1], u);
-         // Update correction
-         zv[i] += u;
-         // Update residual
-         Vector v(BlkA[i]->Height());
-         BlkA[i]->Mult(u, v); rv[i] -= v;
-         // Post smooth
-         S[i - 1]->Mult(rv[i], v); v *= theta;
-         // Update correction
-         zv[i] += v;
-      }
-      z = zv[numGrids];
-   }
-
-   virtual ~BlockMGSolver()
-   {
-      for (int i = numGrids - 1; i >= 0 ; i--)
-      {
-         delete S[i];
-         delete BlkP[i];
-         delete BlkA[i];
-         for (int j=0; j<numBlocks; j++)
-         {
-            for (int k=0; k<numBlocks; k++)
-            {
-               delete A[i](j,k);
-            }
-         }
-         A[i].DeleteAll();
-      }
-      delete BlkA[numGrids];
-      delete invAc;
-      A.clear();
-   }
-
-#ifdef MFEM_USE_STRUMPACK
-   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
-   {
-      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      return strumpack;
-   }
-#endif
-
-};
-
-#define SWTIMING
-#define SERIAL_PROLONGATION
-#define ITERATIVE_COARSE_SOLVE
-#define SPARSE_JACOBI
-//#define SPARSE_ICHOLESKY
-//#define SPARSE_ILU
-//#define COARSE_PA
-//#define BLOCK_DIAG
-
-//#define COARSE_AMS
-
-class BlockMGPASolver : public Solver
-{
-private:
-   /// The linear system matrix
-   Array2D<Operator*>&
-   Af;  // TODO: remove this, as it is used only in the constructor
-   Array2D<double>&
-   Acoef;  // TODO: remove this, as it is used only in the constructor
-   vector<Array<int>> Aoffsets;
-   vector<Array<int>> Poffsets_i;
-   vector<Array<int>> Poffsets_j;
-   Array<int> Coffsets;
-
-   std::vector<Array2D<Operator*>> A;
-#ifdef SERIAL_PROLONGATION
-   std::vector<SparseMatrix *> P;
-#else
-   std::vector<HypreParMatrix *> const& P;
-#endif
-   std::vector<Vector*> const& diag;
-   std::vector<Operator*> Pt;
-   std::vector<BlockOperator *> BlkP;
-   std::vector<BlockOperator *> BlkA;
-   std::vector<BlockOperator *> S;
-   std::vector<OperatorJacobiSmoother*> Jacobi;
-
-   HypreParMatrix * Ac;
-   SparseMatrix AcSp;
-   int numGrids, numBlocks;
-   //STRUMPACKSolver *invAc = nullptr;
-   Operator *invAc = nullptr;
-   double theta = 0.5;
-
-   mutable std::vector<Vector> rv, zv;
-   mutable Vector u, v, w;
-
-   Vector diagAc;
-   Array<int> emptyEssDof;
-   OperatorJacobiSmoother *JacobiAc;
-#ifdef SPARSE_ICHOLESKY
-   Solver *iCholAc;
-#endif
-#ifdef SPARSE_ILU
-   Solver *iluAc;
-#endif
-
-#ifdef BLOCK_DIAG
-   HypreParMatrix *AcDiag;
-   SparseMatrix AcDiagSp;
-#endif
-
-public:
-   BlockMGPASolver(MPI_Comm comm, const int height, const int width,
-                   Array2D<Operator*>& Af_, Array2D<double>& Acoef_,
-#ifdef COARSE_PA
-                   BlockOperator *BlkAc,
-#else
-                   Array2D<HypreParMatrix*> const& BlkAc,
-#endif
-#ifdef COARSE_AMS
-                   ParFiniteElementSpace* coarseFespace,
-#endif
-                   std::vector<HypreParMatrix *> const& P_, std::vector<Vector*> const& diag_,
-                   Array<int>& ess_tdof_list)
-#ifdef SERIAL_PROLONGATION
-      : Solver(height, width), Af(Af_), Acoef(Acoef_), diag(diag_)
-#else
-      : Solver(height, width), Af(Af_), Acoef(Acoef_), P(P_), diag(diag_)
-#endif
-   {
-      timeMult = 0.0;
-      timeMultAc = 0.0;
-      timeMultPresmooth = 0.0;
-      timeMultResidual = 0.0;
-      timeMultRestrict = 0.0;
-
-#ifdef SERIAL_PROLONGATION
-      P.resize(P_.size());
-      for (int i=0; i<P_.size(); ++i)
-      {
-         P[i] = new SparseMatrix();
-         P_[i]->GetDiag(*P[i]);
-         P[i]->BuildTranspose();
-      }
-#endif
-
-      numBlocks = Af.NumRows();
-      MFEM_VERIFY(Af.NumCols() == numBlocks, "");
-#ifndef COARSE_PA
-      MFEM_VERIFY(BlkAc.NumCols() == numBlocks && BlkAc.NumRows() == numBlocks, "");
-#endif
-      numGrids = P.size();
-      MFEM_VERIFY(diag.size() == numBlocks, "");
-      BlkP.resize(numGrids);
-      BlkA.resize(numGrids+1);
-      Pt.resize(numGrids);
-      S.resize(numGrids);
-      A.resize(numGrids + 1);
-      A[numGrids] = Af;
-      Aoffsets.resize(numGrids+1);
-      Poffsets_i.resize(numGrids);
-      Poffsets_j.resize(numGrids);
-
-      Jacobi.resize(numBlocks);
-      for (int i=0; i<numBlocks; ++i)
-      {
-         Jacobi[i] = new OperatorJacobiSmoother(*(diag[i]), ess_tdof_list, 1.0);
-      }
-
-      // Construct Bilinear form Matrices on each level
-      for (int k = numGrids ; k > 0; k--)
-      {
-         A[k - 1].SetSize(numBlocks,numBlocks);
-         Aoffsets[k].SetSize(numBlocks+1); Aoffsets[k][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Aoffsets[k][i+1] = A[k](i,i)->Height();
-         }
-
-         Aoffsets[k].PartialSum();
-         BlkA[k] = new BlockOperator(Aoffsets[k]);
-         S[k-1] = new BlockOperator(Aoffsets[k]);  // Smoother
-         Pt[k - 1] = new TransposeOperator(P[k - 1]);
-
-         for (int i=0; i<numBlocks; i++)
-         {
-            for (int j=0; j<numBlocks; j++)
-            {
-               if (A[k](i,j) == NULL)
-               {
-                  A[k - 1](i,j) = NULL;
-               }
-               else
-               {
-                  //A[k - 1](i,j) = RAP(A[k](i,j), P[k - 1]);
-                  A[k - 1](i,j) = new TripleProductOperator(Pt[k - 1], A[k](i,j), P[k - 1], false,
-                                                            false, false);
-                  BlkA[k]->SetBlock(i, j, A[k](i,j), Acoef(i,j));
-               }
-            }
-
-            if (k == numGrids)  // finest level
-            {
-               S[k - 1]->SetBlock(i,i,Jacobi[i]);
-            }
-            else
-            {
-               S[k - 1]->SetBlock(i,i, new TripleProductOperator(Pt[k], &(S[k]->GetBlock(i,i)),
-                                                                 P[k], false,
-                                                                 false, false));   // TODO: get a diagonal on each level
-            }
-         }
-
-         Poffsets_i[k-1].SetSize(numBlocks+1); Poffsets_i[k-1][0] = 0;
-         Poffsets_j[k-1].SetSize(numBlocks+1); Poffsets_j[k-1][0] = 0;
-         for (int i=0; i<numBlocks; i++)
-         {
-            Poffsets_i[k-1][i+1] = P[k-1]->Height();
-            Poffsets_j[k-1][i+1] = P[k-1]->Width();
-         }
-         Poffsets_i[k-1].PartialSum();
-         Poffsets_j[k-1].PartialSum();
-
-         BlkP[k-1] = new BlockOperator(Poffsets_i[k-1],Poffsets_j[k-1]);
-         for (int i=0; i<numBlocks; i++)
-         {
-            BlkP[k-1]->SetBlock(i,i,P[k-1]);
-         }
-      }
-      // Set up coarse solve operator
-      // Convert the coarse grid blockmatrix to a HypreParMatrix
-      Coffsets.SetSize(numBlocks+1);
-      Coffsets[0]=0;
-      for (int i=0; i<numBlocks; i++)
-      {
-         Coffsets[i+1]=A[0](i,i)->Height();
-#ifndef COARSE_PA
-         MFEM_VERIFY(BlkAc(i,i)->Height() == A[0](i,i)->Height(), "");
-         MFEM_VERIFY(BlkAc(i,i)->Width() == A[0](i,i)->Width(), "");
-#endif
-      }
-      Coffsets.PartialSum();
-
-#ifdef COARSE_PA
-      BlkA[0] = BlkAc;
-#else
-      BlkA[0] = new BlockOperator(Coffsets);
-#endif
-
-      Array2D<SparseMatrix*> Asp;
-      //Array2D<double> Acoef;
-      Asp.SetSize(numBlocks,numBlocks);
-      //Acoef.SetSize(numBlocks,numBlocks);
-      for (int i=0; i<numBlocks; i++)
-      {
-         for (int j=0; j<numBlocks; j++)
-         {
-#ifndef COARSE_PA
-            MFEM_VERIFY((A[0](i,j) == NULL) == (BlkAc(i,j) == NULL), "");
-
-            if (BlkAc(i,j) != NULL)
-            {
-               BlkA[0]->SetBlock(i, j, BlkAc(i,j), Acoef(i,j));
-            }
-#endif
-
-            Asp(i,j) = NULL;
-            //Acoef(i,j) = 1.0;
-         }
-      }
-
-#ifdef COARSE_PA
-      CGSolver *cg_solver = new CGSolver();
-      cg_solver->SetAbsTol(1.0e-6);
-      cg_solver->SetRelTol(1.0e-6);
-      cg_solver->SetMaxIter(1000);
-      cg_solver->SetOperator(*BlkAc);
-      cg_solver->SetPrintLevel(0);
-      cg_solver->iterative_mode = false;
-      invAc = cg_solver;
-#else
-      // Convert to HypreParMatrix
-      HypreParMatrix * Ac;
-
-      std::vector<std::vector<int> > blockProcOffsets(numBlocks);
-      std::vector<std::vector<int> > all_block_num_loc_rows(numBlocks);
-
-      {
-         int nprocs, rank;
-         MPI_Comm_rank(comm, &rank);
-         MPI_Comm_size(comm, &nprocs);
-
-         std::vector<int> allnumrows(nprocs);
-         // TODO: Not valid if blocks are of different size
-         const int blockNumRows = BlkAc(0,0)->Height();
-
-         MPI_Allgather(&blockNumRows, 1, MPI_INT, allnumrows.data(), 1, MPI_INT, comm);
-
-         for (int b=0; b<numBlocks; ++b)
-         {
-            blockProcOffsets[b].resize(nprocs);
-            all_block_num_loc_rows[b].resize(nprocs);
-         }
-
-         blockProcOffsets[0][0] = 0;
-         for (int i=0; i<nprocs-1; ++i)
-         {
-            blockProcOffsets[0][i+1] = blockProcOffsets[0][i] + allnumrows[i];
-         }
-
-         for (int i=0; i<nprocs; ++i)
-         {
-            for (int b=0; b<numBlocks; ++b)
-            {
-               all_block_num_loc_rows[b][i] = allnumrows[i];
-            }
-
-            for (int b=1; b<numBlocks; ++b)
-            {
-               blockProcOffsets[b][i] = blockProcOffsets[0][i];
-            }
-         }
-      }
-
-      Ac = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAc, Asp,
-                                           Acoef, blockProcOffsets, all_block_num_loc_rows);
-
-#ifdef BLOCK_DIAG
-      {
-         Array2D<HypreParMatrix*> BlkAcDiag(numBlocks, numBlocks);
-         Array2D<double> DiagCoef(numBlocks, numBlocks);
-         DiagCoef = 0.0;
-         for (int i=0; i<numBlocks; ++i)
-         {
-            DiagCoef(i,i) = Acoef(i,i);
-            for (int j=0; j<numBlocks; ++j)
-            {
-               BlkAcDiag(i,j) = NULL;
-            }
-
-            BlkAcDiag(i,i) = BlkAc(i,i);
-         }
-
-         AcDiag = CreateHypreParMatrixFromBlocks2(comm, Coffsets, BlkAcDiag, Asp,
-                                                  DiagCoef, blockProcOffsets, all_block_num_loc_rows);
-
-         AcDiag->GetDiag(AcDiagSp);
-         //delete AcDiag;
-      }
-#endif
-
-#ifdef MFEM_USE_STRUMPACK
-      invAc = CreateStrumpackSolver(new STRUMPACKRowLocMatrix(*Ac), comm);
-      delete Ac;
-#else
-      Ac->GetDiag(AcSp);  // AcSp does not own the data
-      delete Ac;
-
-#ifdef ITERATIVE_COARSE_SOLVE
-      //CGSolver *cg_solver = new CGSolver(comm);
-      CGSolver *cg_solver = new CGSolver();
-      cg_solver->SetAbsTol(1.0e-6);
-      cg_solver->SetRelTol(1.0e-6);
-      cg_solver->SetMaxIter(1000);
-      cg_solver->SetOperator(AcSp);
-      cg_solver->SetPrintLevel(0);
-      cg_solver->iterative_mode = false;
-#ifdef SPARSE_JACOBI
-      AcSp.GetDiag(diagAc);
-      JacobiAc = new OperatorJacobiSmoother(diagAc, emptyEssDof);
-      cg_solver->SetPreconditioner(*JacobiAc);
-#endif
-#ifdef SPARSE_ICHOLESKY
-      {
-         Vector tmpX(AcSp.Height());
-         Vector tmpY(AcSp.Height());
-         tmpX = 1.0;
-         tmpY = 0.0;
-#ifdef BLOCK_DIAG
-         AcDiagSp.Finalize();
-         AcDiagSp.SortColumnIndices();
-
-         AcDiagSp.Mult(tmpX, tmpY);
-#else
-         AcSp.Finalize();
-         AcSp.SortColumnIndices();
-
-         AcSp.Mult(tmpX, tmpY);
-#endif
-      }
-#ifdef BLOCK_DIAG
-      iCholAc = new IncompleteCholesky(AcDiagSp);
-#else
-      iCholAc = new IncompleteCholesky(AcSp);
-#endif
-      cg_solver->SetPreconditioner(*iCholAc);
-      cg_solver->SetPrintLevel(-1);
-#endif // SPARSE_ICHOLESKY
-
-#ifdef SPARSE_ILU
-      {
-         Vector tmpX(AcSp.Height());
-         Vector tmpY(AcSp.Height());
-         tmpX = 1.0;
-         tmpY = 0.0;
-#ifdef BLOCK_DIAG
-         AcDiagSp.Finalize();
-         AcDiagSp.SortColumnIndices();
-
-         AcDiagSp.Mult(tmpX, tmpY);
-#else
-         AcSp.Finalize();
-         AcSp.SortColumnIndices();
-
-         AcSp.Mult(tmpX, tmpY);
-#endif
-      }
-#ifdef BLOCK_DIAG
-      iluAc = new ILUcusparse(AcDiagSp);
-#else
-      iluAc = new ILUcusparse(AcSp);
-#endif
-      cg_solver->SetPreconditioner(*iluAc);
-      cg_solver->SetPrintLevel(0);
-#endif // SPARSE_ILU
-
-#ifdef COARSE_AMS
-#ifndef COARSE_PA
-      MFEM_VERIFY(numBlocks == 4, "");
-      // TODO: just set 2 AMS solvers for E and H.
-
-      if (coarseFespace != NULL)
-      {
-         BlockDiagonalPreconditioner *blkAMS = new BlockDiagonalPreconditioner(Coffsets);
-
-         for (int i=0; i<numBlocks; ++i)
-         {
-            HypreAMS *ams = new HypreAMS(*BlkAc(i,i), coarseFespace);
-            blkAMS->SetDiagonalBlock(i, ams);
-         }
-
-         cg_solver->SetPreconditioner(*blkAMS);
-      }
-#endif
-#endif
-
-      invAc = cg_solver;
-#else
-      UMFPackSolver *umf_solver = new UMFPackSolver();
-      umf_solver->Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver->SetOperator(AcSp);
-      invAc = umf_solver;
-#endif
-#endif
-#endif  // COARSE_PA
-
-      // Residual vectors
-      rv.resize(numGrids + 1);
-      // correction vectors
-      zv.resize(numGrids + 1);
-      // allocation
-      for (int i = 0; i <= numGrids ; i++)
-      {
-         int n = (i==0) ? invAc->Height(): BlkA[i]->Width();
-
-         rv[i].SetSize(n);
-         zv[i].SetSize(n);
-         rv[i].UseDevice(true);
-         zv[i].UseDevice(true);
-      }
-   }
-
-   virtual void SetOperator(const Operator &op) {}
-
-   virtual void SetTheta(const double a) { theta = a; }
-
-   virtual void Mult(const Vector &r, Vector &z) const
-   {
-#ifdef SWTIMING
-      StopWatch sw;
-      sw.Clear();
-      sw.Start();
-#endif
-
-      // Initial residual
-      rv[numGrids] = r;
-
-      // smooth and update residuals down to the coarsest level
-      for (int i = numGrids; i > 0 ; i--)
-      {
-         // Pre smooth
-#ifdef SWTIMING
-         StopWatch sws;
-         sws.Clear();
-         sws.Start();
-#endif
-
-         S[i - 1]->Mult(rv[i], zv[i]); zv[i] *= theta;
-
-#ifdef SWTIMING
-         sws.Stop();
-         timeMultPresmooth += sws.RealTime();
-#endif
-
-         // compute residual
-         int n = BlkA[i]->Width();
-         w.SetSize(n);
-         w.UseDevice(true);
-#ifdef SWTIMING
-         StopWatch swop;
-         swop.Clear();
-         swop.Start();
-#endif
-         BlkA[i]->Mult(zv[i], w);
-         rv[i] -= w;
-#ifdef SWTIMING
-         swop.Stop();
-         timeMultResidual += swop.RealTime();
-#endif
-         // Restrict
-#ifdef SWTIMING
-         StopWatch swr;
-         swr.Clear();
-         swr.Start();
-#endif
-         BlkP[i - 1]->MultTranspose(rv[i], rv[i - 1]);
-#ifdef SWTIMING
-         swr.Stop();
-         timeMultRestrict += swr.RealTime();
-#endif
-      }
-
-#ifdef SWTIMING
-      StopWatch swAc;
-      swAc.Clear();
-      swAc.Start();
-#endif
-
-      // Coarse grid Solve
-      invAc->Mult(rv[0], zv[0]);
-      //
-
-#ifdef SWTIMING
-      swAc.Stop();
-      timeMultAc += swAc.RealTime();
-#endif
-
-      for (int i = 1; i <= numGrids ; i++)
-      {
-         // Prolong correction
-         u.SetSize(BlkP[i - 1]->Height());
-         u.UseDevice(true);
-
-         BlkP[i - 1]->Mult(zv[i - 1], u);
-         // Update correction
-         zv[i] += u;
-         // Update residual
-         v.SetSize(BlkA[i]->Height());
-         v.UseDevice(true);
-
-         BlkA[i]->Mult(u, v); rv[i] -= v;
-         // Post smooth
-         S[i - 1]->Mult(rv[i], v); v *= theta;
-         // Update correction
-         zv[i] += v;
-      }
-      z = zv[numGrids];
-
-#ifdef SWTIMING
-      sw.Stop();
-      timeMult += sw.RealTime();
-#endif
-   }
-
-   virtual ~BlockMGPASolver()
-   {
-      for (int i = numGrids - 1; i >= 0 ; i--)
-      {
-         delete S[i];
-         delete BlkP[i];
-         delete BlkA[i];
-         for (int j=0; j<numBlocks; j++)
-         {
-            for (int k=0; k<numBlocks; k++)
-            {
-               delete A[i](j,k);
-            }
-         }
-         A[i].DeleteAll();
-      }
-      delete BlkA[numGrids];
-      delete invAc;
-      A.clear();
-
-#ifdef SPARSE_ICHOLESKY
-      delete iCholAc;
-#endif
-#ifdef SPARSE_ILU
-      delete iluAc;
-#endif
-   }
-
-#ifdef MFEM_USE_STRUMPACK
-   STRUMPACKSolver* CreateStrumpackSolver(Operator *Arow, MPI_Comm comm)
-   {
-      //STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, comm);
-      STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, comm);
-      strumpack->SetPrintFactorStatistics(true);
-      strumpack->SetPrintSolveStatistics(false);
-      strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-      strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-      strumpack->SetOperator(*Arow);
-      strumpack->SetFromCommandLine();
-      return strumpack;
-   }
-#endif
-
-   mutable double timeMult, timeMultAc, timeMultPresmooth, timeMultResidual,
-   timeMultRestrict;
-};
-
-}
-//} // namespace mfem
-
-#endif // BGMULTIGRID
@@ -34,15 +34,6 @@ if (MFEM_USE_MPI)
    )
 endif()

-if (MFEM_USE_SLEPC)
-  list(APPEND PETSC_EXAMPLES_SRCS
-    ex11p.cpp
-    )
-  list(APPEND PETSC_RC_FILES
-    rc_ex11p_lobpcg rc_ex11p_gd
-    )
-endif()
-
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})

@@ -87,22 +78,12 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
-if (MFEM_USE_SLEPC)
-  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
-  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
-  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
-endif()

 # Add the tests: one test per command-line-variable.
-set(TEST_OPTIONS_VARS
-    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-endif()
-
-foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+foreach(TEST_OPTIONS_VAR
+        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
@@ -1,440 +0,0 @@
-//                       MFEM Example 11 - Parallel Version
-//                              PETSc Modification
-//
-// Compile with: make ex11p
-//
-// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
-//
-// Description:  This example code demonstrates the use of MFEM to solve the
-//               eigenvalue problem -Delta u = lambda u with homogeneous
-//               Dirichlet boundary conditions.
-//
-//               We compute a number of the lowest eigenmodes by discretizing
-//               the Laplacian and Mass operators using a FE space of the
-//               specified order, or an isoparametric/isogeometric space if
-//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example demonstrates the use of the SLEPc eigensolver as an
-//               alternative to the LOBPCG eigenvalue solver. The shift and
-//               invert spectral transformation is used to help the convergence
-//               to the smaller eigenvalues. Alternative solver parameters can
-//               be passed in a file with "-slepcopts".
-//
-//               Reusing a single GLVis visualization window for multiple
-//               eigenfunctions is also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_SLEPC
-#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   int nev = 5;
-   int seed = 75;
-   bool slu_solver  = false;
-   bool sp_solver = false;
-   bool visualization = 1;
-   bool use_slepc = true;
-   const char *slepcrc_file = "";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-   args.AddOption(&seed, "-s", "--seed",
-                  "Random seed used to initialize LOBPCG.");
-#ifdef MFEM_USE_SUPERLU
-   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
-                  "--no-superlu", "Use the SuperLU Solver.");
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
-                  "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
-                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
-   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
-                  "SlepcOptions file to use.");
-   args.Parse();
-   if (slu_solver && sp_solver)
-   {
-      if (myid == 0)
-         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
-              << " please choose either one." << endl
-              << "         Defaulting to SuperLU." << endl;
-      sp_solver = false;
-   }
-   // The command line options are also passed to the STRUMPACK
-   // solver. So do not exit if some options are not recognized.
-   if (!sp_solver)
-   {
-      if (!args.Good())
-      {
-         if (myid == 0)
-         {
-            args.PrintUsage(cout);
-         }
-         MPI_Finalize();
-         return 1;
-      }
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
-   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
-   //    specified on the command line with -rs).
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution (1 time by
-   //    default, or specified on the command line with -rp). Once the parallel
-   //    mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << size << endl;
-   }
-
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
-   //    element space. The first corresponds to the Laplacian operator -Delta,
-   //    while the second is a simple mass matrix needed on the right hand side
-   //    of the generalized eigenvalue problem below. The boundary conditions
-   //    are implemented by elimination with special values on the diagonal to
-   //    shift the Dirichlet eigenvalues out of the computational range. After
-   //    serial and parallel assembly we extract the corresponding parallel
-   //    matrices A and M.
-   ConstantCoefficient one(1.0);
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   if (pmesh->bdr_attributes.Size() == 0)
-   {
-      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
-      // closed surface.
-      a->AddDomainIntegrator(new MassIntegrator(one));
-   }
-   a->Assemble();
-   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
-   a->Finalize();
-
-   ParBilinearForm *m = new ParBilinearForm(fespace);
-   m->AddDomainIntegrator(new MassIntegrator(one));
-   m->Assemble();
-   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
-   m->Finalize();
-
-   PetscParMatrix *pA = NULL, *pM = NULL;
-   HypreParMatrix *A = NULL, *M = NULL;
-   Operator::Type tid =
-      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
-   OperatorHandle Ah(tid), Mh(tid);
-
-   a->ParallelAssemble(Ah);
-   if (!use_slepc) { Ah.Get(A); }
-   else { Ah.Get(pA); }
-   Ah.SetOperatorOwner(false);
-
-   m->ParallelAssemble(Mh);
-   if (!use_slepc) {Mh.Get(M); }
-   else {Mh.Get(pM); }
-   Mh.SetOperatorOwner(false);
-
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   Operator * Arow = NULL;
-#ifdef MFEM_USE_SUPERLU
-   if (slu_solver)
-   {
-      Arow = new SuperLURowLocMatrix(*A);
-   }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   if (sp_solver)
-   {
-      Arow = new STRUMPACKRowLocMatrix(*A);
-   }
-#endif
-#endif
-
-   delete a;
-   delete m;
-
-   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
-   //    preconditioner for A to be used within the solver. Set the matrices
-   //    which define the generalized eigenproblem A x = lambda M x.
-   Solver * precond = NULL;
-   if (!use_slepc)
-   {
-      if (!slu_solver && !sp_solver)
-      {
-         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
-         amg->SetPrintLevel(0);
-         precond = amg;
-      }
-      else
-      {
-#ifdef MFEM_USE_SUPERLU
-         if (slu_solver)
-         {
-            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-            superlu->SetPrintStatistics(false);
-            superlu->SetSymmetricPattern(true);
-            superlu->SetColumnPermutation(superlu::PARMETIS);
-            superlu->SetOperator(*Arow);
-            precond = superlu;
-         }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-         if (sp_solver)
-         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpack->SetPrintFactorStatistics(true);
-            strumpack->SetPrintSolveStatistics(false);
-            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
-            strumpack->SetOperator(*Arow);
-            strumpack->SetFromCommandLine();
-            precond = strumpack;
-         }
-#endif
-      }
-   }
-
-   HypreLOBPCG * lobpcg = NULL;
-   SlepcEigenSolver * slepc = NULL;
-   if (!use_slepc)
-   {
-
-      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-      lobpcg->SetNumModes(nev);
-      lobpcg->SetRandomSeed(seed);
-      lobpcg->SetPreconditioner(*precond);
-      lobpcg->SetMaxIter(200);
-      lobpcg->SetTol(1e-8);
-      lobpcg->SetPrecondUsageMode(1);
-      lobpcg->SetPrintLevel(1);
-      lobpcg->SetMassMatrix(*M);
-      lobpcg->SetOperator(*A);
-   }
-   else
-   {
-      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
-      slepc->SetNumModes(nev);
-      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
-      slepc->SetTarget(0.0);
-      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
-      slepc->SetOperators(*pA,*pM);
-   }
-
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
-   Array<double> eigenvalues;
-   if (!use_slepc)
-   {
-      lobpcg->Solve();
-      lobpcg->GetEigenvalues(eigenvalues);
-   }
-   else
-   {
-      slepc->Solve();
-      eigenvalues.SetSize(nev);
-      for (int i=0; i<nev; i++)
-      {
-         slepc->GetEigenvalue(i,eigenvalues[i]);
-      }
-   }
-   Vector temp(fespace->GetTrueVSize());
-   ParGridFunction x(fespace);
-
-   // 10. Save the refined mesh and the modes in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
-   {
-      ostringstream mesh_name, mode_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      for (int i=0; i<nev; i++)
-      {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-
-         }
-
-         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
-                   << setfill('0') << setw(6) << myid;
-
-         ofstream mode_ofs(mode_name.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         mode_name.str("");
-      }
-   }
-
-   // 11. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<nev; i++)
-      {
-         if ( myid == 0 )
-         {
-            cout << "Eigenmode " << i+1 << '/' << nev
-                 << ", Lambda = " << eigenvalues[i] << endl;
-         }
-
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-         }
-
-         mode_sock << "parallel " << num_procs << " " << myid << "\n"
-                   << "solution\n" << *pmesh << x << flush
-                   << "window_title 'Eigenmode " << i+1 << '/' << nev
-                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
-
-         char c;
-         if (myid == 0)
-         {
-            cout << "press (q)uit or (c)ontinue --> " << flush;
-            cin >> c;
-         }
-         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 12. Free the used memory.
-   if (!use_slepc)
-   {
-      delete lobpcg;
-   }
-   else
-   {
-      delete slepc;
-   }
-   delete precond;
-   delete M;
-   delete A;
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   delete Arow;
-#endif
-
-   delete fespace;
-   if (order > 0)
-   {
-      delete fec;
-   }
-   delete pmesh;
-
-   // We finalize SLEPc
-   MFEMFinalizeSlepc();
-   MPI_Finalize();
-
-   return 0;
-}
@@ -23,9 +23,6 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
-ifeq ($(MFEM_USE_SLEPC),YES)
-   PAR_EXAMPLES += ex11p
-endif
 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -90,9 +87,6 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
-EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
-EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd

 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -120,12 +114,6 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
-ifeq ($(MFEM_USE_SLEPC),YES)
-ex11p-test-par: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -1,6 +0,0 @@
-# Options for the eigenvalue solver
-eps_view
-eps_converged_reason
-eps_type gd
-# Options for the spectral transform
-st_type precond
@@ -1,11 +0,0 @@
-# Options for the eigenvalue solver
-eps_monitor
-eps_converged_reason
-eps_view_values
-eps_type lobpcg
-eps_gen_hermitian
-eps_smallest_real
-eps_lobpcg_blocksize 5
-# Options for the spectral transform
-st_type precond
-st_pc_type gamg
@@ -1,363 +0,0 @@
-#include <iostream>
-#include <fstream>
-#include <string>
-#include <vector>
-#include <cstdlib>
-#include <cmath>
-
-using namespace std;
-
-class HypreMat
-{
-private:
-   std::vector<int> I, J;
-   std::vector<double> d;
-   int size;
-
-public:
-   HypreMat() : size(0)
-   {
-
-   }
-
-   int Size() const { return size; }
-
-   void ReadMatrixParallel(std::string filename, const int np)
-   {
-      int lastRow = 0;
-      int count = 0;
-
-      std::vector<int> R;
-
-      for (int p=0; p<np; ++p)
-      {
-         std::ifstream f(filename.c_str() + std::to_string(p), std::ifstream::in);
-
-         int rowFirst, rowLast, colFirst, colLast, row, col;
-         double v;
-
-         f >> rowFirst >> rowLast >> colFirst >> colLast;
-
-         I.resize(rowLast+2);
-
-         for (row=rowFirst+1; row<=rowLast+1; ++row)
-         {
-            I[row] = 0;
-         }
-
-         while (f.good())
-         {
-            f >> row >> col >> v;
-
-            if (row >= 2485)
-            {
-               abort();
-            }
-
-            //if (row < lastRow)  // verify ascending order of rows
-            //  abort();
-
-            lastRow = row;
-
-            I[row+1]++;
-            R.push_back(row);
-            J.push_back(col);
-            d.push_back(v);
-
-            count++;
-         }
-
-         f.close();
-      }
-
-      // Partial sum of I
-
-      size = I.size() - 1;
-
-      I[0] = 0;
-      for (int j=1; j<size; ++j)
-      {
-         I[j+1] += I[j];
-      }
-
-      //int *Idata = I.data();
-
-      if (I[size] != J.size() || I[size] != d.size())
-      {
-         abort();
-      }
-
-      {
-         // Reorder J and d
-         const int nnz = J.size();
-         if (nnz != d.size() || nnz != R.size())
-         {
-            abort();
-         }
-
-         std::vector<int> JJ(nnz);
-         std::vector<double> dd(nnz);
-         std::vector<int> cnt;
-
-         JJ = J;
-         dd = d;
-
-         cnt.assign(I.size(), 0); // slightly larger than necessary
-
-         for (int j=0; j<nnz; ++j)
-         {
-            J[I[R[j]] + cnt[R[j]]] = JJ[j];
-            d[I[R[j]] + cnt[R[j]]] = dd[j];
-            cnt[R[j]]++;
-         }
-      }
-   }
-
-   void CopyReorder(HypreMat const& A, std::vector<int> const& permRow,
-                    std::vector<int> const& permCol)
-   {
-      if (permRow.size() != A.Size() || permCol.size() != A.Size())
-      {
-         abort();
-      }
-
-      size = A.Size();
-      I.resize(size+1);
-
-      I[0] = 0;
-
-      for (int j=0; j<size; ++j)
-      {
-         const std::size_t nnz_j = A.I[j+1] - A.I[j];
-         I[permRow[j]+1] = nnz_j;
-      }
-
-      for (int j=1; j<size; ++j)
-      {
-         I[j+1] += I[j];
-      }
-
-      const std::size_t nnz = I[size];
-      J.resize(nnz);
-      d.resize(nnz);
-
-      std::vector<std::size_t> cnt;
-      cnt.assign(size, 0);
-
-      for (int j=0; j<size; ++j)
-      {
-         const int pj = permRow[j];
-         const std::size_t nnz_j = A.I[j+1] - A.I[j];
-
-         for (int k=0; k<nnz_j; ++k, cnt[pj]++)
-         {
-            J[I[pj] + cnt[pj]] = permCol[A.J[A.I[j] + k]];
-            d[I[pj] + cnt[pj]] = A.d[A.I[j] + k];
-         }
-      }
-   }
-
-   void Print(std::string filename)
-   {
-      std::ofstream f(filename.c_str());
-
-      for (int j=0; j<size; ++j)
-      {
-         for (int k=I[j]; k<I[j+1]; ++k)
-         {
-            f << j << " " << J[k] << " " << d[k] << endl;
-         }
-      }
-
-      f.close();
-   }
-
-   void Compare(HypreMat const& A)
-   {
-      if (size != A.Size())
-      {
-         abort();
-      }
-
-      for (int j=0; j<size; ++j)
-      {
-         for (int k=I[j]; k<I[j+1]; ++k)
-         {
-            int m = -1;
-
-            for (int l=A.I[j]; l<A.I[j+1]; ++l)
-            {
-               if (A.J[l] == J[k])
-               {
-                  m = l;
-               }
-            }
-
-            if (m < 0)
-            {
-               cout << "row " << j << " column " << J[k] << " not found" << endl;
-            }
-            //else if (fabs(A.d[m] - d[k]) > 1.0e-8)
-            else if (fabs(fabs(A.d[m]) - fabs(d[k])) > 0.0)
-            {
-               cout << "entry (" << j << ", " << J[k] << ") " << d[k] << " != " << A.d[m] <<
-                    endl;
-            }
-         }
-      }
-   }
-};
-
-void ReadCrd(std::string filename, std::vector<double>& crd)
-{
-   std::ifstream f(filename.c_str(), std::ifstream::in);
-
-   double c;
-
-   while (f.good())
-   {
-      f >> c;
-      crd.push_back(c);
-   }
-
-   // for some reason, the last value gets pushed twice
-   crd.pop_back();
-
-   f.close();
-}
-
-void ReadCrdParallel(std::string filenamebase, const int np,
-                     std::vector<double>& crd)
-{
-   for (int p=0; p<np; ++p)
-   {
-      ReadCrd(filenamebase + std::to_string(p),
-              crd);  // results get concatenated in crd
-   }
-}
-
-// crd is the DOF coordinates in serial, crdp in parallel.
-void SerialToParallelDOFMap(std::vector<double> const& crd,
-                            std::vector<double> const& crdp,
-                            std::vector<int>& sp)
-{
-   if (crdp.size() != crd.size())
-   {
-      abort();
-   }
-
-   const int dim = 3;
-   const int n = crd.size() / dim;  // number of DOF's
-
-   sp.resize(n);
-
-   const double tol = 1.0e-8;
-
-   // Simple O(n^2) search
-
-   for (int i=0; i<n; ++i)
-   {
-      sp[i] = -1;
-
-      for (int j=0; j<n; ++j)
-      {
-         bool eq = true;
-         for (int l=0; l<dim; ++l)
-         {
-            if (fabs(crd[(dim*i)+l] - crdp[(dim*j)+l]) > tol)
-            {
-               eq = false;
-            }
-         }
-
-         if (eq)
-         {
-            if (sp[i] != -1)
-            {
-               abort();
-            }
-
-            sp[i] = j;
-         }
-      }
-
-      if (sp[i] < 0)
-      {
-         abort();
-      }
-   }
-}
-
-int main()
-{
-   const int np = 2;
-
-   std::vector<double> crd0, crdp0, crd1, crdp1;
-
-   ReadCrd("dofcrd0Ser", crd0);
-   ReadCrd("dofcrd1Ser", crd1);
-
-   const int numCrd = crd0.size() / 3;
-
-   if (3*numCrd != crd0.size())
-   {
-      abort();
-   }
-
-   //for (int i=7440; i<crd.size(); ++i)
-   //cout << "crd[" << i << "] " << crd[i] << endl;
-
-   ReadCrdParallel("dofcrd0Par", np, crdp0);
-   ReadCrdParallel("dofcrd1Par", np, crdp1);
-
-   if (3*numCrd != crdp0.size())
-   {
-      abort();
-   }
-
-   std::vector<int> sp0, sp1;  // serial to parallel DOF map for a subdomain
-   SerialToParallelDOFMap(crd0, crdp0, sp0);
-   SerialToParallelDOFMap(crd1, crdp1, sp1);
-
-   /*
-   for (int i=0; i<sp.size(); ++i)
-     cout << i << " " << sp[i] << endl;
-   */
-
-   cout << "sp0" << endl;
-   for (int i=0; i<sp0.size(); ++i)
-   {
-      cout << i << " " << sp0[i] << endl;
-   }
-
-   cout << "sp1" << endl;
-   for (int i=0; i<sp1.size(); ++i)
-   {
-      cout << i << " " << sp1[i] << endl;
-   }
-
-   HypreMat Aser, Apar, AserToPar;
-
-   /*
-   Aser.ReadMatrixParallel("HypreAsdComplexIm1_Serial.0000", 1);
-   Apar.ReadMatrixParallel("HypreAsdComplexIm1_Par5.0000", np);
-   */
-
-   Aser.ReadMatrixParallel("ifopSer.0000", 1);
-   Apar.ReadMatrixParallel("ifopPar.0000", np);
-
-   //AserToPar.CopyReorder(Aser, sp, sp);
-   AserToPar.CopyReorder(Aser, sp0, sp1);
-
-   /*
-   Aser.Print("rbSer");
-   AserToPar.Print("rbSerToPar");
-   Apar.Print("rbPar");
-   */
-
-   cout << "Comparing " << endl;
-
-   AserToPar.Compare(Apar);
-
-   return 0;
-}
@@ -1,105 +0,0 @@
-#ifndef TESTSTRUMPACK_HPP
-#define TESTSTRUMPACK_HPP
-
-#include "mfem.hpp"
-
-using namespace mfem;
-using namespace std;
-
-
-void TestStrumpackConstructor()
-{
-   int num_procs, rank;
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-   const int num_loc_rows = 100;
-   const int first_loc_row = num_loc_rows * rank;
-   const int glob_nrows = num_loc_rows * num_procs;
-   const int glob_ncols = glob_nrows;
-
-   int *opI = new int[num_loc_rows+1];
-
-   for (int i=0; i<num_loc_rows+1; ++i)
-   {
-      opI[i] = 0;
-   }
-
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      int nnz_i = 3;
-
-      if ((first_loc_row + i) == 0 ||
-          (first_loc_row + i) == glob_nrows-1)  // if first or last row
-      {
-         nnz_i = 2;
-      }
-
-      opI[i+1] = opI[i] + nnz_i;
-   }
-
-   const int nnz = opI[num_loc_rows];
-
-   int *opJ = new int[nnz];
-   double *data = new double[nnz];
-
-   int cnt = 0;
-   for (int i=0; i<num_loc_rows; ++i)
-   {
-      const int globalRow = first_loc_row + i;
-
-      // Diagonal entry
-
-      opJ[cnt] = first_loc_row + i;
-
-      if (globalRow == 0 || globalRow == glob_nrows-1)
-      {
-         data[cnt] = 2.0;
-      }
-      else
-      {
-         data[cnt] = 3.0;
-      }
-
-      cnt++;
-
-      // Upper diagonal
-      if (globalRow < glob_nrows-1)
-      {
-         opJ[cnt] = first_loc_row + i + 1;
-         data[cnt] = -1.0;
-         cnt++;
-      }
-
-      // Lower diagonal
-      if (globalRow > 0)
-      {
-         opJ[cnt] = first_loc_row + i - 1;
-         data[cnt] = -1.0;
-         cnt++;
-      }
-   }
-
-   Operator *op = new STRUMPACKRowLocMatrix(MPI_COMM_WORLD, num_loc_rows,
-                                            first_loc_row, glob_nrows, glob_ncols, opI, opJ, data);
-
-   STRUMPACKSolver * strumpack = new STRUMPACKSolver(0, NULL, MPI_COMM_WORLD);
-   strumpack->SetPrintFactorStatistics(true);
-   strumpack->SetPrintSolveStatistics(false);
-   strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-   strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-   strumpack->SetOperator(*op);
-   strumpack->SetFromCommandLine();
-
-   Vector x(num_loc_rows);
-   Vector y(num_loc_rows);
-
-   x = 1.0;
-   strumpack->Mult(x, y);
-
-   delete opI;
-   delete opJ;
-   delete data;
-}
-
-#endif // TESTSTRUMPACK_HPP
@@ -31,7 +31,6 @@ set(SRCS
  bilininteg_vecmass.cpp
  coefficient.cpp
  complex_fem.cpp
-  convergence.cpp
  datacollection.cpp
  eltrans.cpp
  estimators.cpp
@@ -66,7 +65,6 @@ set(HDRS
  bilininteg.hpp
  coefficient.hpp
  complex_fem.hpp
-  convergence.hpp
  datacollection.hpp
  eltrans.hpp
  estimators.hpp
@@ -76,7 +76,7 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
   precompute_sparsity = 0;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;
 }
@@ -94,7 +94,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   precompute_sparsity = ps;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;

@@ -121,10 +121,9 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
-         ext = new FABilinearFormExtension(this);
+         // ext = new FABilinearFormExtension(this);
+         // Use the original BilinearForm implementation for now
         break;
      case AssemblyLevel::ELEMENT:
         ext = new EABilinearFormExtension(this);
@@ -144,7 +143,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 void BilinearForm::EnableStaticCondensation()
 {
   delete static_cond;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      static_cond = NULL;
      MFEM_WARNING("Static condensation not supported for this assembly level");
@@ -169,7 +168,7 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                       const Array<int> &ess_tdof_list)
 {
   delete hybridization;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      delete constr_integ;
      hybridization = NULL;
@@ -224,7 +223,7 @@ MatrixInverse * BilinearForm::Inverse() const

 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      if (!static_cond) { mat->Finalize(skip_zeros); }
      if (mat_e) { mat_e->Finalize(skip_zeros); }
@@ -627,33 +626,6 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
      MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
                  "Vector for holding diagonal has wrong size!");
      const Operator *P = fes->GetProlongationMatrix();
-      // For an AMR mesh, a convergent diagonal is assembled with |P^T| d_e,
-      // where |P^T| has the entry-wise absolute values of the conforming
-      // prolongation transpose operator.
-      if (P && !fes->Conforming())
-      {
-         Vector local_diag(P->Height());
-         ext->AssembleDiagonal(local_diag);
-         const SparseMatrix *SP = dynamic_cast<const SparseMatrix*>(P);
-#ifdef MFEM_USE_MPI
-         const HypreParMatrix *HP = dynamic_cast<const HypreParMatrix*>(P);
-#endif
-         if (SP)
-         {
-            SP->AbsMultTranspose(local_diag, diag);
-         }
-#ifdef MFEM_USE_MPI
-         else if (HP)
-         {
-            HP->AbsMultTranspose(1.0, local_diag, 0.0, diag);
-         }
-#endif
-         else
-         {
-            MFEM_ABORT("Prolongation matrix has unexpected type.");
-         }
-         return;
-      }
      if (!IsIdentityProlongation(P))
      {
         Vector local_diag(P->Height());
@@ -667,7 +639,8 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
   }
   else
   {
-      mat->GetDiag(diag);
+      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
+                 "matrix and use SparseMatrix::GetDiag?");
   }
 }

@@ -1110,7 +1083,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 0;
-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1135,7 +1108,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   bbfi_marker = mbf->bbfi_marker;
   btfbfi_marker = mbf->btfbfi_marker;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1148,8 +1121,6 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
         // ext = new FAMixedBilinearFormExtension(this);
         // Use the original BilinearForm implementation for now
@@ -1220,7 +1191,7 @@ void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
      return NULL;
@@ -1233,7 +1204,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      mat -> Finalize (skip_zeros);
   }
@@ -1510,7 +1481,7 @@ void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,

 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
      return;
@@ -1771,40 +1742,9 @@ MixedBilinearForm::~MixedBilinearForm()
   delete ext;
 }

-void DiscreteLinearOperator::SetAssemblyLevel(AssemblyLevel assembly_level)
-{
-   if (ext)
-   {
-      MFEM_ABORT("the assembly level has already been set!");
-   }
-   assembly = assembly_level;
-   switch (assembly)
-   {
-      case AssemblyLevel::FULL:
-         // Use the original BilinearForm implementation for now
-         break;
-      case AssemblyLevel::ELEMENT:
-         mfem_error("Element assembly not supported yet... stay tuned!");
-         break;
-      case AssemblyLevel::PARTIAL:
-         ext = new PADiscreteLinearOperatorExtension(this);
-         break;
-      case AssemblyLevel::NONE:
-         mfem_error("Matrix-free action not supported yet... stay tuned!");
-         break;
-      default:
-         mfem_error("Unknown assembly level");
-   }
-}

 void DiscreteLinearOperator::Assemble(int skip_zeros)
 {
-   if (ext)
-   {
-      ext->Assemble();
-      return;
-   }
-
   Array<int> dom_vdofs, ran_vdofs;
   ElementTransformation *T;
   const FiniteElement *dom_fe, *ran_fe;
@@ -29,11 +29,8 @@ namespace mfem
    form classes derived from Operator. */
 enum class AssemblyLevel
 {
-   /// Legacy fully assembled form, i.e. a global sparse matrix in MFEM, Hypre
-   /// or PETSC format. This assembly is ALWAYS performed on the host.
-   LEGACYFULL = 0,
-   /// Fully assembled form, i.e. a global sparse matrix in MFEM format. This
-   /// assembly is compatible with device execution.
+   /// Fully assembled form, i.e. a global sparse matrix in MFEM, Hypre or PETSC
+   /// format.
   FULL,
   /// Form assembled at element level, which computes and stores dense element
   /// matrices.
@@ -122,7 +119,7 @@ protected:
      static_cond = NULL; hybridization = NULL;
      precompute_sparsity = 0;
      diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::LEGACYFULL;
+      assembly = AssemblyLevel::FULL;
      batch = 1;
      ext = NULL;
   }
@@ -375,9 +372,6 @@ public:
   /// Get the output finite element space prolongation matrix
   virtual const Operator *GetOutputProlongation() const
   { return GetProlongation(); }
-   /// Get the output finite element space prolongation matrix (local diagonal)
-   virtual const Operator *GetLocalOutputProlongation() const
-   { return GetOutputProlongation(); }
   /// Get the output finite element space restriction matrix
   virtual const Operator *GetOutputRestriction() const
   { return GetRestriction(); }
@@ -594,18 +588,6 @@ public:
   /// Indicate that integrators are not owned by the BilinearForm
   void UseExternalIntegrators() { extern_bfs = 1; };

-   void GetTimings(double& t1, double& t2)
-   {
-      t1 = 0.0;
-      t2 = 0.0;
-      PABilinearFormExtension *paext = dynamic_cast<PABilinearFormExtension*>(ext);
-      if (paext != NULL)
-      {
-         t1 = paext->timingDomain;
-         t2 = paext->timingBoundary;
-      }
-   }
-
   /// Destroys bilinear form.
   virtual ~BilinearForm();
 };
@@ -999,17 +981,9 @@ public:
   /// Access all interpolators added with AddDomainInterpolator().
   Array<BilinearFormIntegrator*> *GetDI() { return &dbfi; }

-   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
-   /** This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level);
-
   /** @brief Construct the internal matrix representation of the discrete
       linear operator. */
   virtual void Assemble(int skip_zeros = 1);
-
-   /// Get the output finite element space prolongation matrix (local diagonal)
-   virtual const Operator *GetLocalOutputProlongation() const
-   { return test_fes->GetLocalProlongationMatrix(); }
 };

 }
@@ -15,9 +15,6 @@
 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
 #include "libceed/ceed.hpp"
-#include "pgridfunc.hpp"
-
-#include "../general/tic_toc.hpp"

 namespace mfem
 {
@@ -48,9 +45,6 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
   elem_restrict = NULL;
   int_face_restrict_lex = NULL;
   bdr_face_restrict_lex = NULL;
-
-   timingDomain = 0.0;
-   timingBoundary = 0.0;
 }

 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
@@ -101,13 +95,6 @@ void PABilinearFormExtension::Assemble()
      integrators[i]->AssemblePA(*a->FESpace());
   }

-   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
-   const int bdryIntegratorCount = bdryIntegrators.Size();
-   for (int i = 0; i < bdryIntegratorCount; ++i)
-   {
-      bdryIntegrators[i]->AssemblePA(*a->FESpace());
-   }
-
   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
   const int intFaceIntegratorCount = intFaceIntegrators.Size();
   for (int i = 0; i < intFaceIntegratorCount; ++i)
@@ -121,96 +108,6 @@ void PABilinearFormExtension::Assemble()
   {
      bdrFaceIntegrators[i]->AssemblePABoundaryFaces(*a->FESpace());
   }
-
-   timingDomain = 0.0;
-   timingBoundary = 0.0;
-
-   if (bdryIntegratorCount > 0)
-   {
-      FiniteElementSpace *fes = a->GetFES();
-      const int nbe = fes->GetNBE();
-      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
-      gatherMap.SetSize(bedofs);
-      indices.SetSize(bedofs);
-
-      ndofs = fes->GetNDofs();
-      offsets.SetSize(ndofs+1);
-      for (int i = 0; i <= ndofs; ++i)
-      {
-         offsets[i] = 0;
-      }
-
-      for (int i = 0; i < nbe; i++)
-      {
-         const FiniteElement &be = *fes->GetBE(i);
-         Array<int> vdofs;
-         fes -> GetBdrElementVDofs (i, vdofs);
-
-         const TensorBasisElement* el =
-            dynamic_cast<const TensorBasisElement*>(&be);
-
-         MFEM_VERIFY(el != NULL, "");
-
-         const Array<int> &fe_dof_map = el->GetDofMap();
-
-         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
-         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
-
-         for (int j=0; j<vdofs.Size(); ++j)
-         {
-            const int sidj = fe_dof_map[j];
-            const int idj = sidj >= 0 ? sidj : -1 - sidj;
-            const int dof_j = vdofs[idj];
-            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
-            offsets[d+1]++;
-         }
-      }
-
-      for (int i = 1; i <= ndofs; ++i)
-      {
-         // Partial sum
-         offsets[i] += offsets[i - 1];
-      }
-
-      int os = 0;
-      Array<int> cnt(ndofs);
-      cnt = 0;
-      indices = 0;
-
-      for (int i = 0; i < nbe; i++)
-      {
-         const FiniteElement &be = *fes->GetBE(i);
-         Array<int> vdofs;
-         fes -> GetBdrElementVDofs (i, vdofs);
-
-         const TensorBasisElement* el =
-            dynamic_cast<const TensorBasisElement*>(&be);
-
-         MFEM_VERIFY(el != NULL, "");
-
-         const Array<int> &fe_dof_map = el->GetDofMap();
-
-         MFEM_VERIFY(fe_dof_map.Size() == fes->GetBE(i)->GetDof(), "");
-         MFEM_VERIFY(vdofs.Size() == fes->GetBE(i)->GetDof(), "");
-
-         for (int j=0; j<vdofs.Size(); ++j)
-         {
-            const int sidj = fe_dof_map[j];
-            const int idj = sidj >= 0 ? sidj : -1 - sidj;
-            const int dof_j = vdofs[idj];
-            const bool plus = (sidj >= 0 && dof_j >= 0) || (sidj < 0 && dof_j < 0);
-            const int d = dof_j >= 0 ? dof_j : -1-dof_j;
-            const int lid = os + j;
-            gatherMap[lid] = plus ? d : -1-d;
-            indices[offsets[d] + cnt[d]] = plus ? lid : -1-lid;
-            cnt[d]++;
-         }
-
-         os += vdofs.Size();
-      }
-
-      MFEM_VERIFY(os == bedofs, "");
-   }
 }

 void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
@@ -218,7 +115,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
+   if (elem_restrict)
   {
      localY = 0.0;
      for (int i = 0; i < iSz; ++i)
@@ -278,18 +175,10 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
   A.Reset(oper); // A will own oper
 }

-//#define SWTIMING
-
 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

-#ifdef SWTIMING
-   StopWatch swd;
-   swd.Clear();
-   swd.Start();
-#endif
-
   const int iSz = integrators.Size();
   if (DeviceCanUseCeed() || !elem_restrict)
   {
@@ -311,93 +200,6 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
      elem_restrict->MultTranspose(localY, y);
   }

-#ifdef SWTIMING
-   swd.Stop();
-   timingDomain += swd.RealTime();
-
-   StopWatch swb;
-   swb.Clear();
-   swb.Start();
-#endif
-
-   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
-
-   const int biSz = bdryIntegrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
-   {
-      MFEM_ABORT("Not implemented");
-   }
-   else if (biSz > 0)
-   {
-      FiniteElementSpace *fes = a->GetFES();
-
-      const int nbe = fes->GetNBE();
-      const int bedofs = nbe > 0 ? nbe * fes->GetVDim() * fes->GetBE(0)->GetDof() : 0;
-      bdryX.SetSize(bedofs);
-      bdryY.SetSize(bedofs);
-      bdryX.UseDevice(true);
-      bdryY.UseDevice(true);
-      bdryX = 0.0;
-      bdryY = 0.0;
-
-      {
-         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
-         auto d_gatherMap = gatherMap.Read();
-         auto d_x = x.Read();
-         auto d_bdryX = bdryX.Write();
-
-         MFEM_FORALL(i, bedofs,
-         {
-            const int gid = d_gatherMap[i];
-            const bool plus = gid >= 0;
-            const int j = plus ? gid : -1-gid;
-
-            d_bdryX[i] = plus ? d_x[j] : -d_x[j];
-         });
-      }
-
-      for (int i = 0; i < biSz; ++i)
-      {
-         bdryIntegrators[i]->AddMultPA(bdryX, bdryY);
-      }
-      //elem_restrict->MultTranspose(bdryY, y);
-
-      // bdryY contains quantities on all boundary elements. Now add them to y.
-
-      {
-         const int xsize = x.Size();
-         MFEM_VERIFY(y.Size() == xsize, "");
-         MFEM_VERIFY(gatherMap.Size() == bedofs, "");
-         auto d_offsets = offsets.Read();
-         auto d_indices = indices.Read();
-         auto d_y = y.ReadWrite();
-         auto d_bdryY = bdryY.Read();
-
-         MFEM_FORALL(i, ndofs,
-         {
-            const int offset = d_offsets[i];
-            const int nextOffset = d_offsets[i + 1];
-
-            double val = 0.0;
-            for (int j = offset; j < nextOffset; ++j)
-            {
-               const int id = d_indices[j];
-               const bool plus = id >= 0;
-               const int idj = plus ? id : -1-id;
-               const double yval = d_bdryY[idj];
-               val += plus ? yval : -yval;
-            }
-
-            d_y[i] += val;
-         });
-      }
-   }
-
-#ifdef SWTIMING
-   swb.Stop();
-   timingBoundary += swb.RealTime();
-#endif
-
   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
   const int iFISz = intFaceIntegrators.Size();
   if (int_face_restrict_lex && iFISz>0)
@@ -455,10 +257,6 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
      }
   }

-   Array<BilinearFormIntegrator*> &bdryIntegrators = *a->GetBBFI();
-   const int biSz = bdryIntegrators.Size();
-   MFEM_VERIFY(biSz == 0, "TODO");
-
   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
   const int iFISz = intFaceIntegrators.Size();
   if (int_face_restrict_lex && iFISz>0)
@@ -494,8 +292,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const

 // Data and methods for element-assembled bilinear forms
 EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
-   : PABilinearFormExtension(form),
-     factorize_face_terms(form->FESpace()->IsDGSpace())
+   : PABilinearFormExtension(form)
 {
 }

@@ -508,21 +305,19 @@ void EABilinearFormExtension::Assemble()

   ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
   ea_data.UseDevice(true);
+   ea_data = 0.0;

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
   const int integratorCount = integrators.Size();
   for (int i = 0; i < integratorCount; ++i)
   {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);
+      integrators[i]->AssembleEA(*a->FESpace(), ea_data);
   }

   faceDofs = trialFes ->
              GetTraceElement(0, trialFes->GetMesh()->GetFaceBaseGeometry(0)) ->
              GetDof();

-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Element assembly does not support AddBoundaryIntegrator yet.");
-
   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
   const int intFaceIntegratorCount = intFaceIntegrators.Size();
   if (intFaceIntegratorCount>0)
@@ -530,13 +325,14 @@ void EABilinearFormExtension::Assemble()
      nf_int = trialFes->GetNFbyType(FaceType::Interior);
      ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
      ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
+      ea_data_int = 0.0;
+      ea_data_ext = 0.0;
   }
   for (int i = 0; i < intFaceIntegratorCount; ++i)
   {
      intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
                                                     ea_data_int,
-                                                     ea_data_ext,
-                                                     i);
+                                                     ea_data_ext);
   }

   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
@@ -549,18 +345,7 @@ void EABilinearFormExtension::Assemble()
   }
   for (int i = 0; i < boundFaceIntegratorCount; ++i)
   {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr,i);
-   }
-
-   if (factorize_face_terms && int_face_restrict_lex)
-   {
-      auto restFint = dynamic_cast<const L2FaceRestriction&>(*int_face_restrict_lex);
-      restFint.AddFaceMatricesToElementMatrices(ea_data_int, ea_data);
-   }
-   if (factorize_face_terms && bdr_face_restrict_lex)
-   {
-      auto restFbdr = dynamic_cast<const L2FaceRestriction&>(*bdr_face_restrict_lex);
-      restFbdr.AddFaceMatricesToElementMatrices(ea_data_bdr, ea_data);
+      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr);
   }
 }

@@ -614,27 +399,24 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         const int NDOFS = faceDofs;
         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
+         auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
+         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            const int f = glob_j/NDOFS;
+            const int j = glob_j%NDOFS;
+            double res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
+               res += A_int(i, j, 0, f)*X(i, 0, f);
+            }
+            Y(j, 0, f) += res;
+            res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
+            {
+               res += A_int(i, j, 1, f)*X(i, 1, f);
+            }
+            Y(j, 1, f) += res;
+         });
         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
@@ -661,7 +443,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   // Treatment of boundary faces
   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
+   if (bdr_face_restrict_lex && bFISz>0)
   {
      // Apply the Boundary Face Restriction
      bdr_face_restrict_lex->Mult(x, faceBdrX);
@@ -740,27 +522,24 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         const int NDOFS = faceDofs;
         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
+         auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
+         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
+            const int f = glob_j/NDOFS;
+            const int j = glob_j%NDOFS;
+            double res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
+               res += A_int(j, i, 0, f)*X(i, 0, f);
+            }
+            Y(j, 0, f) += res;
+            res = 0.0;
+            for (int i = 0; i < NDOFS; i++)
+            {
+               res += A_int(j, i, 1, f)*X(i, 1, f);
+            }
+            Y(j, 1, f) += res;
+         });
         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
         MFEM_FORALL(glob_j, nf_int*NDOFS,
         {
@@ -787,7 +566,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   // Treatment of boundary faces
   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
+   if (bdr_face_restrict_lex && bFISz>0)
   {
      // Apply the Boundary Face Restriction
      bdr_face_restrict_lex->Mult(x, faceBdrX);
@@ -816,139 +595,6 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   }
 }

-// Data and methods for fully-assembled bilinear forms
-FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
-   : EABilinearFormExtension(form),
-     mat(form->FESpace()->GetVSize(),form->FESpace()->GetVSize(),0),
-     face_mat(form->FESpace()->GetVSize(),0,0),
-     use_face_mat(false)
-{
-#ifdef MFEM_USE_MPI
-   if ( ParFiniteElementSpace* pfes =
-           dynamic_cast<ParFiniteElementSpace*>(form->FESpace()) )
-   {
-      if (pfes->IsDGSpace())
-      {
-         use_face_mat = true;
-         pfes->ExchangeFaceNbrData();
-         face_mat.SetWidth(pfes->GetFaceNbrVSize());
-      }
-   }
-#endif
-}
-
-void FABilinearFormExtension::Assemble()
-{
-   EABilinearFormExtension::Assemble();
-   FiniteElementSpace &fes = *a->FESpace();
-   if (fes.IsDGSpace())
-   {
-      const L2ElementRestriction *restE =
-         static_cast<const L2ElementRestriction*>(elem_restrict);
-      const L2FaceRestriction *restF =
-         static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
-      // 1. Fill I
-      //  1.1 Increment with restE
-      restE->FillI(mat);
-      //  1.2 Increment with restF
-      if (restF) { restF->FillI(mat, face_mat); }
-      //  1.3 Sum the non-zeros in I
-      auto h_I = mat.HostReadWriteI();
-      int cpt = 0;
-      const int vd = fes.GetVDim();
-      const int ndofs = ne*elemDofs*vd;
-      for (int i = 0; i < ndofs; i++)
-      {
-         const int nnz = h_I[i];
-         h_I[i] = cpt;
-         cpt += nnz;
-      }
-      const int nnz = cpt;
-      h_I[ndofs] = nnz;
-      mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
-      mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
-      if (use_face_mat && restF)
-      {
-         auto h_I_face = face_mat.HostReadWriteI();
-         int cpt = 0;
-         for (int i = 0; i < ndofs; i++)
-         {
-            const int nnz = h_I_face[i];
-            h_I_face[i] = cpt;
-            cpt += nnz;
-         }
-         const int nnz_face = cpt;
-         h_I_face[ndofs] = nnz_face;
-         face_mat.GetMemoryJ().New(nnz_face,
-                                   face_mat.GetMemoryJ().GetMemoryType());
-         face_mat.GetMemoryData().New(nnz_face,
-                                      face_mat.GetMemoryData().GetMemoryType());
-      }
-      // 2. Fill J and Data
-      // 2.1 Fill J and Data with Elem ea_data
-      restE->FillJAndData(ea_data, mat);
-      // 2.2 Fill J and Data with Face ea_data_ext
-      if (restF) { restF->FillJAndData(ea_data_ext, mat, face_mat); }
-      // 2.3 Shift indirections in I back to original
-      auto I = mat.HostReadWriteI();
-      for (int i = ndofs; i > 0; i--)
-      {
-         I[i] = I[i-1];
-      }
-      I[0] = 0;
-      if (use_face_mat && restF)
-      {
-         auto I_face = face_mat.HostReadWriteI();
-         for (int i = ndofs; i > 0; i--)
-         {
-            I_face[i] = I_face[i-1];
-         }
-         I_face[0] = 0;
-      }
-   }
-   else // continuous Galerkin case
-   {
-      const ElementRestriction &rest =
-         static_cast<const ElementRestriction&>(*elem_restrict);
-      rest.FillSparseMatrix(ea_data, mat);
-   }
-}
-
-void FABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   mat.Mult(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMult(shared_x, y); }
-   }
-#endif
-}
-
-void FABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   mat.MultTranspose(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMultTranspose(shared_x, y); }
-   }
-#endif
-}
-
-
 MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
   : Operator(form->Height(), form->Width()), a(form)
 {
@@ -996,12 +642,6 @@ void PAMixedBilinearFormExtension::Assemble()
   {
      integrators[i]->AssemblePA(*trialFes, *testFes);
   }
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
-   MFEM_VERIFY(a->GetTFBFI()->Size() == 0,
-               "Partial assembly does not support AddTraceFaceIntegrator yet.");
-   MFEM_VERIFY(a->GetBTFBFI()->Size() == 0,
-               "Partial assembly does not support AddBdrTraceFaceIntegrator yet.");
 }

 void PAMixedBilinearFormExtension::Update()
@@ -1019,6 +659,7 @@ void PAMixedBilinearFormExtension::Update()
      localTrial.UseDevice(true);
      localTrial.SetSize(elem_restrict_trial->Height(),
                         Device::GetMemoryType());
+
   }
   if (elem_restrict_test)
   {
@@ -1218,133 +859,4 @@ void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
   }
 }

-PADiscreteLinearOperatorExtension::PADiscreteLinearOperatorExtension(
-   DiscreteLinearOperator *linop) :
-   PAMixedBilinearFormExtension(linop)
-{
-}
-
-const Operator *PADiscreteLinearOperatorExtension::GetLocalOutputProlongation()
-const
-{
-   return a->GetLocalOutputProlongation();
-}
-
-void PADiscreteLinearOperatorExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssemblePA(*trialFes, *testFes);
-   }
-
-   test_multiplicity.UseDevice(true);
-   test_multiplicity.SetSize(elem_restrict_test->Width()); // l-vector
-   Vector ones(elem_restrict_test->Height()); // e-vector
-   ones = 1.0;
-
-   const ElementRestriction* elem_restrict =
-      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
-   if (elem_restrict)
-   {
-      elem_restrict->MultTransposeUnsigned(ones, test_multiplicity);
-   }
-   else
-   {
-      mfem_error("A real ElementRestriction is required in this setting!");
-   }
-
-   auto tm = test_multiplicity.ReadWrite();
-   MFEM_FORALL(i, test_multiplicity.Size(),
-   {
-      tm[i] = 1.0 / tm[i];
-   });
-}
-
-void PADiscreteLinearOperatorExtension::AddMult(
-   const Vector &x, Vector &y, const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // * G operation
-   SetupMultInputs(elem_restrict_trial, x, localTrial,
-                   elem_restrict_test, y, localTest, c);
-
-   // * B^TDB operation
-   for (int i = 0; i < iSz; ++i)
-   {
-      integrators[i]->AddMultPA(localTrial, localTest);
-   }
-
-   // do a kind of "set" rather than "add" in the below
-   // operation as compared to the BilinearForm case
-   // * G^T operation (kind of...)
-   const ElementRestriction* elem_restrict =
-      dynamic_cast<const ElementRestriction*>(elem_restrict_test);
-   if (elem_restrict)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict->MultLeftInverse(localTest, tempY);
-      y += tempY;
-   }
-   else
-   {
-      mfem_error("In this setting you need a real ElementRestriction!");
-   }
-}
-
-void PADiscreteLinearOperatorExtension::AddMultTranspose(
-   const Vector &x, Vector &y, const double c) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-
-   // do a kind of "set" rather than "add" in the below
-   // operation as compared to the BilinearForm case
-   // * G operation (kinda)
-   Vector xscaled(x);
-   MFEM_VERIFY(x.Size() == test_multiplicity.Size(), "Input vector of wrong size");
-   auto xs = xscaled.ReadWrite();
-   auto tm = test_multiplicity.Read();
-   MFEM_FORALL(i, x.Size(),
-   {
-      xs[i] *= tm[i];
-   });
-   SetupMultInputs(elem_restrict_test, xscaled, localTest,
-                   elem_restrict_trial, y, localTrial, c);
-
-   // * B^TD^TB operation
-   for (int i = 0; i < iSz; ++i)
-   {
-      integrators[i]->AddMultTransposePA(localTest, localTrial);
-   }
-
-   // * G^T operation
-   if (elem_restrict_trial)
-   {
-      tempY.SetSize(y.Size());
-      elem_restrict_trial->MultTranspose(localTrial, tempY);
-      y += tempY;
-   }
-   else
-   {
-      mfem_error("Trial ElementRestriction not defined");
-   }
-}
-
-void PADiscreteLinearOperatorExtension::FormRectangularSystemOperator(
-   const Array<int>& ess1, const Array<int>& ess2, OperatorHandle &A)
-{
-   const Operator *Pi = this->GetProlongation();
-   const Operator *Po = this->GetLocalOutputProlongation();
-   Operator *rap = SetupRAP(Pi, Po);
-
-   RectangularConstrainedOperator *Arco
-      = new RectangularConstrainedOperator(rap, ess1, ess2, rap != this);
-
-   A.Reset(Arco);
-}
-
 } // namespace mfem
@@ -21,7 +21,6 @@ namespace mfem

 class BilinearForm;
 class MixedBilinearForm;
-class DiscreteLinearOperator;

 /// Class extending the BilinearForm class to support different AssemblyLevels.
 /**  FA - Full Assembly
@@ -63,22 +62,38 @@ public:
   virtual void Update() = 0;
 };

+/** @brief Data and methods for fully-assembled bilinear forms.
+    Not yet implemented!  Use the BilinearForm Class instead. */
+class FABilinearFormExtension : public BilinearFormExtension
+{
+public:
+   FABilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~FABilinearFormExtension() {}
+};
+
 /// Data and methods for partially-assembled bilinear forms
 class PABilinearFormExtension : public BilinearFormExtension
 {
 protected:
   const FiniteElementSpace *trialFes, *testFes; // Not owned
   mutable Vector localX, localY;
-   mutable Vector bdryX, bdryY;
   mutable Vector faceIntX, faceIntY;
   mutable Vector faceBdrX, faceBdrY;
   const Operator *elem_restrict; // Not owned
   const Operator *int_face_restrict_lex; // Not owned
   const Operator *bdr_face_restrict_lex; // Not owned
-   Array<int> gatherMap;
-   Array<int> indices;
-   Array<int> offsets;
-   int ndofs;

 public:
   PABilinearFormExtension(BilinearForm*);
@@ -94,8 +109,6 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
   void Update();

-   mutable double timingDomain, timingBoundary;
-
 protected:
   void SetupRestrictionOperators(const L2FaceValues m);
 };
@@ -106,12 +119,10 @@ class EABilinearFormExtension : public PABilinearFormExtension
 protected:
   int ne;
   int elemDofs;
-   // The element matrices are stored row major
   Vector ea_data;
   int nf_int, nf_bdr;
   int faceDofs;
   Vector ea_data_int, ea_data_ext, ea_data_bdr;
-   bool factorize_face_terms;

 public:
   EABilinearFormExtension(BilinearForm *form);
@@ -121,23 +132,6 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
 };

-/// Data and methods for fully-assembled bilinear forms
-class FABilinearFormExtension : public EABilinearFormExtension
-{
-private:
-   SparseMatrix mat;
-   /// face_mat handles parallelism for DG face terms.
-   SparseMatrix face_mat;
-   bool use_face_mat;
-
-public:
-   FABilinearFormExtension(BilinearForm *form);
-
-   void Assemble();
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
 /// Data and methods for matrix-free bilinear forms NOT YET IMPLEMENTED.
 class MFBilinearFormExtension : public BilinearFormExtension
 {
@@ -213,7 +207,7 @@ protected:
   mutable Vector localTrial, localTest, tempY;
   const Operator *elem_restrict_trial; // Not owned
   const Operator *elem_restrict_test;  // Not owned
-
+private:
   /// Helper function to set up inputs/outputs for Mult or MultTranspose
   void SetupMultInputs(const Operator *elem_restrict_x,
                        const Vector &x, Vector &localX,
@@ -259,33 +253,6 @@ public:
   void Update();
 };

-
-/**
-   This acts very much like PAMixedBilinearFormExtension, but its
-   FormRectangularSystemOperator implementation emulates 'Set' rather than
-   'Add' in the assembly case.
-*/
-class PADiscreteLinearOperatorExtension : public PAMixedBilinearFormExtension
-{
-public:
-   PADiscreteLinearOperatorExtension(DiscreteLinearOperator *linop);
-
-   /// Partial assembly of all internal integrators
-   void Assemble();
-
-   void AddMult(const Vector &x, Vector &y, const double c) const;
-
-   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
-
-   void FormRectangularSystemOperator(const Array<int>&, const Array<int>&,
-                                      OperatorHandle& A);
-
-   const Operator * GetLocalOutputProlongation() const;
-
-private:
-   Vector test_multiplicity;
-};
-
 }

 #endif
@@ -22,14 +22,14 @@ namespace mfem

 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
               "   is not implemented for this class.");
 }

 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                        const FiniteElementSpace&)
 {
-   mfem_error ("BilinearFormIntegrator::AssemblePA(fes, fes)\n"
+   mfem_error ("BilinearFormIntegrator::AssemblePA(...)\n"
               "   is not implemented for this class.");
 }

@@ -52,8 +52,7 @@ void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 }

 void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                        Vector &emat,
-                                        const bool add)
+                                        Vector &emat)
 {
   mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
               "   is not implemented for this class.");
@@ -62,8 +61,7 @@ void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
 void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
                                                     &fes,
                                                     Vector &ea_data_int,
-                                                     Vector &ea_data_ext,
-                                                     const bool add)
+                                                     Vector &ea_data_ext)
 {
   mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
               "   is not implemented for this class.");
@@ -71,8 +69,7 @@ void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace

 void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
                                                     &fes,
-                                                     Vector &ea_data_bdr,
-                                                     const bool add)
+                                                     Vector &ea_data_bdr)
 {
   mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
               "   is not implemented for this class.");
@@ -92,7 +89,7 @@ void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const

 void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
 {
-   mfem_error ("BilinearFormIntegrator::AddMultTransposePA(...)\n"
+   mfem_error ("BilinearFormIntegrator::MultAssembledTranspose(...)\n"
               "   is not implemented for this class.");
 }

@@ -900,44 +897,6 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
   return IntRules.Get(trial_fe.GetGeomType(), order);
 }

-/// alpha (n x u, v)
-void VectorFEBoundaryTangentIntegrator::AssembleElementMatrix
-(const FiniteElement &el, ElementTransformation &Trans,
- DenseMatrix &elmat)
-{
-   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, Trans);
-
-   const int nd1 = el.GetDof();
-
-   DenseMatrix vshape(nd1, 3);
-   DenseMatrix vshapeRotated(nd1, 3);
-
-   elmat.SetSize(nd1);
-
-   elmat = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint (&ip);
-
-      Vector n(3);
-      CalcOrtho(Trans.Jacobian(), n);
-
-      el.CalcVShape(Trans, vshape);
-
-      for (int j=0; j<nd1; ++j)
-      {
-         // Set vshapeRotated(j) = n x vshape
-         vshapeRotated(j, 0) = (n[1] * vshape(j, 2)) - (n[2] * vshape(j, 1));
-         vshapeRotated(j, 1) = (n[2] * vshape(j, 0)) - (n[0] * vshape(j, 2));
-         vshapeRotated(j, 2) = (n[0] * vshape(j, 1)) - (n[1] * vshape(j, 0));
-      }
-
-      const double w = alpha * ip.weight; // Trans.Weight() is included in n
-
-      AddMult_a_ABt(w, vshape, vshapeRotated, elmat);
-   }
-}

 void BoundaryMassIntegrator::AssembleFaceMatrix(
   const FiniteElement &el1, const FiniteElement &el2,
@@ -967,14 +926,11 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      // Set the integration point in the face and the neighboring element
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
+      IntegrationPoint eip;
+      Trans.Loc1.Transform(ip, eip);
      el1.CalcShape(eip, shape);

+      Trans.SetIntPoint(&ip);
      w = Trans.Weight() * ip.weight;
      if (Q)
      {
@@ -1563,7 +1519,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
   double w;

 #ifdef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
   curlshape.SetSize(nd,dimc);
@@ -1571,7 +1526,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
   elmat.SetSize(nd);
   if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }

   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
@@ -1615,12 +1569,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
         Mult(curlshape_dFt, M, curlshape);
         AddMultABt(curlshape, curlshape_dFt, elmat);
      }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(curlshape_dFt, D, elmat);
-      }
      else if (Q)
      {
         w *= Q->Eval(Trans, ip);
@@ -2623,17 +2571,16 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
+      Trans.Loc1.Transform(ip, eip1);
+      if (ndof2)
+      {
+         Trans.Loc2.Transform(ip, eip2);
+      }
      el1.CalcShape(eip1, shape1);

+      Trans.SetIntPoint(&ip);
+
      u->Eval(vu, *Trans.Elem1, eip1);

      if (dim == 1)
@@ -2780,15 +2727,10 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip1, eip2;

-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.SetIntPoint(&ip);
      if (dim == 1)
      {
         nor(0) = 2*eip1.x - 1.0;
@@ -2845,6 +2787,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(

      if (ndof2)
      {
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
         w = ip.weight/2/Trans.Elem2->Weight();
@@ -3062,14 +3005,9 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
   for (int pind = 0; pind < ir->GetNPoints(); ++pind)
   {
      const IntegrationPoint &ip = ir->IntPoint(pind);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+      IntegrationPoint eip1, eip2; // integration point in the reference space
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.SetIntPoint(&ip);

      el1.CalcShape(eip1, shape1);
      el1.CalcDShape(eip1, dshape1);
@@ -3089,6 +3027,7 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
      double w, wLM;
      if (ndofs2)
      {
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
         CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3226,22 +3165,17 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
      // Trace finite element shape function
+      Trans.SetIntPoint(&ip);
      trial_face_fe.CalcShape(ip, face_shape);
      // Side 1 finite element shape function
+      Trans.Loc1.Transform(ip, eip1);
      test_fe1.CalcShape(eip1, shape1);
      if (ndof2)
      {
         // Side 2 finite element shape function
+         Trans.Loc2.Transform(ip, eip2);
         test_fe2.CalcShape(eip2, shape2);
      }
      w = ip.weight;
@@ -17,18 +17,9 @@
 #include "fespace.hpp"
 #include "libceed/ceed.hpp"

-//#define SETUPONHOST
-
 namespace mfem
 {

-// Local maximum size of dofs and quads in 1D
-constexpr int HCURL_MAX_D1D = 5;
-constexpr int HCURL_MAX_Q1D = 6;
-
-constexpr int HDIV_MAX_D1D = 5;
-constexpr int HDIV_MAX_Q1D = 6;
-
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -88,10 +79,9 @@ public:
   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;

   /// Method defining element assembly.
-   /** The result of the element assembly is added to the @a emat Vector if
-       @a add is true. Otherwise, if @a add is false, we set @a emat. */
-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add = true);
+   /** The result of the element assembly is added and stored in the @a emat
+       Vector. */
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);
   /** Used with BilinearFormIntegrators that have different spaces. */
   // virtual void AssembleEA(const FiniteElementSpace &trial_fes,
   //                         const FiniteElementSpace &test_fes,
@@ -99,12 +89,10 @@ public:

   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add = true);
+                                        Vector &ea_data_ext);

   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add = true);
+                                        Vector &ea_data_bdr);

   /// Given a particular Finite Element computes the element matrix elmat.
   virtual void AssembleElementMatrix(const FiniteElement &el,
@@ -209,9 +197,6 @@ public:
   { return 0.0; }

   virtual ~BilinearFormIntegrator() { }
-
-   bool isBdryInteg = false;
-   Array<int> *el_marker = NULL;
 };

 /** Wraps a given @a BilinearFormIntegrator and transposes the resulting element
@@ -270,17 +255,14 @@ public:
      bfi->AddMultTransposePA(x, y);
   }

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AssembleEAInteriorFaces(const FiniteElementSpace &fes,
                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add);
+                                        Vector &ea_data_ext);

   virtual void AssembleEABoundaryFaces(const FiniteElementSpace &fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add);
+                                        Vector &ea_data_bdr);

   virtual ~TransposeIntegrator() { if (own_bfi) { delete bfi; } }
 };
@@ -1703,22 +1685,6 @@ protected:
   {
      trial_fe.CalcPhysCurlShape(Trans, shape);
   }
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-private:
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, dofs1Dtest,quad1D, testType, trialType, coeffDim;
 };

 /** Class for integrating the bilinear form a(u,v) := (Q u, curl v) in 3D and
@@ -1758,20 +1724,6 @@ protected:
   {
      test_fe.CalcPhysCurlShape(Trans, shape);
   }
-
-   using BilinearFormIntegrator::AssemblePA;
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector&, Vector&) const;
-
-private:
-   // PA extension
-   Vector pa_data;
-   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, dofs1D, quad1D, testType, trialType, coeffDim;
 };

 /** Class for integrating the bilinear form a(u,v) := - (Q u, grad v) in either
@@ -1814,10 +1766,8 @@ protected:
 };

 /** Class for integrating the bilinear form a(u,v) := (Q grad u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the same (H1) space
-    as u.
-
-    See also MixedVectorGradientIntegrator when v is in H(curl). */
+    scalar coefficient, and v is a vector with components v_i in the same space
+    as u. */
 class GradientIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -1965,8 +1915,7 @@ public:

   virtual void AssemblePA(const FiniteElementSpace &fes);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AssembleDiagonalPA(Vector &diag);

@@ -1975,7 +1924,7 @@ public:
   static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                         const FiniteElement &test_fe);

-   void SetupPA(const FiniteElementSpace &fes);
+   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2041,8 +1990,7 @@ public:

   virtual void AssemblePA(const FiniteElementSpace &fes);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AssembleDiagonalPA(Vector &diag);

@@ -2052,7 +2000,7 @@ public:
                                         const FiniteElement &test_fe,
                                         ElementTransformation &Trans);

-   void SetupPA(const FiniteElementSpace &fes);
+   void SetupPA(const FiniteElementSpace &fes, const bool force = false);
 };

 /** Mass integrator (u, v) restricted to the boundary of a domain */
@@ -2069,20 +2017,6 @@ public:
                                   DenseMatrix &elmat);
 };

-/// alpha (n x u, v)
-class VectorFEBoundaryTangentIntegrator : public MassIntegrator
-{
-private:
-   double alpha;
-
-public:
-   VectorFEBoundaryTangentIntegrator(double a = 1.0) : alpha(a)
-   { }
-
-   void AssembleElementMatrix(const FiniteElement &el,
-                              ElementTransformation &Trans, DenseMatrix &elmat);
-};
-
 /// alpha (q . grad u, v)
 class ConvectionIntegrator : public BilinearFormIntegrator
 {
@@ -2112,8 +2046,7 @@ public:

   virtual void AssemblePA(const FiniteElementSpace&);

-   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat,
-                           const bool add);
+   virtual void AssembleEA(const FiniteElementSpace &fes, Vector &emat);

   virtual void AddMultPA(const Vector&, Vector&) const;

@@ -2330,14 +2263,12 @@ class CurlCurlIntegrator: public BilinearFormIntegrator
 private:
   Vector vec, pointflux;
 #ifndef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape, curlshape_dFt, M;
   DenseMatrix vshape, projcurl;
 #endif

 protected:
   Coefficient *Q;
-   VectorCoefficient *DQ;
   MatrixCoefficient *MQ;

   // PA extension
@@ -2346,17 +2277,12 @@ protected:
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
   int dim, ne, nq, dofs1D, quad1D;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; MQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { DQ = NULL; MQ = NULL; }
-   CurlCurlIntegrator(VectorCoefficient &dq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), DQ(&dq) { Q = NULL; MQ = NULL; }
-   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), MQ(&mq) { Q = NULL; DQ = NULL; }
+   CurlCurlIntegrator(Coefficient &q) : Q(&q) { MQ = NULL; }
+   CurlCurlIntegrator(MatrixCoefficient &m) : MQ(&m) { Q = NULL; }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2434,11 +2360,8 @@ protected:
   Vector pa_data;
   const DofToQuad *mapsO;         ///< Not owned. DOF-to-quad map, open.
   const DofToQuad *mapsC;         ///< Not owned. DOF-to-quad map, closed.
-   const DofToQuad *mapsOtest;     ///< Not owned. DOF-to-quad map, open.
-   const DofToQuad *mapsCtest;     ///< Not owned. DOF-to-quad map, closed.
   const GeometricFactors *geom;   ///< Not owned
-   int dim, ne, nq, dofs1D, dofs1Dtest, quad1D, trial_fetype, test_fetype;
-   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
+   int dim, ne, nq, dofs1D, quad1D, fetype;

 public:
   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
@@ -2459,8 +2382,6 @@ public:

   using BilinearFormIntegrator::AssemblePA;
   virtual void AssemblePA(const FiniteElementSpace &fes);
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
 };
@@ -2690,12 +2611,10 @@ public:

   virtual void AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                        Vector &ea_data_int,
-                                        Vector &ea_data_ext,
-                                        const bool add);
+                                        Vector &ea_data_ext);

   virtual void AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                        Vector &ea_data_bdr,
-                                        const bool add);
+                                        Vector &ea_data_bdr);

   static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                         FaceElementTransformations &T);
@@ -2899,33 +2818,11 @@ class DiscreteInterpolator : public BilinearFormIntegrator { };
 class GradientInterpolator : public DiscreteInterpolator
 {
 public:
-   GradientInterpolator() : fake_fe(NULL) { }
-   virtual ~GradientInterpolator() { delete fake_fe; }
-
   virtual void AssembleElementMatrix2(const FiniteElement &h1_fe,
                                       const FiniteElement &nd_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat)
   { nd_fe.ProjectGrad(h1_fe, Trans, elmat); }
-
-   using BilinearFormIntegrator::AssemblePA;
-   /**
-      trial_fes should be H1 Lagrange
-      test_fes should be Nedelec
-   */
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-private:
-   /// 1D finit element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * fake_fe;
-
-   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
-   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
-   int dim, ne, o_dofs1D, c_dofs1D;
 };


@@ -2940,24 +2837,6 @@ public:
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat)
   { ran_fe.Project(dom_fe, Trans, elmat); }
-
-   using BilinearFormIntegrator::AssemblePA;
-
-   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
-                           const FiniteElementSpace &test_fes);
-
-   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
-
-private:
-   /// 1D finit element that generates and owns the 1D DofToQuad maps below
-   FiniteElement * fake_fe;
-
-   const DofToQuad *maps_C_C; // one-d map with Lobatto rows, Lobatto columns
-   const DofToQuad *maps_O_C; // one-d map with Legendre rows, Lobatto columns
-   int dim, ne, o_dofs1D, c_dofs1D;
-
-   Vector pa_data;
 };


@@ -22,7 +22,6 @@ static void EAConvectionAssemble1D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -33,7 +32,7 @@ static void EAConvectionAssemble1D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -55,14 +54,7 @@ static void EAConvectionAssemble1D(const int NE,
            {
               val += r_Bj[k1] * D(k1, e) * r_Gi[k1];
            }
-            if (add)
-            {
-               A(i1, j1, e) += val;
-            }
-            else
-            {
-               A(i1, j1, e) = val;
-            }
+            A(i1, j1, e) = val;
         }
      }
   });
@@ -74,7 +66,6 @@ static void EAConvectionAssemble2D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -85,7 +76,7 @@ static void EAConvectionAssemble2D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -130,14 +121,7 @@ static void EAConvectionAssemble2D(const int NE,
                               * r_B[k1][j1]* r_B[k2][j2];
                     }
                  }
-                  if (add)
-                  {
-                     A(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     A(i1, i2, j1, j2, e) = val;
-                  }
+                  A(i1, i2, j1, j2, e) = val;
               }
            }
         }
@@ -151,7 +135,6 @@ static void EAConvectionAssemble3D(const int NE,
                                   const Array<double> &g,
                                   const Vector &padata,
                                   Vector &eadata,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -162,7 +145,7 @@ static void EAConvectionAssemble3D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -208,14 +191,7 @@ static void EAConvectionAssemble3D(const int NE,
                              }
                           }
                        }
-                        if (add)
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        A(i1, i2, i3, j1, j2, j3, e) = val;
                     }
                  }
               }
@@ -226,8 +202,7 @@ static void EAConvectionAssemble3D(const int NE,
 }

 void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                      Vector &ea_data,
-                                      const bool add)
+                                      Vector &ea_data)
 {
   AssemblePA(fes);
   const int ne = fes.GetMesh()->GetNE();
@@ -237,47 +212,44 @@ void ConvectionIntegrator::AssembleEA(const FiniteElementSpace &fes,
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EAConvectionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EAConvectionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EAConvectionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EAConvectionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EAConvectionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EAConvectionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EAConvectionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x22: return EAConvectionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EAConvectionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EAConvectionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EAConvectionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EAConvectionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EAConvectionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EAConvectionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EAConvectionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                     dofs1D,quad1D);
+         case 0x23: return EAConvectionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
+         case 0x34: return EAConvectionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
+         case 0x45: return EAConvectionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
+         case 0x56: return EAConvectionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
+         case 0x67: return EAConvectionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
+         case 0x78: return EAConvectionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
+         case 0x89: return EAConvectionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
+         default:   return EAConvectionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -788,34 +788,20 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   {
      vel = cQ->GetVec();
   }
-   else if (VectorQuadratureFunctionCoefficient* cQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      vel.SetSize(dim * nq * ne);
      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      DenseMatrix Q_ir;
+      Vector Vq(dim);
      for (int e = 0; e < ne; ++e)
      {
         ElementTransformation& T = *fes.GetElementTransformation(e);
-         Q->Eval(Q_ir, T, *ir);
         for (int q = 0; q < nq; ++q)
         {
+            Q->Eval(Vq, T, ir->IntPoint(q));
            for (int i = 0; i < dim; ++i)
            {
-               C(i,q,e) = Q_ir(i,q);
+               C(i,q,e) = Vq(i);
            }
         }
      }
@@ -20,8 +20,7 @@ static void EADGTraceAssemble1DInt(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
                                   Vector &eadata_int,
-                                   Vector &eadata_ext,
-                                   const bool add)
+                                   Vector &eadata_ext)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_int = Reshape(eadata_int.ReadWrite(), 2, NF);
@@ -33,41 +32,23 @@ static void EADGTraceAssemble1DInt(const int NF,
      val_ext10 = D(1, 0, f);
      val_ext01 = D(0, 1, f);
      val_int1  = D(1, 1, f);
-      if (add)
-      {
-         A_int(0, f) += val_int0;
-         A_int(1, f) += val_int1;
-         A_ext(0, f) += val_ext01;
-         A_ext(1, f) += val_ext10;
-      }
-      else
-      {
-         A_int(0, f) = val_int0;
-         A_int(1, f) = val_int1;
-         A_ext(0, f) = val_ext01;
-         A_ext(1, f) = val_ext10;
-      }
+      A_int(0, f) += val_int0;
+      A_int(1, f) += val_int1;
+      A_ext(0, f) += val_ext01;
+      A_ext(1, f) += val_ext10;
   });
 }

 static void EADGTraceAssemble1DBdr(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
-                                   Vector &eadata_bdr,
-                                   const bool add)
+                                   Vector &eadata_bdr)
 {
   auto D = Reshape(padata.Read(), 2, 2, NF);
   auto A_bdr = Reshape(eadata_bdr.ReadWrite(), NF);
   MFEM_FORALL(f, NF,
   {
-      if (add)
-      {
-         A_bdr(f) += D(0, 0, f);
-      }
-      else
-      {
-         A_bdr(f) = D(0, 0, f);
-      }
+      A_bdr(f) += D(0, 0, f);
   });
 }

@@ -77,7 +58,6 @@ static void EADGTraceAssemble2DInt(const int NF,
                                   const Vector &padata,
                                   Vector &eadata_int,
                                   Vector &eadata_ext,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -108,20 +88,10 @@ static void EADGTraceAssemble2DInt(const int NF,
               val_ext10 += B(k1,i1) * B(k1,j1) * D(k1, 1, 0, f);
               val_int1  += B(k1,i1) * B(k1,j1) * D(k1, 1, 1, f);
            }
-            if (add)
-            {
-               A_int(i1, j1, 0, f) += val_int0;
-               A_int(i1, j1, 1, f) += val_int1;
-               A_ext(i1, j1, 0, f) += val_ext01;
-               A_ext(i1, j1, 1, f) += val_ext10;
-            }
-            else
-            {
-               A_int(i1, j1, 0, f) = val_int0;
-               A_int(i1, j1, 1, f) = val_int1;
-               A_ext(i1, j1, 0, f) = val_ext01;
-               A_ext(i1, j1, 1, f) = val_ext10;
-            }
+            A_int(i1, j1, 0, f) += val_int0;
+            A_int(i1, j1, 1, f) += val_int1;
+            A_ext(i1, j1, 0, f) += val_ext01;
+            A_ext(i1, j1, 1, f) += val_ext10;
         }
      }
   });
@@ -132,7 +102,6 @@ static void EADGTraceAssemble2DBdr(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
                                   Vector &eadata_bdr,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -156,14 +125,7 @@ static void EADGTraceAssemble2DBdr(const int NF,
            {
               val_bdr  += B(k1,i1) * B(k1,j1) * D(k1, 0, 0, f);
            }
-            if (add)
-            {
-               A_bdr(i1, j1, f) += val_bdr;
-            }
-            else
-            {
-               A_bdr(i1, j1, f) = val_bdr;
-            }
+            A_bdr(i1, j1, f) += val_bdr;
         }
      }
   });
@@ -175,7 +137,6 @@ static void EADGTraceAssemble3DInt(const int NF,
                                   const Vector &padata,
                                   Vector &eadata_int,
                                   Vector &eadata_ext,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -246,20 +207,10 @@ static void EADGTraceAssemble3DInt(const int NF,
                                    * s_D[k1][k2][1][0];
                     }
                  }
-                  if (add)
-                  {
-                     A_int(i1, i2, j1, j2, 0, f) += val_int0;
-                     A_int(i1, i2, j1, j2, 1, f) += val_int1;
-                     A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
-                     A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
-                  }
-                  else
-                  {
-                     A_int(i1, i2, j1, j2, 0, f) = val_int0;
-                     A_int(i1, i2, j1, j2, 1, f) = val_int1;
-                     A_ext(i1, i2, j1, j2, 0, f) = val_ext01;
-                     A_ext(i1, i2, j1, j2, 1, f) = val_ext10;
-                  }
+                  A_int(i1, i2, j1, j2, 0, f) += val_int0;
+                  A_int(i1, i2, j1, j2, 1, f) += val_int1;
+                  A_ext(i1, i2, j1, j2, 0, f) += val_ext01;
+                  A_ext(i1, i2, j1, j2, 1, f) += val_ext10;
               }
            }
         }
@@ -272,7 +223,6 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                   const Array<double> &basis,
                                   const Vector &padata,
                                   Vector &eadata_bdr,
-                                   const bool add,
                                   const int d1d = 0,
                                   const int q1d = 0)
 {
@@ -330,14 +280,7 @@ static void EADGTraceAssemble3DBdr(const int NF,
                                   * s_D[k1][k2][0][0];
                     }
                  }
-                  if (add)
-                  {
-                     A_bdr(i1, i2, j1, j2, f) += val_bdr;
-                  }
-                  else
-                  {
-                     A_bdr(i1, i2, j1, j2, f) = val_bdr;
-                  }
+                  A_bdr(i1, i2, j1, j2, f) += val_bdr;
               }
            }
         }
@@ -347,8 +290,7 @@ static void EADGTraceAssemble3DBdr(const int NF,

 void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                Vector &ea_data_int,
-                                                Vector &ea_data_ext,
-                                                const bool add)
+                                                Vector &ea_data_ext)
 {
   SetupPA(fes, FaceType::Interior);
   nf = fes.GetNFbyType(FaceType::Interior);
@@ -356,7 +298,7 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
   const Array<double> &B = maps->B;
   if (dim == 1)
   {
-      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext,add);
+      return EADGTraceAssemble1DInt(nf,B,pa_data,ea_data_int,ea_data_ext);
   }
   else if (dim == 2)
   {
@@ -364,31 +306,31 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
      {
         case 0x22:
            return EADGTraceAssemble2DInt<2,2>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x33:
            return EADGTraceAssemble2DInt<3,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x44:
            return EADGTraceAssemble2DInt<4,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x55:
            return EADGTraceAssemble2DInt<5,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x66:
            return EADGTraceAssemble2DInt<6,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x77:
            return EADGTraceAssemble2DInt<7,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x88:
            return EADGTraceAssemble2DInt<8,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x99:
            return EADGTraceAssemble2DInt<9,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         default:
            return EADGTraceAssemble2DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,add,dofs1D,quad1D);
+                                          ea_data_ext,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
@@ -397,36 +339,35 @@ void DGTraceIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
      {
         case 0x23:
            return EADGTraceAssemble3DInt<2,3>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x34:
            return EADGTraceAssemble3DInt<3,4>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x45:
            return EADGTraceAssemble3DInt<4,5>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x56:
            return EADGTraceAssemble3DInt<5,6>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x67:
            return EADGTraceAssemble3DInt<6,7>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x78:
            return EADGTraceAssemble3DInt<7,8>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         case 0x89:
            return EADGTraceAssemble3DInt<8,9>(nf,B,pa_data,ea_data_int,
-                                               ea_data_ext,add);
+                                               ea_data_ext);
         default:
            return EADGTraceAssemble3DInt(nf,B,pa_data,ea_data_int,
-                                          ea_data_ext,add,dofs1D,quad1D);
+                                          ea_data_ext,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
 }

 void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                Vector &ea_data_bdr,
-                                                const bool add)
+                                                Vector &ea_data_bdr)
 {
   SetupPA(fes, FaceType::Boundary);
   nf = fes.GetNFbyType(FaceType::Boundary);
@@ -434,37 +375,37 @@ void DGTraceIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
   const Array<double> &B = maps->B;
   if (dim == 1)
   {
-      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr,add);
+      return EADGTraceAssemble1DBdr(nf,B,pa_data,ea_data_bdr);
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x22: return EADGTraceAssemble2DBdr<2,2>(nf,B,pa_data,ea_data_bdr);
+         case 0x33: return EADGTraceAssemble2DBdr<3,3>(nf,B,pa_data,ea_data_bdr);
+         case 0x44: return EADGTraceAssemble2DBdr<4,4>(nf,B,pa_data,ea_data_bdr);
+         case 0x55: return EADGTraceAssemble2DBdr<5,5>(nf,B,pa_data,ea_data_bdr);
+         case 0x66: return EADGTraceAssemble2DBdr<6,6>(nf,B,pa_data,ea_data_bdr);
+         case 0x77: return EADGTraceAssemble2DBdr<7,7>(nf,B,pa_data,ea_data_bdr);
+         case 0x88: return EADGTraceAssemble2DBdr<8,8>(nf,B,pa_data,ea_data_bdr);
+         case 0x99: return EADGTraceAssemble2DBdr<9,9>(nf,B,pa_data,ea_data_bdr);
         default:
-            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
+            return EADGTraceAssemble2DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr,add);
-         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr,add);
+         case 0x23: return EADGTraceAssemble3DBdr<2,3>(nf,B,pa_data,ea_data_bdr);
+         case 0x34: return EADGTraceAssemble3DBdr<3,4>(nf,B,pa_data,ea_data_bdr);
+         case 0x45: return EADGTraceAssemble3DBdr<4,5>(nf,B,pa_data,ea_data_bdr);
+         case 0x56: return EADGTraceAssemble3DBdr<5,6>(nf,B,pa_data,ea_data_bdr);
+         case 0x67: return EADGTraceAssemble3DBdr<6,7>(nf,B,pa_data,ea_data_bdr);
+         case 0x78: return EADGTraceAssemble3DBdr<7,8>(nf,B,pa_data,ea_data_bdr);
+         case 0x89: return EADGTraceAssemble3DBdr<8,9>(nf,B,pa_data,ea_data_bdr);
         default:
-            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,add,dofs1D,quad1D);
+            return EADGTraceAssemble3DBdr(nf,B,pa_data,ea_data_bdr,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -43,7 +43,7 @@ static void PADGTraceSetup2D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);

-   MFEM_FORALL(f, NF, // can be optimized with Q1D thread for NF blocks
+   MFEM_FORALL(f, NF,//can be optimized with Q1D thread for NF blocks
   {
      for (int q = 0; q < Q1D; ++q)
      {
@@ -85,7 +85,7 @@ static void PADGTraceSetup3D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);

-   MFEM_FORALL(f, NF, // can be optimized with Q1D*Q1D threads for NF blocks
+   MFEM_FORALL(f, NF,//can be optimized with Q1D*Q1D threads for NF blocks
   {
      for (int q1 = 0; q1 < Q1D; ++q1)
      {
@@ -156,26 +156,50 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
+   Vector r;
+   if (rho==nullptr)
+   {
+      r.SetSize(1);
+      r(0) = 1.0;
+   }
+   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
+   {
+      r.SetSize(1);
+      r(0) = c_rho->constant;
+   }
+   else
+   {
+      r.SetSize(nq * nf);
+      auto C = Reshape(r.HostWrite(), nq, nf);
+      int f_ind = 0;
+      for (int f = 0; f < fes.GetNF(); ++f)
+      {
+         int e1, e2;
+         int inf1, inf2;
+         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
+         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
+         int face_id = inf1 / 64;
+         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
+             (type==FaceType::Boundary && e2<0 && inf2<0) )
+         {
+            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
+            for (int q = 0; q < nq; ++q)
+            {
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face_id, quad1D, q);
+               C(iq,f_ind) = rho->Eval(T, ir->IntPoint(q));
+            }
+            f_ind++;
+         }
+      }
+      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+   }
   Vector vel;
   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
                                        (u))
   {
      vel = c_u->GetVec();
   }
-   else if (VectorQuadratureFunctionCoefficient* c_u =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
-   {
-      // Assumed to be in lexicographical ordering
-      const QuadratureFunction &qFun = c_u->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      vel.SetSize(dim * nq * nf);
@@ -192,15 +216,12 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
             (type==FaceType::Boundary && e2<0 && inf2<0) )
         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
+            ElementTransformation& T = *fes.GetMesh()->GetFaceTransformation(f);
            for (int q = 0; q < nq; ++q)
            {
               // Convert to lexicographic ordering
               int iq = ToLexOrdering(dim, face_id, quad1D, q);
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               u->Eval(Vq, *T.Elem1, eip1);
+               u->Eval(Vq, T, ir->IntPoint(q));
               for (int i = 0; i < dim; ++i)
               {
                  C(i,iq,f_ind) = Vq(i);
@@ -211,80 +232,6 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      }
      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
   }
-   Vector r;
-   if (rho==nullptr)
-   {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
-   }
-   else if (QuadratureFunctionCoefficient* c_rho =
-               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
-   {
-      const QuadratureFunction &qFun = c_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      r.SetSize(nq * nf);
-      auto C_vel = Reshape(vel.HostRead(), dim, nq, nf);
-      auto n = Reshape(geom->normal.HostRead(), nq, dim, nf);
-      auto C = Reshape(r.HostWrite(), nq, nf);
-      int f_ind = 0;
-      for (int f = 0; f < fes.GetNF(); ++f)
-      {
-         int e1, e2;
-         int inf1, inf2;
-         fes.GetMesh()->GetFaceElements(f, &e1, &e2);
-         fes.GetMesh()->GetFaceInfos(f, &inf1, &inf2);
-         int face_id = inf1 / 64;
-         if ((type==FaceType::Interior && (e2>=0 || (e2<0 && inf2>=0))) ||
-             (type==FaceType::Boundary && e2<0 && inf2<0) )
-         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face_id, quad1D, q);
-
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-               double r;
-
-               if (inf2 < 0)
-               {
-                  r = rho->Eval(*T.Elem1, eip1);
-               }
-               else
-               {
-                  double udotn = 0.0;
-                  for (int d=0; d<dim; ++d)
-                  {
-                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
-                  }
-                  if (udotn >= 0.0) { r = rho->Eval(*T.Elem2, eip2); }
-                  else { r = rho->Eval(*T.Elem1, eip1); }
-               }
-               C(iq,f_ind) = r;
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
-   }
   PADGTraceSetup(dim, dofs1D, quad1D, nf, ir->GetWeights(),
                  geom->detJ, geom->normal, r, vel,
                  alpha, beta, pa_data);
@@ -22,7 +22,6 @@ static void EADiffusionAssemble1D(const int NE,
                                  const Array<double> &g,
                                  const Vector &padata,
                                  Vector &eadata,
-                                  const bool add,
                                  const int d1d = 0,
                                  const int q1d = 0)
 {
@@ -32,7 +31,7 @@ static void EADiffusionAssemble1D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -54,14 +53,7 @@ static void EADiffusionAssemble1D(const int NE,
            {
               val += r_Gj[k1] * D(k1, e) * r_Gi[k1];
            }
-            if (add)
-            {
-               A(i1, j1, e) += val;
-            }
-            else
-            {
-               A(i1, j1, e) = val;
-            }
+            A(i1, j1, e) = val;
         }
      }
   });
@@ -73,7 +65,6 @@ static void EADiffusionAssemble2D(const int NE,
                                  const Array<double> &g,
                                  const Vector &padata,
                                  Vector &eadata,
-                                  const bool add,
                                  const int d1d = 0,
                                  const int q1d = 0)
 {
@@ -84,7 +75,7 @@ static void EADiffusionAssemble2D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, 3, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -129,14 +120,7 @@ static void EADiffusionAssemble2D(const int NE,
                               + gbi * D11 * gbj;
                     }
                  }
-                  if (add)
-                  {
-                     A(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     A(i1, i2, j1, j2, e) = val;
-                  }
+                  A(i1, i2, j1, j2, e) = val;
               }
            }
         }
@@ -146,11 +130,10 @@ static void EADiffusionAssemble2D(const int NE,

 template<int T_D1D = 0, int T_Q1D = 0>
 static void EADiffusionAssemble3D(const int NE,
-                                  const Array<double> &b,
                                  const Array<double> &g,
+                                  const Array<double> &b,
                                  const Vector &padata,
                                  Vector &eadata,
-                                  const bool add,
                                  const int d1d = 0,
                                  const int q1d = 0)
 {
@@ -161,7 +144,7 @@ static void EADiffusionAssemble3D(const int NE,
   auto B = Reshape(b.Read(), Q1D, D1D);
   auto G = Reshape(g.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -225,14 +208,7 @@ static void EADiffusionAssemble3D(const int NE,
                              }
                           }
                        }
-                        if (add)
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           A(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        A(i1, i2, i3, j1, j2, j3, e) = val;
                     }
                  }
               }
@@ -243,8 +219,7 @@ static void EADiffusionAssemble3D(const int NE,
 }

 void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data,
-                                     const bool add)
+                                     Vector &ea_data)
 {
   AssemblePA(fes);
   const int ne = fes.GetMesh()->GetNE();
@@ -254,47 +229,44 @@ void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data);
+         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data);
+         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data);
+         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data);
+         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data);
+         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data);
+         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data);
+         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data);
+         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data);
+         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data);
+         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data);
+         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data);
+         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data);
+         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data);
+         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -96,28 +96,26 @@ void PADiffusionSetup2D<2>(const int Q1D,
                           const Vector &c,
                           Vector &d)
 {
+   const int NQ = Q1D*Q1D;
   const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,2,2,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
-   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
+   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
+   auto D = Reshape(d.Write(), NQ, 3, NE);
+
+   MFEM_FORALL(e, NE,
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-            const double c_detJ = W(qx,qy) * coeff / ((J11*J22)-(J21*J12));
-            D(qx,qy,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
-            D(qx,qy,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
-            D(qx,qy,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
-         }
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double coeff = const_c ? C(0,0) : C(q,e);
+         const double c_detJ = W[q] * coeff / ((J11*J22)-(J21*J12));
+         D(q,0,e) =  c_detJ * (J12*J12 + J22*J22); // 1,1
+         D(q,1,e) = -c_detJ * (J12*J11 + J22*J21); // 1,2
+         D(q,2,e) =  c_detJ * (J11*J11 + J21*J21); // 2,2
      }
   });
 }
@@ -133,35 +131,33 @@ void PADiffusionSetup2D<3>(const int Q1D,
 {
   constexpr int DIM = 2;
   constexpr int SDIM = 3;
+   const int NQ = Q1D*Q1D;
   const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,SDIM,DIM,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D, 3, NE);
-   MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
+
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, SDIM, DIM, NE);
+   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
+   auto D = Reshape(d.Write(), NQ, 3, NE);
+   MFEM_FORALL(e, NE,
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      for (int q = 0; q < NQ; ++q)
      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            const double wq = W(qx,qy);
-            const double J11 = J(qx,qy,0,0,e);
-            const double J21 = J(qx,qy,1,0,e);
-            const double J31 = J(qx,qy,2,0,e);
-            const double J12 = J(qx,qy,0,1,e);
-            const double J22 = J(qx,qy,1,1,e);
-            const double J32 = J(qx,qy,2,1,e);
-            const double E = J11*J11 + J21*J21 + J31*J31;
-            const double G = J12*J12 + J22*J22 + J32*J32;
-            const double F = J11*J12 + J21*J22 + J31*J32;
-            const double iw = 1.0 / sqrt(E*G - F*F);
-            const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-            const double alpha = wq * coeff * iw;
-            D(qx,qy,0,e) =  alpha * G; // 1,1
-            D(qx,qy,1,e) = -alpha * F; // 1,2
-            D(qx,qy,2,e) =  alpha * E; // 2,2
-         }
+         const double wq = W[q];
+         const double J11 = J(q,0,0,e);
+         const double J21 = J(q,1,0,e);
+         const double J31 = J(q,2,0,e);
+         const double J12 = J(q,0,1,e);
+         const double J22 = J(q,1,1,e);
+         const double J32 = J(q,2,1,e);
+         const double E = J11*J11 + J21*J21 + J31*J31;
+         const double G = J12*J12 + J22*J22 + J32*J32;
+         const double F = J11*J12 + J21*J22 + J31*J32;
+         const double iw = 1.0 / sqrt(E*G - F*F);
+         const double coeff = const_c ? C(0,0) : C(q,e);
+         const double alpha = wq * coeff * iw;
+         D(q,0,e) =  alpha * G; // 1,1
+         D(q,1,e) = -alpha * F; // 1,2
+         D(q,2,e) =  alpha * E; // 2,2
      }
   });
 }
@@ -174,54 +170,47 @@ static void PADiffusionSetup3D(const int Q1D,
                               const Vector &c,
                               Vector &d)
 {
+   const int NQ = Q1D*Q1D*Q1D;
   const bool const_c = c.Size() == 1;
-   const auto W = Reshape(w.Read(), Q1D,Q1D,Q1D);
-   const auto J = Reshape(j.Read(), Q1D,Q1D,Q1D,3,3,NE);
-   const auto C = const_c ? Reshape(c.Read(), 1,1,1,1) :
-                  Reshape(c.Read(), Q1D,Q1D,Q1D,NE);
-   auto D = Reshape(d.Write(), Q1D,Q1D,Q1D, 6, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   auto W = w.Read();
+   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
+   auto C = const_c ? Reshape(c.Read(), 1, 1) : Reshape(c.Read(), NQ, NE);
+   auto D = Reshape(d.Write(), NQ, 6, NE);
+   MFEM_FORALL(eq, NE*NQ,
   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qz,z,Q1D)
-            {
-               const double J11 = J(qx,qy,qz,0,0,e);
-               const double J21 = J(qx,qy,qz,1,0,e);
-               const double J31 = J(qx,qy,qz,2,0,e);
-               const double J12 = J(qx,qy,qz,0,1,e);
-               const double J22 = J(qx,qy,qz,1,1,e);
-               const double J32 = J(qx,qy,qz,2,1,e);
-               const double J13 = J(qx,qy,qz,0,2,e);
-               const double J23 = J(qx,qy,qz,1,2,e);
-               const double J33 = J(qx,qy,qz,2,2,e);
-               const double detJ = J11 * (J22 * J33 - J32 * J23) -
-               /* */               J21 * (J12 * J33 - J32 * J13) +
-               /* */               J31 * (J12 * J23 - J22 * J13);
-               const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-               const double c_detJ = W(qx,qy,qz) * coeff / detJ;
-               // adj(J)
-               const double A11 = (J22 * J33) - (J23 * J32);
-               const double A12 = (J32 * J13) - (J12 * J33);
-               const double A13 = (J12 * J23) - (J22 * J13);
-               const double A21 = (J31 * J23) - (J21 * J33);
-               const double A22 = (J11 * J33) - (J13 * J31);
-               const double A23 = (J21 * J13) - (J11 * J23);
-               const double A31 = (J21 * J32) - (J31 * J22);
-               const double A32 = (J31 * J12) - (J11 * J32);
-               const double A33 = (J11 * J22) - (J12 * J21);
-               // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
-               D(qx,qy,qz,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
-               D(qx,qy,qz,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
-               D(qx,qy,qz,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
-               D(qx,qy,qz,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
-               D(qx,qy,qz,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
-               D(qx,qy,qz,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
-            }
-         }
-      }
+      const int e = eq / NQ;
+      const int q = eq % NQ;
+      const double J11 = J(q,0,0,e);
+      const double J21 = J(q,1,0,e);
+      const double J31 = J(q,2,0,e);
+      const double J12 = J(q,0,1,e);
+      const double J22 = J(q,1,1,e);
+      const double J32 = J(q,2,1,e);
+      const double J13 = J(q,0,2,e);
+      const double J23 = J(q,1,2,e);
+      const double J33 = J(q,2,2,e);
+      const double detJ = J11 * (J22 * J33 - J32 * J23) -
+      /* */               J21 * (J12 * J33 - J32 * J13) +
+      /* */               J31 * (J12 * J23 - J22 * J13);
+      const double coeff = const_c ? C(0,0) : C(q,e);
+      const double c_detJ = W[q] * coeff / detJ;
+      // adj(J)
+      const double A11 = (J22 * J33) - (J23 * J32);
+      const double A12 = (J32 * J13) - (J12 * J33);
+      const double A13 = (J12 * J23) - (J22 * J13);
+      const double A21 = (J31 * J23) - (J21 * J33);
+      const double A22 = (J11 * J33) - (J13 * J31);
+      const double A23 = (J21 * J13) - (J11 * J23);
+      const double A31 = (J21 * J32) - (J31 * J22);
+      const double A32 = (J31 * J12) - (J11 * J32);
+      const double A33 = (J11 * J22) - (J12 * J21);
+      // detJ J^{-1} J^{-T} = (1/detJ) adj(J) adj(J)^T
+      D(q,0,e) = c_detJ * (A11*A11 + A12*A12 + A13*A13); // 1,1
+      D(q,1,e) = c_detJ * (A11*A21 + A12*A22 + A13*A23); // 2,1
+      D(q,2,e) = c_detJ * (A11*A31 + A12*A32 + A13*A33); // 3,1
+      D(q,3,e) = c_detJ * (A21*A21 + A22*A22 + A23*A23); // 2,2
+      D(q,4,e) = c_detJ * (A21*A31 + A22*A32 + A23*A33); // 3,2
+      D(q,5,e) = c_detJ * (A31*A31 + A32*A32 + A33*A33); // 3,3
   });
 }

@@ -263,7 +252,8 @@ static void PADiffusionSetup(const int dim,
   }
 }

-void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
+void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
+                                  const bool force)
 {
   // Assuming the same element type
   fespace = &fes;
@@ -272,7 +262,7 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
   const FiniteElement &el = *fes.GetFE(0);
   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
 #ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
+   if (DeviceCanUseCeed() && !force)
   {
      if (ceedDataPtr) { delete ceedDataPtr; }
      CeedData* ptr = new CeedData();
@@ -280,6 +270,8 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
      InitCeedCoeff(Q, ptr);
      return CeedPADiffusionAssemble(fes, *ir, *ptr);
   }
+#else
+   MFEM_CONTRACT_VAR(force);
 #endif
   const int dims = el.GetDim();
   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
@@ -303,19 +295,6 @@ void DiffusionIntegrator::SetupPA(const FiniteElementSpace &fes)
      coeff.SetSize(1);
      coeff(0) = cQ->constant;
   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
   else
   {
      coeff.SetSize(nq * ne);
@@ -756,17 +735,9 @@ static void PADiffusionAssembleDiagonal(const int dim,

 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonalPA(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
-                                  maps->B, maps->G, pa_data, diag);
-   }
+   if (pa_data.Size()==0) { SetupPA(*fespace, true); }
+   PADiffusionAssembleDiagonal(dim, dofs1D, quad1D, ne,
+                               maps->B, maps->G, pa_data, diag);
 }


@@ -1335,33 +1306,7 @@ static void PADiffusionApply3D(const int NE,
   });
 }

-// Half of B and G are stored in shared to get B, Bt, G and Gt.
-// Indices computation for SmemPADiffusionApply3D.
-static MFEM_HOST_DEVICE inline int qi(const int q, const int d, const int Q)
-{
-   return (q<=d) ? q : Q-1-q;
-}
-
-static MFEM_HOST_DEVICE inline int dj(const int q, const int d, const int D)
-{
-   return (q<=d) ? d : D-1-d;
-}
-
-static MFEM_HOST_DEVICE inline int qk(const int q, const int d, const int Q)
-{
-   return (q<=d) ? Q-1-q : q;
-}
-
-static MFEM_HOST_DEVICE inline int dl(const int q, const int d, const int D)
-{
-   return (q<=d) ? D-1-d : d;
-}
-
-static MFEM_HOST_DEVICE inline double sign(const int q, const int d)
-{
-   return (q<=d) ? -1.0 : 1.0;
-}
-
+// Shared memory PA Diffusion Apply 3D kernel
 template<int T_D1D = 0, int T_Q1D = 0>
 static void SmemPADiffusionApply3D(const int NE,
                                   const Array<double> &b_,
@@ -1374,27 +1319,28 @@ static void SmemPADiffusionApply3D(const int NE,
 {
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
-   MFEM_VERIFY(D1D <= M1D, "");
-   MFEM_VERIFY(Q1D <= M1Q, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
   auto b = Reshape(b_.Read(), Q1D, D1D);
   auto g = Reshape(g_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto d = Reshape(d_.Read(), Q1D*Q1D*Q1D, 6, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
   {
+      const int tidz = MFEM_THREAD_ID(z);
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sBG[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sBG;
-      double (*G)[MD1] = (double (*)[MD1]) sBG;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sBG;
-      double (*Gt)[MQ1] = (double (*)[MQ1]) sBG;
+      constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
+      MFEM_SHARED double sBG[2][MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) (sBG+0);
+      double (*G)[MD1] = (double (*)[MD1]) (sBG+1);
+      double (*Bt)[MQ1] = (double (*)[MQ1]) (sBG+0);
+      double (*Gt)[MQ1] = (double (*)[MQ1]) (sBG+1);
      MFEM_SHARED double sm0[3][MDQ*MDQ*MDQ];
      MFEM_SHARED double sm1[3][MDQ*MDQ*MDQ];
      double (*X)[MD1][MD1]    = (double (*)[MD1][MD1]) (sm0+2);
@@ -1412,127 +1358,108 @@ static void SmemPADiffusionApply3D(const int NE,
      double (*QDD0)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+0);
      double (*QDD1)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+1);
      double (*QDD2)[MD1][MD1] = (double (*)[MD1][MD1]) (sm0+2);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
         }
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(qx,dy,Q1D);
-            const int j = dj(qx,dy,D1D);
-            const int k = qk(qx,dy,Q1D);
-            const int l = dl(qx,dy,D1D);
-            B[i][j] = b(qx,dy);
-            G[k][l] = g(qx,dy) * sign(qx,dy);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][d] = b(q,d);
+               G[q][d] = g(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(dy,y,D1D)
         {
-            double u[D1D], v[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
+               double u = 0.0;
+               double v = 0.0;
+               for (int dx = 0; dx < D1D; ++dx)
               {
                  const double coords = X[dz][dy][dx];
-                  u[dz] += coords * B[i][j];
-                  v[dz] += coords * G[k][l] * s;
+                  u += coords * B[qx][dx];
+                  v += coords * G[qx][dx];
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ0[dz][dy][qx] = u[dz];
-               DDQ1[dz][dy][qx] = v[dz];
+               DDQ0[dz][dy][qx] = u;
+               DDQ1[dz][dy][qx] = v;
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(dz,z,D1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int dy = 0; dy < D1D; ++dy)
               {
-                  u[dz] += DDQ1[dz][dy][qx] * B[i][j];
-                  v[dz] += DDQ0[dz][dy][qx] * G[k][l] * s;
-                  w[dz] += DDQ0[dz][dy][qx] * B[i][j];
+                  u += DDQ1[dz][dy][qx] * B[qy][dy];
+                  v += DDQ0[dz][dy][qx] * G[qy][dy];
+                  w += DDQ0[dz][dy][qx] * B[qy][dy];
               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ0[dz][qy][qx] = u[dz];
-               DQQ1[dz][qy][qx] = v[dz];
-               DQQ2[dz][qy][qx] = w[dz];
+               DQQ0[dz][qy][qx] = u;
+               DQQ1[dz][qy][qx] = v;
+               DQQ2[dz][qy][qx] = w;
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int dz = 0; dz < D1D; ++dz)
               {
-                  const int i = qi(qz,dz,Q1D);
-                  const int j = dj(qz,dz,D1D);
-                  const int k = qk(qz,dz,Q1D);
-                  const int l = dl(qz,dz,D1D);
-                  const double s = sign(qz,dz);
-                  u[qz] += DQQ0[dz][qy][qx] * B[i][j];
-                  v[qz] += DQQ1[dz][qy][qx] * B[i][j];
-                  w[qz] += DQQ2[dz][qy][qx] * G[k][l] * s;
+                  u += DQQ0[dz][qy][qx] * B[qz][dz];
+                  v += DQQ1[dz][qy][qx] * B[qz][dz];
+                  w += DQQ2[dz][qy][qx] * G[qz][dz];
               }
+               QQQ0[qz][qy][qx] = u;
+               QQQ1[qz][qy][qx] = v;
+               QQQ2[qz][qy][qx] = w;
            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
            {
-               const double O11 = d(qx,qy,qz,0,e);
-               const double O12 = d(qx,qy,qz,1,e);
-               const double O13 = d(qx,qy,qz,2,e);
-               const double O22 = d(qx,qy,qz,3,e);
-               const double O23 = d(qx,qy,qz,4,e);
-               const double O33 = d(qx,qy,qz,5,e);
-               const double gX = u[qz];
-               const double gY = v[qz];
-               const double gZ = w[qz];
+               const int q = qx + ((qy*Q1D) + (qz*Q1D*Q1D));
+               const double O11 = d(q,0,e);
+               const double O12 = d(q,1,e);
+               const double O13 = d(q,2,e);
+               const double O22 = d(q,3,e);
+               const double O23 = d(q,4,e);
+               const double O33 = d(q,5,e);
+               const double gX = QQQ0[qz][qy][qx];
+               const double gY = QQQ1[qz][qy][qx];
+               const double gZ = QQQ2[qz][qy][qx];
               QQQ0[qz][qy][qx] = (O11*gX) + (O12*gY) + (O13*gZ);
               QQQ1[qz][qy][qx] = (O12*gX) + (O22*gY) + (O23*gZ);
               QQQ2[qz][qy][qx] = (O13*gX) + (O23*gY) + (O33*gZ);
@@ -1540,112 +1467,78 @@ static void SmemPADiffusionApply3D(const int NE,
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
+      if (tidz == 0)
      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
+         MFEM_FOREACH_THREAD(d,y,D1D)
         {
-            const int i = qi(q,d,Q1D);
-            const int j = dj(q,d,D1D);
-            const int k = qk(q,d,Q1D);
-            const int l = dl(q,d,D1D);
-            Bt[j][i] = b(q,d);
-            Gt[l][k] = g(q,d) * sign(q,d);
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bt[d][q] = b(q,d);
+               Gt[d][q] = g(q,d);
+            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
+            MFEM_FOREACH_THREAD(dx,x,D1D)
            {
-               const int i = qi(qx,dx,Q1D);
-               const int j = dj(qx,dx,D1D);
-               const int k = qk(qx,dx,Q1D);
-               const int l = dl(qx,dx,D1D);
-               const double s = sign(qx,dx);
-               MFEM_UNROLL(MQ1)
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  u += QQQ0[qz][qy][qx] * Gt[dx][qx];
+                  v += QQQ1[qz][qy][qx] * Bt[dx][qx];
+                  w += QQQ2[qz][qy][qx] * Bt[dx][qx];
+               }
+               QQD0[qz][qy][dx] = u;
+               QQD1[qz][qy][dx] = v;
+               QQD2[qz][qy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  u += QQD0[qz][qy][dx] * Bt[dy][qy];
+                  v += QQD1[qz][qy][dx] * Gt[dy][qy];
+                  w += QQD2[qz][qy][dx] * Bt[dy][qy];
+               }
+               QDD0[qz][dy][dx] = u;
+               QDD1[qz][dy][dx] = v;
+               QDD2[qz][dy][dx] = w;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dz,z,D1D)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,D1D)
+            {
+               double u = 0.0;
+               double v = 0.0;
+               double w = 0.0;
               for (int qz = 0; qz < Q1D; ++qz)
               {
-                  u[qz] += QQQ0[qz][qy][qx] * Gt[l][k] * s;
-                  v[qz] += QQQ1[qz][qy][qx] * Bt[j][i];
-                  w[qz] += QQQ2[qz][qy][qx] * Bt[j][i];
+                  u += QDD0[qz][dy][dx] * Bt[dz][qz];
+                  v += QDD1[qz][dy][dx] * Bt[dz][qz];
+                  w += QDD2[qz][dy][dx] * Gt[dz][qz];
               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD0[qz][qy][dx] = u[qz];
-               QQD1[qz][qy][dx] = v[qz];
-               QQD2[qz][qy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D], v[Q1D], w[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz) { u[qz] = v[qz] = w[qz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const int i = qi(qy,dy,Q1D);
-               const int j = dj(qy,dy,D1D);
-               const int k = qk(qy,dy,Q1D);
-               const int l = dl(qy,dy,D1D);
-               const double s = sign(qy,dy);
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD0[qz][qy][dx] * Bt[j][i];
-                  v[qz] += QQD1[qz][qy][dx] * Gt[l][k] * s;
-                  w[qz] += QQD2[qz][qy][dx] * Bt[j][i];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD0[qz][dy][dx] = u[qz];
-               QDD1[qz][dy][dx] = v[qz];
-               QDD2[qz][dy][dx] = w[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D], v[D1D], w[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz) { u[dz] = v[dz] = w[dz] = 0.0; }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  const int i = qi(qz,dz,Q1D);
-                  const int j = dj(qz,dz,D1D);
-                  const int k = qk(qz,dz,Q1D);
-                  const int l = dl(qz,dz,D1D);
-                  const double s = sign(qz,dz);
-                  u[dz] += QDD0[qz][dy][dx] * Bt[j][i];
-                  v[dz] += QDD1[qz][dy][dx] * Bt[j][i];
-                  w[dz] += QDD2[qz][dy][dx] * Gt[l][k] * s;
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += (u[dz] + v[dz] + w[dz]);
+               y(dx,dy,dz,e) += (u + v + w);
            }
         }
      }
@@ -1680,11 +1573,9 @@ static void PADiffusionApply(const int dim,
      MFEM_ABORT("OCCA PADiffusionApply unknown kernel!");
   }
 #endif // MFEM_USE_OCCA
-   const int ID = (D1D << 4) | Q1D;
-
   if (dim == 2)
   {
-      switch (ID)
+      switch ((D1D << 4 ) | Q1D)
      {
         case 0x22: return SmemPADiffusionApply2D<2,2,16>(NE,B,G,D,X,Y);
         case 0x33: return SmemPADiffusionApply2D<3,3,16>(NE,B,G,D,X,Y);
@@ -1697,10 +1588,9 @@ static void PADiffusionApply(const int dim,
         default:   return PADiffusionApply2D(NE,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
-
-   if (dim == 3)
+   else if (dim == 3)
   {
-      switch (ID)
+      switch ((D1D << 4 ) | Q1D)
      {
         case 0x23: return SmemPADiffusionApply3D<2,3>(NE,B,G,D,X,Y);
         case 0x34: return SmemPADiffusionApply3D<3,4>(NE,B,G,D,X,Y);
@@ -1723,7 +1613,29 @@ void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 #ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAddMultPA(ceedDataPtr, x, y);
+      const CeedScalar *x_ptr;
+      CeedScalar *y_ptr;
+      CeedMemType mem;
+      CeedGetPreferredMemType(internal::ceed, &mem);
+      if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
+      {
+         x_ptr = x.Read();
+         y_ptr = y.ReadWrite();
+      }
+      else
+      {
+         x_ptr = x.HostRead();
+         y_ptr = y.HostReadWrite();
+         mem = CEED_MEM_HOST;
+      }
+      CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
+                         const_cast<CeedScalar*>(x_ptr));
+      CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
+
+      CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
+                           CEED_REQUEST_IMMEDIATE);
+
+      CeedVectorSyncArray(ceedDataPtr->v, mem);
   }
   else
 #endif
@@ -334,7 +334,7 @@ static void PAGradientApplyTranspose2D(const int NE,
                                       const int q1d = 0)
 {
   // TODO
-   MFEM_ASSERT(false, "PAGradientApplyTranspose2D not implemented.");
+   MFEM_ASSERT(false, "GradientPAApplyTranspose 3D not implemented.");
 }

 // PA Gradient Apply 3D kernel
@@ -23,6 +23,11 @@ using namespace std;
 namespace mfem
 {

+// Local maximum size of dofs and quads in 1D
+constexpr int HDIV_MAX_D1D = 5;
+constexpr int HDIV_MAX_Q1D = 6;
+
+
 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
                   const int NE,
@@ -109,8 +114,6 @@ void PAHdivMassApply2D(const int D1D,
                       Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
@@ -235,7 +238,6 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  Vector &_diag)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Bc = Reshape(_Bc.Read(), Q1D, D1D);
@@ -612,8 +614,6 @@ static void PADivDivApply2D(const int D1D,
                            Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Bot = Reshape(_Bot.Read(), D1D-1, Q1D);
@@ -977,7 +977,6 @@ static void PADivDivAssembleDiagonal2D(const int D1D,
                                       Vector &_diag)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
@@ -1401,8 +1400,6 @@ static void PAHdivL2Apply2D(const int D1D,
                            Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto Bo = Reshape(_Bo.Read(), Q1D, D1D-1);
   auto Gc = Reshape(_Gc.Read(), Q1D, D1D);
@@ -1669,8 +1666,6 @@ static void PAHdivL2ApplyTranspose2D(const int D1D,
                                     Vector &_y)
 {
   constexpr static int VDIM = 2;
-   constexpr static int MAX_D1D = HDIV_MAX_D1D;
-   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;

   auto L2Bo = Reshape(_L2Bo.Read(), Q1D, L2D1D);
   auto Gct = Reshape(_Gct.Read(), D1D, Q1D);
@@ -1729,7 +1724,7 @@ static void PAHdivL2ApplyTranspose2D(const int D1D,

      for (int qy = 0; qy < Q1D; ++qy)
      {
-         double aX[MAX_D1D];
+         double aX[HDIV_MAX_D1D];

         int osc = 0;
         for (int c = 0; c < VDIM; ++c)  // loop over x, y components
@@ -21,7 +21,6 @@ static void EAMassAssemble1D(const int NE,
                             const Array<double> &basis,
                             const Vector &padata,
                             Vector &eadata,
-                             const bool add,
                             const int d1d = 0,
                             const int q1d = 0)
 {
@@ -31,7 +30,7 @@ static void EAMassAssemble1D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(basis.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
+   auto M = Reshape(eadata.Write(), D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -53,14 +52,7 @@ static void EAMassAssemble1D(const int NE,
            {
               val += r_Bi[k1] * r_Bj[k1] * D(k1, e);
            }
-            if (add)
-            {
-               M(i1, j1, e) += val;
-            }
-            else
-            {
-               M(i1, j1, e) = val;
-            }
+            M(i1, j1, e) = val;
         }
      }
   });
@@ -71,7 +63,6 @@ static void EAMassAssemble2D(const int NE,
                             const Array<double> &basis,
                             const Vector &padata,
                             Vector &eadata,
-                             const bool add,
                             const int d1d = 0,
                             const int q1d = 0)
 {
@@ -81,7 +72,7 @@ static void EAMassAssemble2D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(basis.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
+   auto M = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, 1,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -123,14 +114,7 @@ static void EAMassAssemble2D(const int NE,
                               * s_D[k1][k2];
                     }
                  }
-                  if (add)
-                  {
-                     M(i1, i2, j1, j2, e) += val;
-                  }
-                  else
-                  {
-                     M(i1, i2, j1, j2, e) = val;
-                  }
+                  M(i1, i2, j1, j2, e) = val;
               }
            }
         }
@@ -143,7 +127,6 @@ static void EAMassAssemble3D(const int NE,
                             const Array<double> &basis,
                             const Vector &padata,
                             Vector &eadata,
-                             const bool add,
                             const int d1d = 0,
                             const int q1d = 0)
 {
@@ -153,7 +136,7 @@ static void EAMassAssemble3D(const int NE,
   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
   auto B = Reshape(basis.Read(), Q1D, D1D);
   auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
-   auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto M = Reshape(eadata.Write(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, D1D, D1D, D1D,
   {
      const int D1D = T_D1D ? T_D1D : d1d;
@@ -206,14 +189,7 @@ static void EAMassAssemble3D(const int NE,
                              }
                           }
                        }
-                        if (add)
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) += val;
-                        }
-                        else
-                        {
-                           M(i1, i2, i3, j1, j2, j3, e) = val;
-                        }
+                        M(i1, i2, i3, j1, j2, j3, e) = val;
                     }
                  }
               }
@@ -224,8 +200,7 @@ static void EAMassAssemble3D(const int NE,
 }

 void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                Vector &ea_data,
-                                const bool add)
+                                Vector &ea_data)
 {
   AssemblePA(fes);
   const int ne = fes.GetMesh()->GetNE();
@@ -234,47 +209,44 @@ void MassIntegrator::AssembleEA(const FiniteElementSpace &fes,
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x22: return EAMassAssemble1D<2,2>(ne,B,pa_data,ea_data);
+         case 0x33: return EAMassAssemble1D<3,3>(ne,B,pa_data,ea_data);
+         case 0x44: return EAMassAssemble1D<4,4>(ne,B,pa_data,ea_data);
+         case 0x55: return EAMassAssemble1D<5,5>(ne,B,pa_data,ea_data);
+         case 0x66: return EAMassAssemble1D<6,6>(ne,B,pa_data,ea_data);
+         case 0x77: return EAMassAssemble1D<7,7>(ne,B,pa_data,ea_data);
+         case 0x88: return EAMassAssemble1D<8,8>(ne,B,pa_data,ea_data);
+         case 0x99: return EAMassAssemble1D<9,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble1D(ne,B,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 2)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data,add);
-         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data,add);
-         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data,add);
-         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data,add);
-         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data,add);
-         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data,add);
-         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data,add);
-         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x22: return EAMassAssemble2D<2,2>(ne,B,pa_data,ea_data);
+         case 0x33: return EAMassAssemble2D<3,3>(ne,B,pa_data,ea_data);
+         case 0x44: return EAMassAssemble2D<4,4>(ne,B,pa_data,ea_data);
+         case 0x55: return EAMassAssemble2D<5,5>(ne,B,pa_data,ea_data);
+         case 0x66: return EAMassAssemble2D<6,6>(ne,B,pa_data,ea_data);
+         case 0x77: return EAMassAssemble2D<7,7>(ne,B,pa_data,ea_data);
+         case 0x88: return EAMassAssemble2D<8,8>(ne,B,pa_data,ea_data);
+         case 0x99: return EAMassAssemble2D<9,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble2D(ne,B,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   else if (dim == 3)
   {
      switch ((dofs1D << 4 ) | quad1D)
      {
-         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data,add);
-         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data,add);
-         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data,add);
-         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data,add);
-         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data,add);
-         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data,add);
-         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data,add);
-         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,add,
-                                               dofs1D,quad1D);
+         case 0x23: return EAMassAssemble3D<2,3>(ne,B,pa_data,ea_data);
+         case 0x34: return EAMassAssemble3D<3,4>(ne,B,pa_data,ea_data);
+         case 0x45: return EAMassAssemble3D<4,5>(ne,B,pa_data,ea_data);
+         case 0x56: return EAMassAssemble3D<5,6>(ne,B,pa_data,ea_data);
+         case 0x67: return EAMassAssemble3D<6,7>(ne,B,pa_data,ea_data);
+         case 0x78: return EAMassAssemble3D<7,8>(ne,B,pa_data,ea_data);
+         case 0x89: return EAMassAssemble3D<8,9>(ne,B,pa_data,ea_data);
+         default:   return EAMassAssemble3D(ne,B,pa_data,ea_data,dofs1D,quad1D);
      }
   }
   MFEM_ABORT("Unknown kernel.");
@@ -1,13 +1,13 @@
-// Copyright (c) 2010, Lawrence Livermore National Security, LLC. Produced at
-// the Lawrence Livermore National Laboratory. LLNL-CODE-443211. All Rights
-// reserved. See file COPYRIGHT for details.
+// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
 //
 // This file is part of the MFEM library. For more information and source code
-// availability see http://mfem.org.
+// availability visit https://mfem.org.
 //
 // MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the GNU Lesser General Public License (as published by the Free
-// Software Foundation) version 2.1 dated February 1999.
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.

 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
@@ -23,8 +23,9 @@ namespace mfem

 // PA Mass Assemble kernel

-void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
+void MassIntegrator::SetupPA(const FiniteElementSpace &fes, const bool force)
 {
+
   // Assuming the same element type
   fespace = &fes;
   Mesh *mesh = fes.GetMesh();
@@ -33,7 +34,7 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
   ElementTransformation *T = mesh->GetElementTransformation(0);
   const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el, *T);
 #ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
+   if (DeviceCanUseCeed() && !force)
   {
      if (ceedDataPtr) { delete ceedDataPtr; }
      CeedData* ptr = new CeedData();
@@ -50,35 +51,31 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   pa_data.SetSize(ne*nq, Device::GetMemoryType());
-   Vector coeff;
+   pa_data.SetSize(ne*nq, Device::GetDeviceMemoryType());
+   Vector *coeff{nullptr};
+   bool own_coeff{true};
   if (Q == nullptr)
   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
+      coeff = new Vector;
+      coeff->SetSize(1);
+      (*coeff)(0) = 1.0;
   }
   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
+      coeff = new Vector;
+      coeff->SetSize(1);
+      (*coeff)(0) = 1.0;
   }
-   else if (QuadratureFunctionCoefficient* cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   else if (QuadratureCoefficient* cQ = dynamic_cast<QuadratureCoefficient*>(Q))
   {
-      const QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      coeff = cQ->Data();
+      own_coeff = false;
   }
   else
   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      coeff = new Vector;
+      coeff->SetSize(nq * ne);
+      auto C = Reshape(coeff->HostWrite(), nq, ne);
      for (int e = 0; e < ne; ++e)
      {
         ElementTransformation& T = *fes.GetElementTransformation(e);
@@ -92,67 +89,54 @@ void MassIntegrator::SetupPA(const FiniteElementSpace &fes)
   if (dim==2)
   {
      const int NE = ne;
-      const int Q1D = quad1D;
-      const bool const_c = coeff.Size() == 1;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D);
-      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,2,2,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D, NE);
-      MFEM_FORALL_2D(e, NE, Q1D,Q1D,1,
+      const int NQ = nq;
+      const bool const_c = coeff->Size() == 1;
+      auto w = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,2,2,NE);
+      auto C =
+         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
+      auto v = Reshape(pa_data.Write(), NQ, NE);
+      MFEM_FORALL(e, NE,
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         for (int q = 0; q < NQ; ++q)
         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               const double J11 = J(qx,qy,0,0,e);
-               const double J12 = J(qx,qy,1,0,e);
-               const double J21 = J(qx,qy,0,1,e);
-               const double J22 = J(qx,qy,1,1,e);
-               const double detJ = (J11*J22)-(J21*J12);
-               const double coeff = const_c ? C(0,0,0) : C(qx,qy,e);
-               v(qx,qy,e) =  W(qx,qy) * coeff * detJ;
-            }
+            const double J11 = J(q,0,0,e);
+            const double J12 = J(q,1,0,e);
+            const double J21 = J(q,0,1,e);
+            const double J22 = J(q,1,1,e);
+            const double detJ = (J11*J22)-(J21*J12);
+            const double coeff = const_c ? C(0,0) : C(q,e);
+            v(q,e) =  w[q] * coeff * detJ;
         }
      });
   }
   if (dim==3)
   {
      const int NE = ne;
-      const int Q1D = quad1D;
-      const bool const_c = coeff.Size() == 1;
-      const auto W = Reshape(ir->GetWeights().Read(), Q1D,Q1D,Q1D);
-      const auto J = Reshape(geom->J.Read(), Q1D,Q1D,Q1D,3,3,NE);
-      const auto C = const_c ? Reshape(coeff.Read(), 1,1,1,1) :
-                     Reshape(coeff.Read(), Q1D,Q1D,Q1D,NE);
-      auto v = Reshape(pa_data.Write(), Q1D,Q1D,Q1D,NE);
-      MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+      const int NQ = nq;
+      const bool const_c = coeff->Size() == 1;
+      auto W = ir->GetWeights().Read();
+      auto J = Reshape(geom->J.Read(), NQ,3,3,NE);
+      auto C =
+         const_c ? Reshape(coeff->Read(), 1,1) : Reshape(coeff->Read(), NQ,NE);
+      auto v = Reshape(pa_data.Write(), NQ,NE);
+      MFEM_FORALL(e, NE,
      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         for (int q = 0; q < NQ; ++q)
         {
-            MFEM_FOREACH_THREAD(qy,y,Q1D)
-            {
-               MFEM_FOREACH_THREAD(qz,z,Q1D)
-               {
-                  const double J11 = J(qx,qy,qz,0,0,e);
-                  const double J21 = J(qx,qy,qz,1,0,e);
-                  const double J31 = J(qx,qy,qz,2,0,e);
-                  const double J12 = J(qx,qy,qz,0,1,e);
-                  const double J22 = J(qx,qy,qz,1,1,e);
-                  const double J32 = J(qx,qy,qz,2,1,e);
-                  const double J13 = J(qx,qy,qz,0,2,e);
-                  const double J23 = J(qx,qy,qz,1,2,e);
-                  const double J33 = J(qx,qy,qz,2,2,e);
-                  const double detJ = J11 * (J22 * J33 - J32 * J23) -
-                  /* */               J21 * (J12 * J33 - J32 * J13) +
-                  /* */               J31 * (J12 * J23 - J22 * J13);
-                  const double coeff = const_c ? C(0,0,0,0) : C(qx,qy,qz,e);
-                  v(qx,qy,qz,e) = W(qx,qy,qz) * coeff * detJ;
-               }
-            }
+            const double J11 = J(q,0,0,e), J12 = J(q,0,1,e), J13 = J(q,0,2,e);
+            const double J21 = J(q,1,0,e), J22 = J(q,1,1,e), J23 = J(q,1,2,e);
+            const double J31 = J(q,2,0,e), J32 = J(q,2,1,e), J33 = J(q,2,2,e);
+            const double detJ = J11 * (J22 * J33 - J32 * J23) -
+            /* */               J21 * (J12 * J33 - J32 * J13) +
+            /* */               J31 * (J12 * J23 - J22 * J13);
+            const double coeff = const_c ? C(0,0) : C(q,e);
+            v(q,e) = W[q] * coeff * detJ;
         }
      });
   }
+
+   if (own_coeff) { delete coeff; }
 }

 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
@@ -468,16 +452,8 @@ static void PAMassAssembleDiagonal(const int dim, const int D1D,

 void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 {
-#ifdef MFEM_USE_CEED
-   if (DeviceCanUseCeed())
-   {
-      CeedAssembleDiagonalPA(ceedDataPtr, diag);
-   }
-   else
-#endif
-   {
-      PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
-   }
+   if (pa_data.Size()==0) { SetupPA(*fespace, true); }
+   PAMassAssembleDiagonal(dim, dofs1D, quad1D, ne, maps->B, pa_data, diag);
 }


@@ -675,7 +651,6 @@ static void SmemPAMassApply2D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
-   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
@@ -939,7 +914,6 @@ static void SmemPAMassApply3D(const int NE,
                              const int d1d = 0,
                              const int q1d = 0)
 {
-   MFEM_CONTRACT_VAR(bt_);
   const int D1D = T_D1D ? T_D1D : d1d;
   const int Q1D = T_Q1D ? T_Q1D : q1d;
   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -950,9 +924,8 @@ static void SmemPAMassApply3D(const int NE,
   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, Q1D,
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
   {
-      const int tidz = MFEM_THREAD_ID(z);
      const int D1D = T_D1D ? T_D1D : d1d;
      const int Q1D = T_Q1D ? T_Q1D : q1d;
      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
@@ -969,130 +942,188 @@ static void SmemPAMassApply3D(const int NE,
      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(dx,x,D1D)
         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
            {
               X[dz][dy][dx] = x(dx,dy,dz,e);
            }
         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(dx,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][d] = b(q,d);
-            }
+            B[dx][dy] = b(dx,dy);
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(dy,y,D1D)
      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
            {
-               double u = 0.0;
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  u += X[dz][dy][dx] * B[qx][dx];
-               }
-               DDQ[dz][dy][qx] = u;
+               u[dz] = 0;
            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            MFEM_UNROLL(MD1)
+            for (int dx = 0; dx < D1D; ++dx)
            {
-               double u = 0.0;
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  u += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-               DQQ[dz][qy][qx] = u;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u = 0.0;
+               MFEM_UNROLL(MD1)
               for (int dz = 0; dz < D1D; ++dz)
               {
-                  u += DQQ[dz][qy][qx] * B[qz][dz];
+                  u[dz] += X[dz][dy][dx] * B[qx][dx];
               }
-               QQQ[qz][qy][qx] = u * d(qx,qy,qz,e);
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ[dz][dy][qx] = u[dz];
            }
         }
      }
      MFEM_SYNC_THREAD;
-      if (tidz == 0)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
      {
-         MFEM_FOREACH_THREAD(d,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
            {
-               Bt[d][q] = b(q,d);
+               u[dz] = 0;
            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            MFEM_UNROLL(MD1)
+            for (int dy = 0; dy < D1D; ++dy)
            {
-               double u = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; dz++)
               {
-                  u += QQQ[qz][qy][qx] * Bt[dx][qx];
+                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
               }
-               QQD[qz][qy][dx] = u;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ[dz][qy][qx] = u[dz];
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qz,z,Q1D)
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
            {
-               double u = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; qz++)
               {
-                  u += QQD[qz][qy][dx] * Bt[dy][qy];
+                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
               }
-               QDD[qz][dy][dx] = u;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
            }
         }
      }
      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dz,z,D1D)
+      MFEM_FOREACH_THREAD(d,y,D1D)
      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
+         MFEM_FOREACH_THREAD(q,x,Q1D)
         {
-            MFEM_FOREACH_THREAD(dx,x,D1D)
+            Bt[d][q] = b(q,d);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
            {
-               double u = 0.0;
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               MFEM_UNROLL(MQ1)
               for (int qz = 0; qz < Q1D; ++qz)
               {
-                  u += QDD[qz][dy][dx] * Bt[dz][qz];
+                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
               }
-               y(dx,dy,dz,e) += u;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD[qz][qy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD[qz][dy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               y(dx,dy,dz,e) += u[dz];
            }
         }
      }
@@ -1123,14 +1154,20 @@ static void PAMassApply(const int dim,
      MFEM_ABORT("OCCA PA Mass Apply unknown kernel!");
   }
 #endif // MFEM_USE_OCCA
+   const int id = (D1D << 4) | Q1D;
   if (dim == 2)
   {
-      switch ((D1D << 4) | Q1D)
+      switch (id)
      {
         case 0x22: return SmemPAMassApply2D<2,2,16>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply2D<2,4,16>(NE,B,Bt,D,X,Y);
         case 0x33: return SmemPAMassApply2D<3,3,16>(NE,B,Bt,D,X,Y);
+         case 0x34: return SmemPAMassApply2D<3,4,16>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply2D<3,6,16>(NE,B,Bt,D,X,Y);
         case 0x44: return SmemPAMassApply2D<4,4,8>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply2D<4,8,4>(NE,B,Bt,D,X,Y);
         case 0x55: return SmemPAMassApply2D<5,5,8>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply2D<5,8,2>(NE,B,Bt,D,X,Y);
         case 0x66: return SmemPAMassApply2D<6,6,4>(NE,B,Bt,D,X,Y);
         case 0x77: return SmemPAMassApply2D<7,7,4>(NE,B,Bt,D,X,Y);
         case 0x88: return SmemPAMassApply2D<8,8,2>(NE,B,Bt,D,X,Y);
@@ -1140,18 +1177,25 @@ static void PAMassApply(const int dim,
   }
   else if (dim == 3)
   {
-      switch ((D1D << 4) | Q1D)
+      switch (id)
      {
         case 0x23: return SmemPAMassApply3D<2,3>(NE,B,Bt,D,X,Y);
+         case 0x24: return SmemPAMassApply3D<2,4>(NE,B,Bt,D,X,Y);
         case 0x34: return SmemPAMassApply3D<3,4>(NE,B,Bt,D,X,Y);
+         case 0x36: return SmemPAMassApply3D<3,6>(NE,B,Bt,D,X,Y);
         case 0x45: return SmemPAMassApply3D<4,5>(NE,B,Bt,D,X,Y);
+         case 0x46: return SmemPAMassApply3D<4,6>(NE,B,Bt,D,X,Y);
+         case 0x48: return SmemPAMassApply3D<4,8>(NE,B,Bt,D,X,Y);
         case 0x56: return SmemPAMassApply3D<5,6>(NE,B,Bt,D,X,Y);
+         case 0x58: return SmemPAMassApply3D<5,8>(NE,B,Bt,D,X,Y);
         case 0x67: return SmemPAMassApply3D<6,7>(NE,B,Bt,D,X,Y);
         case 0x78: return SmemPAMassApply3D<7,8>(NE,B,Bt,D,X,Y);
         case 0x89: return SmemPAMassApply3D<8,9>(NE,B,Bt,D,X,Y);
+         case 0x9A: return SmemPAMassApply3D<9,10>(NE,B,Bt,D,X,Y);
         default:   return PAMassApply3D(NE,B,Bt,D,X,Y,D1D,Q1D);
      }
   }
+   mfem::out << "Unknown kernel 0x" << std::hex << id << std::endl;
   MFEM_ABORT("Unknown kernel.");
 }

@@ -1160,7 +1204,29 @@ void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 #ifdef MFEM_USE_CEED
   if (DeviceCanUseCeed())
   {
-      CeedAddMultPA(ceedDataPtr, x, y);
+      const CeedScalar *x_ptr;
+      CeedScalar *y_ptr;
+      CeedMemType mem;
+      CeedGetPreferredMemType(internal::ceed, &mem);
+      if ( Device::Allows(Backend::CUDA) && mem==CEED_MEM_DEVICE )
+      {
+         x_ptr = x.Read();
+         y_ptr = y.ReadWrite();
+      }
+      else
+      {
+         x_ptr = x.HostRead();
+         y_ptr = y.HostReadWrite();
+         mem = CEED_MEM_HOST;
+      }
+      CeedVectorSetArray(ceedDataPtr->u, mem, CEED_USE_POINTER,
+                         const_cast<CeedScalar*>(x_ptr));
+      CeedVectorSetArray(ceedDataPtr->v, mem, CEED_USE_POINTER, y_ptr);
+
+      CeedOperatorApplyAdd(ceedDataPtr->oper, ceedDataPtr->u, ceedDataPtr->v,
+                           CEED_REQUEST_IMMEDIATE);
+
+      CeedVectorSyncArray(ceedDataPtr->v, mem);
   }
   else
 #endif
@@ -16,171 +16,88 @@ namespace mfem
 {

 void TransposeIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                     Vector &ea_data, const bool add)
+                                     Vector &ea_data)
 {
-   if (add)
+   Vector ea_data_tmp(ea_data.Size());
+   ea_data_tmp = 0.0;
+   bfi->AssembleEA(fes, ea_data_tmp);
+   const int ne = fes.GetNE();
+   if (ne == 0) { return; }
+   const int dofs = fes.GetFE(0)->GetDof();
+   auto A = Reshape(ea_data_tmp.Write(), dofs, dofs, ne);
+   auto AT = Reshape(ea_data.Write(), dofs, dofs, ne);
+   MFEM_FORALL(e, ne,
   {
-      Vector ea_data_tmp(ea_data.Size());
-      bfi->AssembleEA(fes, ea_data_tmp, false);
-      const int ne = fes.GetNE();
-      if (ne == 0) { return; }
-      const int dofs = fes.GetFE(0)->GetDof();
-      auto A = Reshape(ea_data_tmp.Read(), dofs, dofs, ne);
-      auto AT = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
+      for (int i = 0; i < dofs; i++)
      {
-         for (int i = 0; i < dofs; i++)
+         for (int j = 0; j < dofs; j++)
         {
-            for (int j = 0; j < dofs; j++)
-            {
-               const double a = A(i, j, e);
-               AT(j, i, e) += a;
-            }
+            const double a = A(i, j, e);
+            AT(j, i, e) += a;
         }
-      });
-   }
-   else
-   {
-      bfi->AssembleEA(fes, ea_data, false);
-      const int ne = fes.GetNE();
-      if (ne == 0) { return; }
-      const int dofs = fes.GetFE(0)->GetDof();
-      auto A = Reshape(ea_data.ReadWrite(), dofs, dofs, ne);
-      MFEM_FORALL(e, ne,
-      {
-         for (int i = 0; i < dofs; i++)
-         {
-            for (int j = i+1; j < dofs; j++)
-            {
-               const double aij = A(i, j, e);
-               const double aji = A(j, i, e);
-               A(j, i, e) = aij;
-               A(i, j, e) = aji;
-            }
-         }
-      });
-   }
+      }
+   });
 }

 void TransposeIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace& fes,
                                                  Vector &ea_data_int,
-                                                  Vector &ea_data_ext,
-                                                  const bool add)
+                                                  Vector &ea_data_ext)
 {
   const int nf = fes.GetNFbyType(FaceType::Interior);
   if (nf == 0) { return; }
-   if (add)
+   Vector ea_data_int_tmp(ea_data_int.Size());
+   Vector ea_data_ext_tmp(ea_data_ext.Size());
+   ea_data_int_tmp = 0.0;
+   ea_data_ext_tmp = 0.0;
+   bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp);
+   const int faceDofs = fes.GetTraceElement(0,
+                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+   auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
+   auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
+   auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
+   auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
+   MFEM_FORALL(f, nf,
   {
-      Vector ea_data_int_tmp(ea_data_int.Size());
-      Vector ea_data_ext_tmp(ea_data_ext.Size());
-      bfi->AssembleEAInteriorFaces(fes, ea_data_int_tmp, ea_data_ext_tmp, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_int = Reshape(ea_data_int_tmp.Read(), faceDofs, faceDofs, 2, nf);
-      auto A_ext = Reshape(ea_data_ext_tmp.Read(), faceDofs, faceDofs, 2, nf);
-      auto AT_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      auto AT_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
+      for (int i = 0; i < faceDofs; i++)
      {
-         for (int i = 0; i < faceDofs; i++)
+         for (int j = 0; j < faceDofs; j++)
         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double a_int0 = A_int(i, j, 0, f);
-               const double a_int1 = A_int(i, j, 1, f);
-               const double a_ext0 = A_ext(i, j, 0, f);
-               const double a_ext1 = A_ext(i, j, 1, f);
-               AT_int(j, i, 0, f) += a_int0;
-               AT_int(j, i, 1, f) += a_int1;
-               AT_ext(j, i, 0, f) += a_ext1;
-               AT_ext(j, i, 1, f) += a_ext0;
-            }
+            const double a_int0 = A_int(i, j, 0, f);
+            const double a_int1 = A_int(i, j, 1, f);
+            const double a_ext0 = A_ext(i, j, 0, f);
+            const double a_ext1 = A_ext(i, j, 1, f);
+            AT_int(j, i, 0, f) += a_int0;
+            AT_int(j, i, 1, f) += a_int1;
+            AT_ext(j, i, 0, f) += a_ext1;
+            AT_ext(j, i, 1, f) += a_ext0;
         }
-      });
-   }
-   else
-   {
-      bfi->AssembleEAInteriorFaces(fes, ea_data_int, ea_data_ext, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_int = Reshape(ea_data_int.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      auto A_ext = Reshape(ea_data_ext.ReadWrite(), faceDofs, faceDofs, 2, nf);
-      MFEM_FORALL(f, nf,
-      {
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = i+1; j < faceDofs; j++)
-            {
-               const double aij_int0 = A_int(i, j, 0, f);
-               const double aij_int1 = A_int(i, j, 1, f);
-               const double aji_int0 = A_int(j, i, 0, f);
-               const double aji_int1 = A_int(j, i, 1, f);
-               A_int(j, i, 0, f) = aij_int0;
-               A_int(j, i, 1, f) = aij_int1;
-               A_int(i, j, 0, f) = aji_int0;
-               A_int(i, j, 1, f) = aji_int1;
-            }
-         }
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double aij_ext0 = A_ext(i, j, 0, f);
-               const double aji_ext1 = A_ext(j, i, 1, f);
-               A_ext(j, i, 1, f) = aij_ext0;
-               A_ext(i, j, 0, f) = aji_ext1;
-            }
-         }
-      });
-   }
+      }
+   });
 }

 void TransposeIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace& fes,
-                                                  Vector &ea_data_bdr,
-                                                  const bool add)
+                                                  Vector &ea_data_bdr)
 {
   const int nf = fes.GetNFbyType(FaceType::Boundary);
   if (nf == 0) { return; }
-   if (add)
+   Vector ea_data_bdr_tmp(ea_data_bdr.Size());
+   ea_data_bdr_tmp = 0.0;
+   bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp);
+   const int faceDofs = fes.GetTraceElement(0,
+                                            fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
+   auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
+   auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
+   MFEM_FORALL(f, nf,
   {
-      Vector ea_data_bdr_tmp(ea_data_bdr.Size());
-      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr_tmp, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_bdr = Reshape(ea_data_bdr_tmp.Read(), faceDofs, faceDofs, nf);
-      auto AT_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
+      for (int i = 0; i < faceDofs; i++)
      {
-         for (int i = 0; i < faceDofs; i++)
+         for (int j = 0; j < faceDofs; j++)
         {
-            for (int j = 0; j < faceDofs; j++)
-            {
-               const double a_bdr = A_bdr(i, j, f);
-               AT_bdr(j, i, f) += a_bdr;
-            }
+            const double a_bdr = A_bdr(i, j, f);
+            AT_bdr(j, i, f) += a_bdr;
         }
-      });
-   }
-   else
-   {
-      bfi->AssembleEABoundaryFaces(fes, ea_data_bdr, false);
-      const int faceDofs = fes.GetTraceElement(0,
-                                               fes.GetMesh()->GetFaceBaseGeometry(0))->GetDof();
-      auto A_bdr = Reshape(ea_data_bdr.ReadWrite(), faceDofs, faceDofs, nf);
-      MFEM_FORALL(f, nf,
-      {
-         for (int i = 0; i < faceDofs; i++)
-         {
-            for (int j = i+1; j < faceDofs; j++)
-            {
-               const double aij_bdr = A_bdr(i, j, f);
-               const double aji_bdr = A_bdr(j, i, f);
-               A_bdr(j, i, f) = aij_bdr;
-               A_bdr(i, j, f) = aji_bdr;
-            }
-         }
-      });
-   }
+      }
+   });
 }

 }
@@ -12,6 +12,7 @@
 // Implementation of Coefficient class

 #include "fem.hpp"
+#include "../linalg/dtensor.hpp"

 #include <cmath>
 #include <limits>
@@ -21,6 +22,13 @@ namespace mfem

 using namespace std;

+double QuadratureCoefficient::Eval(ElementTransformation & T,
+                                   const IntegrationPoint & ip)
+{
+   auto coeff = mfem::Reshape(qData->HostRead(), nip, NE);
+   return coeff(ip.index, T.ElementNo);
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                const IntegrationPoint & ip)
 {
@@ -209,24 +217,18 @@ void GradientGridFunctionCoefficient::Eval(
   GridFunc->GetGradients(T, ir, M);
 }

-CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
+CurlGridFunctionCoefficient::CurlGridFunctionCoefficient (
   const GridFunction *gf)
-   : VectorCoefficient(0)
+   : VectorCoefficient ((gf) ?
+                        gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0)
 {
-   SetGridFunction(gf);
+   GridFunc = gf;
 }

 void CurlGridFunctionCoefficient::SetGridFunction(const GridFunction *gf)
 {
-   if (gf)
-   {
-      int sdim = gf -> FESpace() -> GetMesh() -> SpaceDimension();
-      MFEM_VERIFY(sdim == 2 || sdim == 3,
-                  "CurlGridFunctionCoefficient "
-                  "only defind for spaces of dimension 2 or 3.");
-   }
-   GridFunc = gf;
-   vdim = (gf) ? (2 * gf -> FESpace() -> GetMesh() -> SpaceDimension() - 3) : 0;
+   GridFunc = gf; vdim = (gf) ?
+                         gf -> FESpace() -> GetMesh() -> SpaceDimension() : 0;
 }

 void CurlGridFunctionCoefficient::Eval(Vector &V, ElementTransformation &T,
@@ -319,31 +321,6 @@ void MatrixFunctionCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
   }
 }

-void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
-                                              ElementTransformation &T,
-                                              const IntegrationPoint &ip)
-{
-   MFEM_VERIFY(symmetric && height == width && height < 4 && SymmFunction,
-               "MatrixFunctionCoefficient is not symmetric");
-
-   double x[3];
-   Vector transip(x, 3);
-
-   T.Transform(ip, transip);
-
-   K.SetSize((width * (width + 1)) / 2); // 1x1: 1, 2x2: 3, 3x3: 6
-
-   if (SymmFunction)
-   {
-      (*SymmFunction)(transip, K);
-   }
-
-   if (Q)
-   {
-      K *= Q->Eval(T, ip, GetTime());
-   }
-}
-
 MatrixArrayCoefficient::MatrixArrayCoefficient (int dim)
   : MatrixCoefficient (dim)
 {
@@ -30,10 +30,7 @@ class ParMesh;
 /** @brief Base class Coefficients that optionally depend on space and time.
    These are used by the BilinearFormIntegrator, LinearFormIntegrator, and
    NonlinearFormIntegrator classes to represent the physical coefficients in
-    the PDEs that are being discretized. This class can also be used in a more
-    general way to represent functions that don't necessarily belong to a FE
-    space, e.g., to project onto GridFunctions to use as initial conditions,
-    exact solutions, etc. See, e.g., ex4 or ex22 for these uses. */
+    the PDEs that are being discretized. */
 class Coefficient
 {
 protected:
@@ -87,6 +84,33 @@ public:
   { return (constant); }
 };

+
+/// class for quadrature coefficient
+class QuadratureCoefficient : public Coefficient
+{
+
+private:
+   const int nip;
+   const int NE;
+public:
+   Vector *qData{nullptr};
+
+   //Set external data
+   QuadratureCoefficient(Vector *Data, int in_nip, int in_NE)
+      : qData(Data), nip(in_nip), NE(in_NE)
+   { }
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip);
+
+   Vector *Data()
+   {
+      return qData;
+   }
+
+};
+
+/// class for piecewise constant coefficient
 /** @brief A piecewise constant coefficient with the constants keyed
    off the element attribute numbers. */
 class PWConstCoefficient : public Coefficient
@@ -695,16 +719,13 @@ class MatrixCoefficient
 protected:
   int height, width;
   double time;
-   bool symmetric;

 public:
   /// Construct a dim x dim matrix coefficient.
-   explicit MatrixCoefficient(int dim, bool symm=false)
-   { height = width = dim; time = 0.; symmetric = symm; }
+   explicit MatrixCoefficient(int dim) { height = width = dim; time = 0.; }

   /// Construct a h x w matrix coefficient.
-   MatrixCoefficient(int h, int w, bool symm=false) :
-      height(h), width(w), time(0.), symmetric(symm) { }
+   MatrixCoefficient(int h, int w) : height(h), width(w), time(0.) { }

   /// Set the time for time dependent coefficients
   void SetTime(double t) { time = t; }
@@ -721,9 +742,6 @@ public:
   /// For backward compatibility get the width of the matrix.
   int GetVDim() const { return width; }

-   void SetSymmetric(bool s) { symmetric = s; }
-   bool IsSymmetric() const { return symmetric; }
-
   /** @brief Evaluate the matrix coefficient in the element described by @a T
       at the point @a ip, storing the result in @a K. */
   /** @note When this method is called, the caller must make sure that the
@@ -732,15 +750,6 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip) = 0;

-   /** @brief Evaluate the upper triangular entries of the matrix coefficient
-       in the symmetric case, similarly to Eval. Matrix entry (i,j) is stored
-       in K[j - i + os_i] for 0 <= i <= j < width, os_0 = 0,
-       os_{i+1} = os_i + width - i. That is, K = {M(0,0), ..., M(0,w-1),
-       M(1,1), ..., M(1,w-1), ..., M(w-1,w-1) with w = width. */
-   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
-                              const IntegrationPoint &ip)
-   { mfem_error("MatrixCoefficient::EvalSymmetric"); }
-
   virtual ~MatrixCoefficient() { }
 };

@@ -768,7 +777,6 @@ class MatrixFunctionCoefficient : public MatrixCoefficient
 {
 private:
   void (*Function)(const Vector &, DenseMatrix &);
-   void (*SymmFunction)(const Vector &, Vector &);
   void (*TDFunction)(const Vector &, double, DenseMatrix &);
   Coefficient *Q;
   DenseMatrix mat;
@@ -806,26 +814,10 @@ public:
      mat.SetSize(0);
   }

-   /// Construct a symmetric square matrix coefficient from a C-function
-   /// defining a vector function used by EvalSymmetric
-   MatrixFunctionCoefficient(int dim, void (*F)(const Vector &, Vector &),
-                             Coefficient *q = NULL)
-      : MatrixCoefficient(dim, true), Q(q)
-   {
-      SymmFunction = F;
-      Function = NULL;
-      TDFunction = NULL;
-      mat.SetSize(0);
-   }
-
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

-   /// Evaluate the symmetric matrix coefficient at @a ip.
-   virtual void EvalSymmetric(Vector &K, ElementTransformation &T,
-                              const IntegrationPoint &ip);
-
   virtual ~MatrixFunctionCoefficient() { }
 };

@@ -38,8 +38,8 @@ protected:
   void Destroy() { delete gfr; delete gfi; }

 public:
-   /** @brief Construct a ComplexGridFunction associated with the
-       FiniteElementSpace @a *f. */
+   /* @brief Construct a ComplexGridFunction associated with the
+      FiniteElementSpace @a *f. */
   ComplexGridFunction(FiniteElementSpace *f);

   void Update();
@@ -71,14 +71,6 @@ public:
   const GridFunction & real() const { return *gfr; }
   const GridFunction & imag() const { return *gfi; }

-   /// Update the memory location of the real and imaginary GridFunction @a gfr
-   /// and @a gfi to match the ComplexGridFunction.
-   void Sync() { gfr->SyncMemory(*this); gfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary GridFunction
-   /// @a gfr and @a gfi to match the ComplexGridFunction.
-   void SyncAlias() { gfr->SyncAliasMemory(*this); gfi->SyncAliasMemory(*this); }
-
   /// Destroys the grid function.
   virtual ~ComplexGridFunction() { Destroy(); }

@@ -107,8 +99,8 @@ public:
                     ComplexOperator::Convention
                     convention = ComplexOperator::HERMITIAN);

-   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a fes, using
-       the same integrators as the LinearForms @a lf_r (real) and @a lf_i (imag).
+   /** @brief Create a ComplexLinearForm on the FiniteElementSpace @a f, using
+       the same integrators as the LinearForms @a lfr (real) and @a lfi (imag) .

       The pointer @a fes is not owned by the newly constructed object.

@@ -165,14 +157,6 @@ public:
   const LinearForm & real() const { return *lfr; }
   const LinearForm & imag() const { return *lfi; }

-   /// Update the memory location of the real and imaginary LinearForm @a lfr
-   /// and @a lfi to match the ComplexLinearForm.
-   void Sync() { lfr->SyncMemory(*this); lfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary LinearForm @a
-   /// lfr and @a lfi to match the ComplexLinearForm.
-   void SyncAlias() { lfr->SyncAliasMemory(*this); lfi->SyncAliasMemory(*this); }
-
   void Update();
   void Update(FiniteElementSpace *f);

@@ -211,8 +195,8 @@ private:
   BilinearForm *blfr;
   BilinearForm *blfi;

-   /* These methods check if the real/imag parts of the sesquilinear form are
-      not empty */
+   /* These methods check if the real/imag parts of the sesqulinear form are not
+      empty */
   bool RealInteg();
   bool ImagInteg();

@@ -220,7 +204,7 @@ public:
   SesquilinearForm(FiniteElementSpace *fes,
                    ComplexOperator::Convention
                    convention = ComplexOperator::HERMITIAN);
-   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a fes, using
+   /** @brief Create a SesquilinearForm on the FiniteElementSpace @a f, using
       the same integrators as the BilinearForms @a bfr and @a bfi .

       The pointer @a fes is not owned by the newly constructed object.
@@ -235,21 +219,6 @@ public:
   void SetConvention(const ComplexOperator::Convention &
                      convention) { conv = convention; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      blfr->SetAssemblyLevel(assembly_level);
-      blfi->SetAssemblyLevel(assembly_level);
-   }
-
   BilinearForm & real() { return *blfr; }
   BilinearForm & imag() { return *blfi; }
   const BilinearForm & real() const { return *blfr; }
@@ -339,8 +308,8 @@ protected:

 public:

-   /** @brief Construct a ParComplexGridFunction associated with the
-       ParFiniteElementSpace @a *pf. */
+   /* @brief Construct a ParComplexGridFunction associated with the
+      ParFiniteElementSpace @a *f. */
   ParComplexGridFunction(ParFiniteElementSpace *pf);

   void Update();
@@ -381,15 +350,6 @@ public:
   const ParGridFunction & real() const { return *pgfr; }
   const ParGridFunction & imag() const { return *pgfi; }

-   /// Update the memory location of the real and imaginary ParGridFunction @a
-   /// pgfr and @a pgfi to match the ParComplexGridFunction.
-   void Sync() { pgfr->SyncMemory(*this); pgfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary
-   /// ParGridFunction @a pgfr and @a pgfi to match the ParComplexGridFunction.
-   void SyncAlias() { pgfr->SyncAliasMemory(*this); pgfi->SyncAliasMemory(*this); }
-
-
   virtual double ComputeL2Error(Coefficient &exsolr, Coefficient &exsoli,
                                 const IntegrationRule *irs[] = NULL) const
   {
@@ -441,8 +401,8 @@ public:
                        convention = ComplexOperator::HERMITIAN);

   /** @brief Create a ParComplexLinearForm on the ParFiniteElementSpace @a pf,
-       using the same integrators as the LinearForms @a plf_r (real) and
-       @a plf_i (imag).
+       using the same integrators as the LinearForms @a plfr (real) and @a plfi
+       (imag) .

      The pointer @a fes is not owned by the newly constructed object.

@@ -500,14 +460,6 @@ public:
   const ParLinearForm & real() const { return *plfr; }
   const ParLinearForm & imag() const { return *plfi; }

-   /// Update the memory location of the real and imaginary ParLinearForm @a lfr
-   /// and @a lfi to match the ParComplexLinearForm.
-   void Sync() { plfr->SyncMemory(*this); plfi->SyncMemory(*this); }
-
-   /// Update the alias memory location of the real and imaginary ParLinearForm
-   /// @a plfr and @a plfi to match the ParComplexLinearForm.
-   void SyncAlias() { plfr->SyncAliasMemory(*this); plfi->SyncAliasMemory(*this); }
-
   void Update(ParFiniteElementSpace *pf = NULL);

   /// Assembles the linear form i.e. sums over all domain/bdr integrators.
@@ -526,7 +478,7 @@ public:
 /** Class for a parallel sesquilinear form

    A sesquilinear form is a generalization of a bilinear form to complex-valued
-    fields. Sesquilinear forms are linear in the second argument but the
+    fields. Sesquilinear forms are linear in the second argument but but the
    first argument involves a complex conjugate in the sense that:

                a(alpha u, beta v) = conj(alpha) beta a(u, v)
@@ -572,21 +524,6 @@ public:
   void SetConvention(const ComplexOperator::Convention &
                      convention) { conv = convention; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::FULL  (default)
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
-   void SetAssemblyLevel(AssemblyLevel assembly_level)
-   {
-      pblfr->SetAssemblyLevel(assembly_level);
-      pblfi->SetAssemblyLevel(assembly_level);
-   }
-
   ParBilinearForm & real() { return *pblfr; }
   ParBilinearForm & imag() { return *pblfi; }
   const ParBilinearForm & real() const { return *pblfr; }
@@ -1,297 +0,0 @@
-#include "convergence.hpp"
-
-using namespace std;
-
-
-namespace mfem
-{
-
-void ConvergenceStudy::Reset()
-{
-   counter=0;
-   dcounter=0;
-   fcounter=0;
-   cont_type=-1;
-   print_flag=1;
-   L2Errors.SetSize(0);
-   L2Rates.SetSize(0);
-   DErrors.SetSize(0);
-   DRates.SetSize(0);
-   EnErrors.SetSize(0);
-   EnRates.SetSize(0);
-   DGFaceErrors.SetSize(0);
-   DGFaceRates.SetSize(0);
-   ndofs.SetSize(0);
-}
-
-double ConvergenceStudy::GetNorm(GridFunction *gf, Coefficient *scalar_u,
-                                 VectorCoefficient *vector_u)
-{
-   bool norm_set = false;
-   double norm=0.0;
-   int order = gf->FESpace()->GetOrder(0);
-   int order_quad = std::max(2, 2*order+1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i=0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-#ifdef MFEM_USE_MPI
-   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
-   if (pgf)
-   {
-      ParMesh *pmesh = pgf->ParFESpace()->GetParMesh();
-      if (scalar_u)
-      {
-         norm = ComputeGlobalLpNorm(2.0,*scalar_u,*pmesh,irs);
-      }
-      else if (vector_u)
-      {
-         norm = ComputeGlobalLpNorm(2.0,*vector_u,*pmesh,irs);
-      }
-      norm_set = true;
-   }
-#endif
-   if (!norm_set)
-   {
-      Mesh *mesh = gf->FESpace()->GetMesh();
-      if (scalar_u)
-      {
-         norm = ComputeLpNorm(2.0,*scalar_u,*mesh,irs);
-      }
-      else if (vector_u)
-      {
-         norm = ComputeLpNorm(2.0,*vector_u,*mesh,irs);
-      }
-   }
-   return norm;
-}
-
-void ConvergenceStudy::AddL2Error(GridFunction *gf,
-                                  Coefficient *scalar_u, VectorCoefficient *vector_u)
-{
-   int tdofs=0;
-#ifdef MFEM_USE_MPI
-   ParGridFunction *pgf = dynamic_cast<ParGridFunction *>(gf);
-   if (pgf)
-   {
-      MPI_Comm comm = pgf->ParFESpace()->GetComm();
-      int rank;
-      MPI_Comm_rank(comm, &rank);
-      print_flag = 0;
-      if (rank==0) { print_flag = 1; }
-      tdofs = pgf->ParFESpace()->GlobalTrueVSize();
-   }
-#endif
-   if (!tdofs) { tdofs = gf->FESpace()->GetTrueVSize(); }
-   ndofs.Append(tdofs);
-   double L2Err;
-   if (scalar_u)
-   {
-      L2Err = gf->ComputeL2Error(*scalar_u);
-      CoeffNorm = GetNorm(gf,scalar_u,nullptr);
-   }
-   else if (vector_u)
-   {
-      L2Err = gf->ComputeL2Error(*vector_u);
-      CoeffNorm = GetNorm(gf,nullptr,vector_u);
-   }
-   else
-   {
-      MFEM_ABORT("Exact Solution Coefficient pointer is NULL");
-   }
-   L2Errors.Append(L2Err);
-   // Compute the rate of convergence by:
-   // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-   double val = (counter) ? log(L2Errors[counter-1]/L2Err)/log(2.0) : 0.0;
-   L2Rates.Append(val);
-   counter++;
-}
-
-void ConvergenceStudy::AddGf(GridFunction *gf, Coefficient *scalar_u,
-                             VectorCoefficient *grad,
-                             Coefficient *ell_coeff, double Nu)
-{
-   cont_type = gf->FESpace()->FEColl()->GetContType();
-
-   MFEM_VERIFY((cont_type == mfem::FiniteElementCollection::CONTINUOUS) ||
-               (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS),
-               "This constructor is intended for H1 or L2 Elements")
-
-   AddL2Error(gf,scalar_u, nullptr);
-
-   if (grad)
-   {
-      double GradErr = gf->ComputeGradError(grad);
-      DErrors.Append(GradErr);
-      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1]+GradErr*GradErr);
-      EnErrors.Append(err);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val = (dcounter) ? log(DErrors[dcounter-1]/GradErr)/log(2.0) : 0.0;
-      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
-      DRates.Append(val);
-      EnRates.Append(eval);
-      CoeffDNorm = GetNorm(gf,nullptr,grad);
-      dcounter++;
-      MFEM_VERIFY(counter == dcounter,
-                  "Number of added solutions and derivatives do not match")
-   }
-
-   if (cont_type == mfem::FiniteElementCollection::DISCONTINUOUS && ell_coeff)
-   {
-      double DGErr = gf->ComputeDGFaceJumpError(scalar_u,ell_coeff,Nu);
-      DGFaceErrors.Append(DGErr);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val=(fcounter) ? log(DGFaceErrors[fcounter-1]/DGErr)/log(2.0):0.;
-      DGFaceRates.Append(val);
-      fcounter++;
-      MFEM_VERIFY(fcounter == counter, "Number of added solutions mismatch");
-   }
-}
-
-void ConvergenceStudy::AddGf(GridFunction *gf, VectorCoefficient *vector_u,
-                             VectorCoefficient *curl, Coefficient *div)
-{
-   cont_type = gf->FESpace()->FEColl()->GetContType();
-
-   AddL2Error(gf,nullptr,vector_u);
-   double DErr = 0.0;
-   bool derivative = false;
-   if (curl)
-   {
-      DErr = gf->ComputeCurlError(curl);
-      CoeffDNorm = GetNorm(gf,nullptr,curl);
-      derivative = true;
-   }
-   else if (div)
-   {
-      DErr = gf->ComputeDivError(div);
-      // update coefficient norm
-      CoeffDNorm = GetNorm(gf,div,nullptr);
-      derivative = true;
-   }
-   if (derivative)
-   {
-      double err = sqrt(L2Errors[counter-1]*L2Errors[counter-1] + DErr*DErr);
-      DErrors.Append(DErr);
-      EnErrors.Append(err);
-      // Compute the rate of convergence by:
-      // rate = log (||u - u_h|| / ||u - u_{h/2}||)/log(2)
-      double val = (dcounter) ? log(DErrors[dcounter-1]/DErr)/log(2.0) : 0.0;
-      double eval = (dcounter) ? log(EnErrors[dcounter-1]/err)/log(2.0) : 0.0;
-      DRates.Append(val);
-      EnRates.Append(eval);
-      dcounter++;
-      MFEM_VERIFY(counter == dcounter,
-                  "Number of added solutions and derivatives do not match")
-   }
-}
-
-void ConvergenceStudy::Print(bool relative, std::ostream &out)
-{
-   if (print_flag)
-   {
-      std::string title = (relative) ? "Relative " : "Absolute ";
-      out << "\n";
-      out << " -------------------------------------------" << "\n";
-      out <<  std::setw(21) << title << "L2 Error " << "\n";
-      out << " -------------------------------------------"
-          << "\n";
-      out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13) << "Error ";
-      out <<  std::setw(15) << "Rate " << "\n";
-      out << " -------------------------------------------"
-          << "\n";
-      out << std::setprecision(4);
-      double d = (relative) ? CoeffNorm : 1.0;
-      for (int i =0; i<counter; i++)
-      {
-         out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-             << std::scientific << L2Errors[i]/d << std::setw(13)
-             << std::fixed << L2Rates[i] << "\n";
-      }
-      out << "\n";
-      if (dcounter == counter)
-      {
-         std::string dname;
-         switch (cont_type)
-         {
-            case 0: dname = "Grad"; break;
-            case 1: dname = "Curl"; break;
-            case 2: dname = "Div";  break;
-            case 3: dname = "DG Grad";  break;
-            default: break;
-         }
-         out << " -------------------------------------------" << "\n";
-         out <<  std::setw(21) << title << dname << " Error  " << "\n";
-         out << " -------------------------------------------" << "\n";
-         out << std::right<<std::setw(11)<< "DOFs "<< std::setw(13) << "Error";
-         out << std::setw(15) << "Rate " << "\n";
-         out << " -------------------------------------------"
-             << "\n";
-         out << std::setprecision(4);
-         d = (relative) ? CoeffDNorm : 1.0;
-         for (int i =0; i<dcounter; i++)
-         {
-            out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                << std::scientific << DErrors[i]/d << std::setw(13)
-                << std::fixed << DRates[i] << "\n";
-         }
-         out << "\n";
-         switch (cont_type)
-         {
-            case 0: dname = "H1"; break;
-            case 1: dname = "H(Curl)"; break;
-            case 2: dname = "H(Div)";  break;
-            case 3: dname = "DG H1";  break;
-            default: break;
-         }
-
-         if (dcounter)
-         {
-            d = (relative) ?
-                sqrt(CoeffNorm*CoeffNorm + CoeffDNorm*CoeffDNorm):1.0;
-
-            out << " -------------------------------------------" << "\n";
-            out << std::setw(21) << title << dname << " Error   " << "\n";
-            out << " -------------------------------------------" << "\n";
-            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
-            out << "Error ";
-            out << std::setw(15) << "Rate " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::setprecision(4);
-            for (int i =0; i<dcounter; i++)
-            {
-               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                   << std::scientific << EnErrors[i]/d << std::setw(13)
-                   << std::fixed << EnRates[i] << "\n";
-            }
-            out << "\n";
-         }
-         if (cont_type == 3 && fcounter)
-         {
-            out << " -------------------------------------------" << "\n";
-            out << "            DG Face Jump Error          " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::right<< std::setw(11)<< "DOFs "<< std::setw(13);
-            out << "Error ";
-            out << std::setw(15) << "Rate " << "\n";
-            out << " -------------------------------------------"
-                << "\n";
-            out << std::setprecision(4);
-            for (int i =0; i<fcounter; i++)
-            {
-               out << std::right << std::setw(10)<< ndofs[i] << std::setw(16)
-                   << std::scientific << DGFaceErrors[i] << std::setw(13)
-                   << std::fixed << DGFaceRates[i] << "\n";
-            }
-            out << "\n";
-         }
-      }
-   }
-}
-
-} // namespace mfem
@@ -1,149 +0,0 @@
-// Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_CONVERGENCE
-#define MFEM_CONVERGENCE
-
-#include "../linalg/linalg.hpp"
-#include "gridfunc.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#endif
-
-namespace mfem
-{
-
-/** @brief Class to compute error and convergence rates.
-    It supports H1, H(curl) (ND elements), H(div) (RT elements) and L2 (DG).
-
-    For "smooth enough" solutions the Galerkin error measured in the appropriate
-    norm satisfies || u - u_h || ~ h^k
-
-    Here, k is called the asymptotic rate of convergence
-
-    For successive uniform h-refinements the rate can be estimated by
-    k = log(||u - u_h|| / ||u - u_{h/2}||)/log(2)
-*/
-class ConvergenceStudy
-{
-private:
-   // counters for solutions/derivatives
-   int counter=0;
-   int dcounter=0;
-   int fcounter=0;
-
-   // space continuity type
-   int cont_type=-1;
-
-   // printing flag for helpful for MPI calls
-   int print_flag=1;
-
-   // exact solution and derivatives
-   double CoeffNorm;
-   double CoeffDNorm;
-
-   // Arrays to store error/rates
-   Array<double> L2Errors, DGFaceErrors, DErrors, EnErrors;
-   Array<double> L2Rates, DGFaceRates, DRates, EnRates;
-   Array<int> ndofs;
-
-   void AddL2Error(GridFunction *gf, Coefficient *scalar_u,
-                   VectorCoefficient *vector_u);
-   void AddGf(GridFunction *gf, Coefficient *scalar_u,
-              VectorCoefficient *grad=nullptr,
-              Coefficient *ell_coeff=nullptr, double Nu=1.0);
-   void AddGf(GridFunction *gf, VectorCoefficient *vector_u,
-              VectorCoefficient *curl, Coefficient *div);
-   // returns the L2-norm of scalar_u or vector_u
-   double GetNorm(GridFunction *gf, Coefficient *scalar_u,
-                  VectorCoefficient *vector_u);
-
-public:
-
-   /// Clear any internal data
-   void Reset();
-
-   /// Add L2 GridFunction, the exact solution and possibly its gradient and/or
-   /// DG face jumps parameters
-   void AddL2GridFunction(GridFunction *gf, Coefficient *scalar_u,
-                          VectorCoefficient *grad=nullptr,
-                          Coefficient *ell_coeff=nullptr, double Nu=1.0)
-   {
-      AddGf(gf, scalar_u, grad, ell_coeff, Nu);
-   }
-
-   /// Add H1 GridFunction, the exact solution and possibly its gradient
-   void AddH1GridFunction(GridFunction *gf, Coefficient *scalar_u,
-                          VectorCoefficient *grad=nullptr)
-   {
-      AddGf(gf, scalar_u, grad);
-   }
-
-   /// Add H(curl) GridFunction, the exact solution and possibly its curl
-   void AddHcurlGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
-                             VectorCoefficient *curl=nullptr)
-   {
-      AddGf(gf, vector_u, curl, nullptr);
-   }
-
-   /// Add H(div) GridFunction, the exact solution and possibly its div
-   void AddHdivGridFunction(GridFunction *gf, VectorCoefficient *vector_u,
-                            Coefficient *div=nullptr)
-   {
-      AddGf(gf,vector_u, nullptr, div);
-   }
-
-   /// Get the L2 error at step n
-   double GetL2Error(int n)
-   {
-      MFEM_VERIFY( n <= counter,"Step out of bounds")
-      return L2Errors[n];
-   }
-
-   /// Get all L2 errors
-   void GetL2Errors(Array<double> & L2Errors_)
-   {
-      L2Errors_ = L2Errors;
-   }
-
-   /// Get the Grad/Curl/Div error at step n
-   double GetDError(int n)
-   {
-      MFEM_VERIFY(n <= dcounter,"Step out of bounds")
-      return DErrors[n];
-   }
-
-   /// Get all Grad/Curl/Div errors
-   void GetDErrors(Array<double> & DErrors_)
-   {
-      DErrors_ = DErrors;
-   }
-
-   /// Get the DGFaceJumps error at step n
-   double GetDGFaceJumpsError(int n)
-   {
-      MFEM_VERIFY(n<= fcounter,"Step out of bounds")
-      return DGFaceErrors[n];
-   }
-
-   /// Get all DGFaceJumps errors
-   void GetDGFaceJumpsErrors(Array<double> & DGFaceErrors_)
-   {
-      DGFaceErrors_ = DGFaceErrors;
-   }
-
-   /// Print rates and errors
-   void Print(bool relative = false, std::ostream &out = mfem::out);
-};
-
-} // namespace mfem
-
-#endif // MFEM_CONVERGENCE
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tom Stitt	0e8625cf89	add optional MemoryType to the SparseMatrix copy-ctor	2020-07-24 09:13:21 -07:00
Tom Stitt	3f7a432e74	Cherry-picking changes on top of our current hash. There are things that hurt performance in laster mfem@master commits, not sure what yet. This includes: - feature/artv3/cusparse-Spmv - feature/tomstitt/temp-mem-type - patches to tmop.cpp and tmop_tools.cpp to address memory issues - patch to mem_manager/device to use mfem's default allcator instead of the host umpire one	2020-07-22 15:51:04 -07:00
Tom Stitt	a57fd02a4c	Use flattened FORALL instead of FORALL_2D.	2020-06-23 12:11:33 -07:00
Tom Stitt	2ae97ff2da	Converts PADiffusionSetup3D and QuadratureInterpolator::Eval3D kernels from 1 element per thread to 1 qpt/dof per thread for better performance when offloading (there are not enough units of work with 1 element/thread)	2020-06-22 14:42:48 -07:00
Arturo Vargas	2b712207c6	fix merge conflicts	2020-06-22 14:29:28 -07:00
Arturo Vargas	2930c1477f	promove temp vector class member	2020-06-18 13:35:11 -07:00
Arturo Vargas	b3ee631aa6	tmop_tools	2020-05-14 14:33:32 -07:00
Arturo Vargas	daf2fdecec	use tmop_tools from master	2020-05-14 14:24:52 -07:00
Arturo Vargas	d3a0d0a181	Merge branch 'master' into feature/apps/mesh-helper-tmop-ea	2020-05-14 14:19:13 -07:00
Arturo Vargas	07853b9c62	add missing host reads to tmop	2020-05-09 21:41:26 -07:00
Arturo Vargas	49a31c0cf7	Merge branch 'yohann/mass-emat' into feature/apps/mesh-helper-tmop-ea	2020-05-01 10:17:42 -07:00
Arturo Vargas	ba9b251007	Merge branch 'master' into feature/apps/mesh-helper-tmop-ea	2020-05-01 10:13:46 -07:00
Arturo Vargas	5ce2fa9ab9	fix free	2020-04-30 18:06:29 -07:00
Arturo Vargas	78c93de6ce	Merge branch 'gpu/artv3/batch-lu' into feature/apps/mesh-helper-tmop-ea	2020-04-30 13:09:53 -07:00
Arturo Vargas	3d4aa157cb	testing super code	2020-04-28 16:54:23 -07:00
Tomov	a7f1c177c5	Fixed a parallel communication bug in tmop's AdvectorCG.	2020-03-30 22:29:51 -07:00
Arturo Vargas	2d4e3cf77e	minor helper functions	2020-03-24 14:11:37 -07:00