updting io benchmark

Merge branch 'PCFinalProject' of github.com:mfem/mfem into PCFinalProject
adding io benchmark
2020-05-05 15:44:44 -04:00 · 2020-05-04 11:14:22 -07:00 · 2020-05-04 11:13:42 -07:00 · 2020-05-04 12:42:18 -04:00 · 2020-05-04 12:42:13 -04:00 · 2020-05-04 09:40:57 -07:00
396 changed files with 19270 additions and 71737 deletions
@@ -15,10 +15,8 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

-# Install METIS, use a mirror because the original source server is not always
-# up. Original url:
-#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
- ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
+# Install METIS
+- ps: Start-FileDownload 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
 - ps: ( get-content "GKlib\gk_arch.h") | % { If ($_.ReadCount -ge 52) {$_ -replace "#ifdef __MSC__","#ifdef DISABLE_THIS_ANCIENT_MSC_CHECK"} Else {$_} } | set-content "GKlib\gk_arch.h"
@@ -28,17 +26,17 @@ install:
 - cd ..

 # Install hypre
- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
- 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
- cd hypre-2.19.0/src
- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/V2-10-0b.tar.gz'
+- 7z x V2-10-0b.tar.gz -so | 7z x -si -ttar > nul
+- cd hypre-2-10-0b
+- cmake -H. -Bbuild -DHYPRE_USING_FEI=OFF -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
- cd ../..
+- cd ..

 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_LIBRARIES=%cd%\hypre-2-10-0b\hypre\lib\HYPRE.lib -DHYPRE_INCLUDE_DIRS=%cd%\hypre-2-10-0b\hypre\include -DHYPRE_VERSION=21000 -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -1,61 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 30
-
-# Number of days of inactivity before an Issue or Pull Request with the stale
-# label is closed.  Set to false to disable. If disabled, issues still need to
-# be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale.
-# Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set
-# to `[]` to disable
-exemptLabels:
-  - bug
-  - WIP
-  - ready-for-review
-  - in-review
-  - in-next
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking an issue as stale
-staleLabel: stale
-
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  :warning: This issue or PR has been automatically marked as stale because it has not
-  had any activity in the last month. *If no activity occurs in the next week, it will 
-  be automatically closed.* Thank you for your contributions.
-
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-limitPerRun: 30
-
-# Limit to only `issues` or `pulls`
-# only: issues
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed
@@ -29,8 +29,6 @@ config/sample-runs-build.log
 doc/CodeDocumentation.conf
 doc/CodeDocumentation.html
 doc/CodeDocumentation
-doc/undoc.log
-doc/warnings.log

 # Temporary files created by the tests.
 *.stderr
@@ -102,56 +100,11 @@ examples/ex25.mesh
 examples/ex25-*.gf
 examples/ex25p-*.*

-examples/amgx/ex1
-examples/amgx/ex1p
-examples/amgx/.logamgx
-examples/amgx/refined.mesh
-examples/amgx/sol.gf
-examples/amgx/mesh.*
-examples/amgx/sol.*
-
-examples/gingko/ex1
-examples/gingko/refined.mesh
-examples/gingko/sol.gf
-examples/gingko/mesh.*
-examples/gingko/sol.*
-
-examples/hiop/ex9
-examples/hiop/ex9p
-examples/hiop/ex9.mesh
-examples/hiop/ex9-mesh.*
-examples/hiop/ex9-init.*
-examples/hiop/ex9-final.*
-
-examples/petsc/ex[1-69]p
-examples/petsc/ex1[0-1]p
-examples/petsc/mesh.*
-examples/petsc/sol.*
-examples/petsc/sol_p.*
-examples/petsc/sol_u.*
-examples/petsc/Example5*
-examples/petsc/ex9.mesh
-examples/petsc/ex9-mesh.*
-examples/petsc/ex9-init.*
-examples/petsc/ex9-final.*
-examples/petsc/Example9*
-examples/petsc/deformed.*
-examples/petsc/velocity.*
-examples/petsc/elastic_energy.*
-examples/petsc/mode_*
-
-examples/pumi/ex1
-examples/pumi/ex[126]p
-examples/pumi/refined.mesh
-examples/pumi/sol.gf
-examples/pumi/mesh.*
-examples/pumi/sol.*
-examples/pumi/displaced.mesh
-
 examples/sundials/ex9
 examples/sundials/ex1[06]
 examples/sundials/ex9p
 examples/sundials/ex1[06]p
+
 examples/sundials/ex9.mesh
 examples/sundials/ex9-mesh.*
 examples/sundials/ex9-init.*
@@ -166,27 +119,47 @@ examples/sundials/ex16-init.*
 examples/sundials/ex16-final.*
 examples/sundials/Example16*

-examples/superlu/ex1p
-examples/superlu/mesh.*
-examples/superlu/sol.*
+examples/petsc/ex[1-69]p
+examples/petsc/ex10p

-miniapps/adjoint/cvsRoberts_ASAi_dns
-miniapps/adjoint/adjoint_advection_diffusion
+examples/petsc/mesh.*
+examples/petsc/sol.*
+examples/petsc/sol_p.*
+examples/petsc/sol_u.*
+examples/petsc/Example5*
+examples/petsc/ex9.mesh
+examples/petsc/ex9-mesh.*
+examples/petsc/ex9-init.*
+examples/petsc/ex9-final.*
+examples/petsc/Example9*
+examples/petsc/deformed.*
+examples/petsc/velocity.*
+examples/petsc/elastic_energy.*
+
+examples/pumi/ex1
+examples/pumi/ex[126]p
+
+examples/hiop/ex9.mesh
+examples/hiop/ex9-mesh.*
+examples/hiop/ex9-init.*
+examples/hiop/ex9-final.*
+
+examples/pumi/refined.mesh
+examples/pumi/sol.gf
+examples/pumi/mesh.*
+examples/pumi/sol.*
+examples/pumi/displaced.mesh

 miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
+
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_*

-miniapps/gslib/field-diff
-miniapps/gslib/field-interp
-miniapps/gslib/findpts
-miniapps/gslib/pfindpts
-
 miniapps/meshing/mobius-strip
 miniapps/meshing/klein-bottle
 miniapps/meshing/toroid
@@ -194,12 +167,11 @@ miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
-miniapps/meshing/trimmer
 miniapps/meshing/mesh-optimizer
 miniapps/meshing/pmesh-optimizer
 miniapps/meshing/minimal-surface
 miniapps/meshing/pminimal-surface
-miniapps/meshing/polar-nc
+
 miniapps/meshing/mobius-strip.mesh
 miniapps/meshing/klein-bottle.mesh
 miniapps/meshing/toroid-*.mesh
@@ -208,30 +180,12 @@ miniapps/meshing/mesh-explorer.mesh
 miniapps/meshing/partitioning.txt
 miniapps/meshing/shaper.mesh
 miniapps/meshing/extruder.mesh
-miniapps/meshing/trimmer.mesh
 miniapps/meshing/optimized*
 miniapps/meshing/perturbed*
-miniapps/meshing/polar-nc.mesh
-
-miniapps/navier/navier_mms
-miniapps/navier/navier_kovasznay
-miniapps/navier/navier_tgv
-miniapps/navier/navier_shear
-miniapps/navier/navier_3dfoc
-miniapps/navier/tgv_out*.txt
-miniapps/navier/*_output
-
-miniapps/nurbs/nurbs_ex1
-miniapps/nurbs/nurbs_ex1p
-miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/refined.mesh
-miniapps/nurbs/mesh.*
-miniapps/nurbs/sol.*
-miniapps/nurbs/mode_*
-miniapps/nurbs/Example1*

 miniapps/performance/ex1
 miniapps/performance/ex1p
+
 miniapps/performance/refined.mesh
 miniapps/performance/mesh.*
 miniapps/performance/sol.*
@@ -249,6 +203,7 @@ miniapps/toys/rubik
 miniapps/toys/snake
 miniapps/toys/lissajous
 miniapps/toys/mondrian
+
 miniapps/toys/snake-init.mesh
 miniapps/toys/snake-user.mesh
 miniapps/toys/snake-joined.mesh
@@ -263,23 +218,38 @@ miniapps/toys/lissajous.mesh
 miniapps/toys/lissajous.gf
 miniapps/toys/mondrian.mesh

+miniapps/nurbs/nurbs_ex1
+miniapps/nurbs/nurbs_ex1p
+miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/refined.mesh
+miniapps/nurbs/mesh.*
+miniapps/nurbs/sol.*
+miniapps/nurbs/mode_*
+miniapps/nurbs/Example1*
+
+miniapps/gslib/field-diff
+miniapps/gslib/findpts
+miniapps/gslib/pfindpts
+
+miniapps/navier/navier_mms
+miniapps/navier/navier_kovasznay
+miniapps/navier/navier_tgv
+miniapps/navier/navier_shear
+miniapps/navier/navier_3dfoc
+miniapps/navier/tgv_out*.txt
+miniapps/navier/*_output
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
 tests/unit/punit_tests
 tests/unit/sedov_tests_*
 tests/unit/psedov_tests_*
-tests/unit/ceed_tests

 # Test script output
 tests/scripts/*.err
 tests/scripts/*.out
 tests/scripts/*.msg

-# Other tests
-tests/convergence/rates
-tests/convergence/prates
-tests/par-mesh-format/ex1p
-
 # VPATH builds
 build-*/*
@@ -71,8 +71,6 @@ stages:
  - build
  - test
  - deallocate
-  - lassen_build
-  - lassen_test
  - baseline_check
  - baseline_publish

@@ -81,11 +79,7 @@ stages:
 # TODO: updating tests and tpls is not necessary anymore since pipelines are
 # now using unique directories so repo are never shared with another pipeline.
 # This is not memory efficient (we keep a lot of data), hence this reminder.
-# Setup
-setup:
-  tags:
-    - shell
-    - quartz
+.setup:
  stage: setup
  variables:
    GIT_STRATEGY: none
@@ -106,15 +100,6 @@ setup:
  before_script:
    - module load gcc/6.1.0

-# On lassen
-.with_gcc_8_3_1:
-  variables:
-    TOOLCHAIN: gcc_8_3_1
-    CXX: g++
-    CC: gcc
-  before_script:
-    - module load gcc/8.3.1
-
 .with_gcc_4_9_3:
  variables:
    TOOLCHAIN: gcc_4_9_3
@@ -305,4 +290,3 @@ setup:
 # The list on jobs is defined in machine-specific files.
 include:
  - local: .gitlab/quartz.yml
-  - local: .gitlab/lassen.yml
@@ -1,57 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# GitLab pipelines configurations for the Lassen machine at LLNL
-
-.on_lassen:
-  tags:
-    - shell
-    - lassen
-  variables:
-    PLAT: lassen
-
-# Build MFEM
-build_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 5 -q pdebug make -j cuda CUDA_ARCH=sm_70
-
-build_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  needs: [setup]
-  stage: lassen_build
-  script:
-    - mkdir -p ${BUILD_PATH}
-    - cp -r ${CI_PROJECT_DIR} ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 5 -q pdebug make -j cuda MFEM_DEBUG="YES" CPPFLAGS=-O2 CUDA_ARCH=sm_70
-
-# Sanity check
-sanitycheck_mfem_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser
-    - lalloc 1 -W 15 -q pdebug make -j test
-
-sanitycheck_mfem_debug_ser_lassen:
-  extends: [.with_gcc_8_3_1, .on_lassen]
-  stage: lassen_test
-  needs: [build_mfem_debug_ser_lassen]
-  script:
-    - cd ${BUILD_PATH}/${CI_PROJECT_NAME}_lassen_ser_debug
-    - lalloc 1 -W 30 -q pdebug make -j test
@@ -22,6 +22,10 @@
    MAKE_PAR: 6
    BASELINE_PAR: 18

+# Setup
+setup_quartz:
+  extends: [.setup, .on_quartz]
+
 # Allocate
 allocate_quartz:
  variables:
@@ -11,20 +11,13 @@

 language: cpp

-os: linux
-dist: bionic
+sudo: false

 stages:
  - checks
  - tests
  - optional

-env:
-  global:
-    - HYPRE_ARCHIVE=v2.19.0.tar.gz
-      HYPRE_URL=https://github.com/hypre-space/hypre/archive/$HYPRE_ARCHIVE
-      HYPRE_TOP_DIR=hypre-2.19.0
-
 jobs:
  include:

@@ -37,7 +30,6 @@ jobs:

    - stage: checks
      os: linux
-      dist: xenial
      name: "code-style"
      addons:
        apt:
@@ -56,6 +48,9 @@ jobs:
          packages:
            - doxygen
            - graphviz
+            - mpich
+            - libmpich-dev
+      env: MPI=YES
      script:
        - cd ${TRAVIS_BUILD_DIR}
        - cd tests/scripts
@@ -70,24 +65,13 @@ jobs:
            - mpich
            - libmpich-dev
      env: MPI=YES
-      before_script:
+      script:
        - cd ${TRAVIS_BUILD_DIR}
-        - mpicxx -v
        - make config MFEM_USE_MPI=YES MFEM_MPI_NP=2
        - make all -j3
        - make test-noclean
-      script:
        - cd tests/scripts
        - ./runtest gitignore
-      cache:
-        ccache: true
-        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
-          - $TRAVIS_BUILD_DIR/../metis-4.0
-      before_cache:
-        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
-          mv libmetis.a Lib ..; rm -rf * ; mv ../libmetis.a ../Lib .;
-          rm -f Lib/*.{c,o}

    # ========================
    #  Optional Checks/Tests
@@ -96,7 +80,6 @@ jobs:

    - stage: optional
      name: "branch-history"
-      if: branch != next
      # need full git history for the binary/big files check
      git:
        depth: false
@@ -125,8 +108,6 @@ jobs:
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true

    - os: linux
      compiler: gcc
@@ -135,8 +116,6 @@ jobs:
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true

    - os: linux
      compiler: gcc
@@ -160,9 +139,9 @@ jobs:
           MFEM_TEST_TARGET=check
           NPROCS=2
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -191,9 +170,9 @@ jobs:
           MFEM_TEST_TARGET=test
           NPROCS=2
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -216,16 +195,16 @@ jobs:
        - cd ${TRAVIS_BUILD_DIR}/build
        - cmake ..
             -DMFEM_USE_MPI=ON
-             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../$HYPRE_TOP_DIR/src/hypre
+             -DHYPRE_DIR=${TRAVIS_BUILD_DIR}/../hypre-2.10.0b/src/hypre
             -DMFEM_MPI_NP=$NPROCS
        - make -j3 mfem examples
        - cd ${TRAVIS_BUILD_DIR}/build/tests/unit
        - make -j3
        - ctest --output-on-failure
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
      before_cache:
        - cd $TRAVIS_BUILD_DIR/../metis-4.0;
@@ -241,43 +220,27 @@ jobs:
    # - parallel

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Serial + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=YES
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=check
-      cache:
-        ccache: true

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Serial"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=NO
           MPI=NO
           CODECOV=NO
           MFEM_TEST_TARGET=test
-      cache:
-        ccache: true

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Parallel + Debug"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=YES
           MPI=YES
           CODECOV=NO
@@ -285,9 +248,9 @@ jobs:
           NPROCS=4
           TMPDIR=/tmp
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -296,13 +259,9 @@ jobs:
          rm -f Lib/*.{c,o}

    - os: osx
-      osx_image: xcode11.2
+      # osx_image: xcode7.3
      compiler: clang
      name: "Mac: Parallel"
-      addons:
-        homebrew:
-          packages:
-            - ccache
      env: DEBUG=NO
           MPI=YES
           CODECOV=YES
@@ -310,9 +269,9 @@ jobs:
           NPROCS=4
           TMPDIR=/tmp
      cache:
-        ccache: true
        directories:
-          - $TRAVIS_BUILD_DIR/../$HYPRE_TOP_DIR/src/hypre
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/lib
+          - $TRAVIS_BUILD_DIR/../hypre-2.10.0b/src/hypre/include
          - $TRAVIS_BUILD_DIR/../metis-4.0
          - $HOME/local-cached
      before_cache:
@@ -326,19 +285,14 @@ before_install:
   #      brew install open-mpi;
   #   fi

-   # Disable ccache while building dependencies that are cached:
-   - echo "before \$PATH = $PATH";
-     export PATH=${PATH//\/usr\/lib\/ccache:/};
-     echo "after  \$PATH = $PATH"
-
-   # On Mac OS X, build and cache OpenMPI 2.1.6:
+   # On Mac OS X, build and cache OpenMPI 2.1.1:
   - if [ $TRAVIS_OS_NAME == "osx" ] && [ $MPI == "YES" ]; then
        if [ ! -e $HOME/local-cached/bin/mpicc ]; then
           mkdir -p $HOME/builds && cd $HOME/builds &&
-           wget https://download.open-mpi.org/release/open-mpi/v2.1/openmpi-2.1.6.tar.bz2 &&
-           tar jxf openmpi-2.1.6.tar.bz2 &&
+           wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.bz2 &&
+           tar jxf openmpi-2.1.1.tar.bz2 &&
           mkdir openmpi-build && cd openmpi-build &&
-           ../openmpi-2.1.6/configure --prefix=$HOME/local-cached &&
+           ../openmpi-2.1.1/configure --prefix=$HOME/local-cached &&
           make -j3 all && make install;
        fi;
        PATH=$HOME/local-cached/bin:$PATH;
@@ -383,28 +337,26 @@ install:

   # hypre
   - if [ $MPI == "YES" ]; then
-        if [ ! -e $HYPRE_TOP_DIR/src/hypre/lib/libHYPRE.a ]; then
-           wget $HYPRE_URL;
-           rm -rf $HYPRE_TOP_DIR;
-           tar xvzf $HYPRE_ARCHIVE;
-           cd $HYPRE_TOP_DIR/src;
-           ./configure --disable-fortran CC=mpicc CXX=mpic++;
+        if [ ! -e hypre-2.10.0b/src/hypre/lib/libHYPRE.a ]; then
+           wget https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz --no-check-certificate;
+           rm -rf hypre-2.10.0b;
+           tar xvzf hypre-2.10.0b.tar.gz;
+           cd hypre-2.10.0b/src;
+           ./configure --disable-fortran --without-fei CC=mpicc CXX=mpic++;
           make -j3;
           cd ../..;
        else
-            echo "Reusing cached $HYPRE_TOP_DIR/";
+            echo "Reusing cached hypre-2.10.0b/";
        fi;
-        ln -s $HYPRE_TOP_DIR hypre;
+        ln -s hypre-2.10.0b hypre;
     else
        echo "Serial build, not using hypre";
     fi

-   # METIS, use a mirror because the original source server is not always up.
-   # Original url:
-   #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz
+   # METIS
   - if [ $MPI == "YES" ]; then
        if [ ! -e metis-4.0/libmetis.a ]; then
-           wget https://mfem.github.io/tpls/metis-4.0.3.tar.gz;
+           wget http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz;
           tar xvzf metis-4.0.3.tar.gz;
           make -j3 -C metis-4.0.3/Lib CC="$CC" OPTFLAGS="-O2";
           rm -rf metis-4.0;
@@ -414,26 +366,12 @@ install:
        fi;
     fi

-   # Re-enable ccache on linux; enable ccache on mac os:
-   - if [ $TRAVIS_OS_NAME == "linux" ]; then
-        export PATH="/usr/lib/ccache:$PATH";
-     else
-        if [ $TRAVIS_OS_NAME == "osx" ]; then
-           export PATH="/usr/local/opt/ccache/libexec:$PATH";
-        fi;
-     fi
-
-   - printf "which \$CC  = "; which $CC;
-     printf "which \$CXX = "; which $CXX
-
 script:
   # Compiler
   - if [ $MPI == "YES" ]; then
        export MYCXX=mpic++;
-        export MAKE_CXX_FLAG=MPICXX=$MYCXX;
     else
        export MYCXX="$CXX";
-        export MAKE_CXX_FLAG=CXX=$MYCXX;
     fi

   # Print the compiler version
@@ -446,12 +384,12 @@ script:
     if [ "$CODECOV" == "YES" ]; then
        CPPFLAGS="--coverage -g";
     fi;
-     if [ "$TRAVIS_OS_NAME" != "linux" ] || [ "$DEBUG" == "YES" ]; then
-        CPPFLAGS+=" -pedantic -Wall -Werror";
+     if [ "$CXX" == "clang++" ]; then
+        export MFEM_PERF_SW=clang;
     fi

   # Configure the library
-   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG $MAKE_CXX_FLAG
+   - make config MFEM_USE_MPI=$MPI MFEM_DEBUG=$DEBUG MFEM_CXX="$MYCXX"
        MFEM_MPI_NP=$NPROCS CPPFLAGS="$CPPFLAGS"
   # Show the configuration
   - make info
@@ -11,101 +11,22 @@
 Version 4.1.1 (development)
 ===========================

-libCEED integration improvements
--------------------------------
- Add support for all types of (scalar) Coefficient.
-
- Add support for VectorMassIntegrator and VectorDiffusionIntegrator.
-
- Add support for AssemblyLevel::NONE for MassIntegrator, DiffusionIntegrator,
-  VectorMassIntegrator, and VectorDiffusionIntegrator. This level of assembly
-  fully applies on the fly the finite element operator.
-
 Meshing improvements
 --------------------
 - The graph linear ordering library Gecko, previously an external dependency, is
  now included directly in MFEM. As a result, Mesh::GetGeckoElementOrdering is
  always available. The interface has also been improved, see for example the
-  Mesh Explorer miniapp.
-
- Improved Gmsh reader (version 2.2), which now supports both high-order and
-  periodic meshes. Segments, triangles, quadrilaterals, and tetrahedra are
-  supported up to order 10. Wedges and hexahedra are supported up to order 9.
-  For sample periodic meshes, see the periodic*.msh files in the data directory.
+  mesh-explorer miniapp.

 - Added support for finite difference-based gradient and Hessian approximation
  in the TMOP mesh optimization algorithms. This improves the accuracy of the
  Hessian for r-adaptivity using discrete fields, and allows use of skewness
  and orientation based metrics.

- Added support for r-adaptivity with more than one discrete field. This allows
-  the user to specify different discrete functions for controlling the
-  size, aspect-ratio, orientation, and skew of elements in the mesh.
-
- Added TMOP capability for approximate tangential mesh relaxation. Added
-  support and examples for using TMOP on mixed meshes.
-
- Added complete action of the TMOP Integrator to account for the spatial
-  derivatives of discrete and analytic targets.
-
- Added support for initialization of (serial) non-conforming meshes. Hanging
-  nodes can be marked with Mesh::AddVertexParents when building the mesh with
-  the "init" constructor. The usage is demonstrated in a new meshing miniapp
-  (polar-nc) which generates meshes that are non-conforming from the start.
-
-Performance improvements
------------------------
- Added support for explicit vectorization in the high-performance templated
-  code, which can now take advantage of specific intrinsics classes on the
-  following architectures:
-    - x86 (SSE/AVX/AVX2/AVX512),
-    - Power8 & Power9 (VSX),
-    - BG/Q (QPX).
-  These are disabled by default, and can be enabled with MFEM_USE_SIMD=YES.
-  See the new file linalg/simd.hpp and the new directory linalg/simd.
-
 Improved GPU capabilities
 -------------------------
- Added a new solver class for simple integration with NVIDIA's multigrid
-  library, AmgX. The AmgX class is designed to work as a standalone solver or
-  preconditioner for existing MFEM solvers. It uses MFEM's sparse matrix format
-  for serial runs and the HypreParMatrix format for parallel runs.
-
-  The new solver may be configured to run with one GPU per MPI rank or with more
-  MPI ranks than GPUs. In the latter case, matrices and vectors are consolidated
-  to ranks communicating with the GPUs and the solution is then broadcasted.
-
-  Although CUDA is required to build, the AmgX support is compatible with the
-  MFEM CPU device configuration. The examples/amgx folder illustrates how to
-  integrate AmgX in existing MFEM applications.
-
-  The AmgX solver class is partially based on: "AmgXWrapper: An interface
-  between PETSc and the NVIDIA AmgX library", by Pi-Yueh Chuang and Lorena
-  A. Barba, doi:10.21105/joss.00280.
-
 - Added support for Chebyshev accelerated polynomial smoother on GPU.

- Optimized AMD/HIP kernel support.
-
- Enabled HIP support in the libCEED integration, which is now available via the
-  "ceed-hip" device backend.
-
- Added a Full Assembly mode compatible with Device kernel execution. This
-  assembly level builds on top of the current Element Assembly kernels to
-  compute a global sparse matrix. All integrators supported by element assembly
-  are also supported by full assembly. See the '-fa' option in Example 9.
-
- Added CUDA support for sparse matrix-vector multiplication with cuSPARSE.
-
- Added support for BlockOperator on GPU. See the updated Example 5.
-
- Added partial assembly and GPU support for complex operators, including the
-  classes ComplexOperator, [Par]ComplexGridFunction, [Par]ComplexLinearForm, and
-  [Par]SesquilinearForm. See the updated Example 22.
-
- Added CUDA support for SUNDIALS ODE integrators. See the updated SUNDIALS
-  modification of Example 9/9p.
-
 Discretization improvements
 ---------------------------
 - Added support for matrix-free interpolation and restriction operators between
@@ -114,43 +35,6 @@ Discretization improvements

 - Added support for simplices in GSLIB-FindPoints.

- Added support for H1 and L2 element matrix assembly in the mass, convection,
-  diffusion, transpose, and the face DG trace integrators. This is compatible
-  with GPU device execution and is illustrated in Example 9/9p, see the option
-  '-ea'. When enabled, this level of assembly stores independent dense matrices
-  for the elements, and independent dense matrices for the faces in the DG case.
-
- Added new partial assembly kernels for H(div) bilinear forms, as well as
-  VectorFEDivergenceIntegrator.
-
- Improved the documentation of the GridFunction GetValue and GetVectorValue
-  methods. Expanded the GetValue and GetVectorValue methods which accept an
-  ElementTransformation argument to support evaluation on boundary elements
-  and, in the continuous field case, arbitrary mesh edges and faces.
-
- Added new coefficient and vector coefficient classes for QuadratureFunctions.
-  Additionally, new LinearForm integrators were also added which make use of
-  these new QuadratureFunction coefficient classes.
-
- Non-conforming meshes are now supported with block nonlinear forms. See the
-  updated Example 19/19p.
-
- Added support face integrals on the boundaries of NURBS meshes.
-
- Added support for interpolation of functions in L2, H(div) and H(curl)
-  spaces using GSLIB-FindPoints.
-
- Added support for computing asymptotic error estimates and convergence rates
-  for the whole de Rham sequence based on the new class ConvergenceStudy and new
-  member methods in GridFunction and ParGridFunction. See the rates.cpp file in
-  the tests/convergence directory for sample usage.
-
- The C-function based coefficient classes (FunctionCoefficient,
-  VectorFunctionCoefficient, and MatrixFunctionCoefficient) now use the more
-  general std::function class template. This allows the classes to be backward
-  compatible (i.e. they can still work with C-functions) and, in addition,
-  support any "callable", e.g. lambda functions.
-
 Linear and nonlinear solvers
 ----------------------------
 - Added power method to iteratively estimate the largest eigenvalue and the
@@ -159,38 +43,6 @@ Linear and nonlinear solvers
 - Added initial support for h- and p-multigrid solvers and preconditioners for
  matrix-based and matrix-free discretizations with basic GPU capability.

- Added wrappers for Hypre's flexible GMRES solver and the new parallel ILU
-  preconditioner. The latter requires hypre version 2.19.0 or later.
-
- Added a new IterativeSolverMonitor class that allows to monitor the residual
-  and solution during the solving process of an IterativeSolver after every
-  iteration.
-
- Added support for the CVODES package in SUNDIALS which provides ODE
-  solvers with sensitivity analysis capabilities. See the CVODESSolver
-  class and the new adjoint miniapps below.
-
- Block arrays of parallel matrices can now be merged into a single parallel
-  matrix with the function HypreParMatrixFromBlocks. This could be useful for
-  solving block systems with parallel direct solvers such as STRUMPACK.
-
- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
-  solver can work with non-SPD preconditioner B.
-
- Added support for the SLEPc eigensolver package.
-
- Added partially assembled convergent diagonal preconditioner for adaptively
-  refined meshes (i.e. non-conforming finite element spaces), see Example 6/6p.
-
- Upgraded SuperLU interface to use SuperLU_DIST 6.3.1. Added a simple SuperLU
-  example in the new directory examples/superlu.
-
- Extended the KINSOL (SUNDIALS) nonlinear solver interface to support the
-  Jacobian-free Newton-Krylov method. A usage example is shown in Example 10p.
-
- Added an interface to the MKL CPardiso solver -- an MPI-parallel sparse direct
-  solver developed by Intel. See Example 11p for an illustration of its usage.
-
 New and updated examples and miniapps
 -------------------------------------
 - Added a new example, Example 25/25p, to demonstrate the use of a Perfectly
@@ -200,25 +52,11 @@ New and updated examples and miniapps
 - Added a new Example 26/26p to demonstrate the construction of a matrix-free
  geometric and p-multigrid preconditioner for the Laplace problem.

- Added a new example, Example 27/27p, to demonstrate the enforcement of various
-  boundary conditions with the Laplace operator. The example shows the procedure
-  for applying Dirichlet, Neumann (both homogeneous and inhomogeneous), Robin,
-  and periodic boundary conditions with either H1 or DG discretizations.
-
- Added a new miniapp, Navier, that solves the time-dependent Navier-Stokes
-  equations of incompressible fluid dynamics. See the miniapps/navier directory
-  for more details.
-
- Added a new miniapps/adjoint directory with two miniapps demonstrating how to
-  solve adjoint problems in MFEM using the CVODES package in SUNDIALS. Both of
-  these miniapps require the MFEM_USE_SUNDIALS configuration option.
-  * The cvsRoberts_ASAi_dns miniapp solves a backward adjoint problem for a
-    system of ODEs, evaluating both forward and adjoint quadratures in serial.
-  * The adjoint_advection_diffusion miniapp solves a backward adjoint problem
-    for an advection diffusion PDE, evaluating adjoint quadratures in parallel.
-
- Ported Example 11p to SLEPc, to demonstrate solving the Laplace eigenvalue
-  equation with the shift-and-invert spectral transformation method.
+- Added a new example, Example 27/27p, to demonstrate the enforcement of
+  various boundary conditions with the Laplace operator.  The example shows the
+  procedures for applying Dirichlet, Neumann (both homogeneous and
+  inhomogeneous), Robin, and periodic boundary conditions with either H1 or DG
+  discretizations.

 - Added a simple meshing miniapp, Twist, which demonstrates MFEM's strategy of
  stitching together opposite surfaces of a mesh to create a topologically
@@ -227,71 +65,23 @@ New and updated examples and miniapps
 - Added a new meshing miniapp, Minimal Surface, which solves Plateau's problem:
  the Dirichlet problem for the minimal surface equation.

- Added a new meshing miniapp, Polar NC, which demonstrates the construction of
-  polar non-conforming meshes.
-
- Added partial assembly support to Example 4/4p and Example 5/5p, with diagonal
-  preconditioning.
-
- Added full assembly support in Example 9/9p.
-
- Added a new test problem in Example 24/24p, demonstrating a mixed bilinear
-  form for H1, H(curl), H(div) and L_2, with partial assembly support.
-
- Added weak Dirichlet boundary conditions (Nitsche) to the NURBS miniapp.
-
- Added a simple mesh editing miniapp, Trimmer, which trims away portions of a
-  mesh based on element attributes. Any newly exposed boundary elements are
-  assigned attribute numbers related to the trimmed element attributes.
-
- Added a new miniapp (field-interp) that demonstrates transfer of grid function
-  between different meshes using GSLIB-FindPoints.
-
- Added diagonal preconditioner in Example 6/6p for partial assembly with AMR.
-
- Added device support in Example 5/5p.
-
- Added partial assembly and device support to Example 22/22p, with diagonal
-  preconditioning.
-
- Added the option to plot a function in Mesh Explorer.
-
 Improved testing
 ----------------
- Upgraded the Catch unit test framework from version 1.6.1 to version 2.13.0.
-
 - Added a GitLab pipeline that automates PR testing on supercomputing systems
  and Linux clusters at Lawrence Livermore National Lab (LLNL). This can be
  triggered only by LLNL developers, see .gitlab-ci.yml, the .gitlab directory
  and the updated CONTRIBUTING.md file.

- Add tests for the libCEED integration in MFEM.
-
- Added testing of the parallel mesh format in tests/par-mesh-format.
-
 Miscellaneous
 -------------
+- In SLISolver, changed the residual inner product from (Br,r) to (Br,Br) so the
+  solver can work with non-SPD preconditioner B.
+
 - Added support for ADIOS2 for parallel I/O with ParaView visualization. The
  classes adios2stream and ADIOS2DataCollection are introduced in mfem as the
  interfaces to generate ADIOS2 Binary Pack (BP4) directory datasets for the
-  entire spatial and temporal node data. Cell centered data is accessible by
-  ADIOS2 data readers (e.g. Python), but currently not yet implement as of
-  ParaView v5.8.1. In addition, ADIOS2 allows for setting a user-defined number
-  of data substreams/subfiles at scale. See examples 5, 9, 12, 16.
-
- Added VTU output of boundary elements and attributes and parallel VTU (PVTU)
-  output of parallel meshes for visualization using ParaView.
-
- The integration order used in the ComputeLpError and ComputeElementLpError
-  methods of class GridFunction has been increased.
-
- Various other simplifications, extensions, and bugfixes in the code.
-
- Renamed "Backend::DEBUG" to "Backend::DEBUG_DEVICE" to avoid conflicts,
-  as DEBUG is sometimes used as a macro.
-
- Change the IntegrationRule inside VectorDiffusionIntegrator to use the same
-  quadrature as DiffusionIntegrator.
+  entire spatial and temporal data. In addition, ADIOS2 allows for setting a
+  user-defined number of data substreams/subfiles. See examples 5, 9, 12, 16.


 Version 4.1, released on March 10, 2020
@@ -89,38 +89,8 @@ enable_language(CXX)
 if (MFEM_USE_CUDA)
   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-   # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
-   if (NOT CMAKE_CUDA_HOST_COMPILER)
-      set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
-   endif()
   enable_language(CUDA)
-   set(CMAKE_CUDA_STANDARD 11)
-   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-   set(CMAKE_CUDA_EXTENSIONS OFF)
-   set(CUDA_FLAGS "--expt-extended-lambda")
-   if (CMAKE_VERSION VERSION_LESS 3.18.0)
-      set(CUDA_FLAGS "-arch=${CUDA_ARCH} ${CUDA_FLAGS}")
-   elseif (NOT CMAKE_CUDA_ARCHITECTURES)
-      string(REGEX REPLACE "^sm_" "" ARCH_NUMBER "${CUDA_ARCH}")
-      if ("${CUDA_ARCH}" STREQUAL "sm_${ARCH_NUMBER}")
-         set(CMAKE_CUDA_ARCHITECTURES "${ARCH_NUMBER}")
-      else()
-         message(FATAL_ERROR "Unknown CUDA_ARCH: ${CUDA_ARCH}")
-      endif()
-   else()
-      set(CUDA_ARCH "CMAKE_CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}")
-   endif()
   message(STATUS "Using CUDA architecture: ${CUDA_ARCH}")
-   if (CMAKE_VERSION VERSION_LESS 3.12.0)
-      # CMake versions 3.8 and 3.9 require this to work; 3.10 and 3.11 are not
-      # tested and may not actually need this (but should be ok to keep).
-      set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
-      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
-   endif()
-   set(CMAKE_CUDA_FLAGS "${CUDA_FLAGS}" CACHE STRING
-       "CUDA flags set for MFEM" FORCE)
-   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
 endif()

 if (XSDK_ENABLE_C)
@@ -179,13 +149,9 @@ if (MFEM_USE_MPI)
      message(FATAL_ERROR "PETSc version >= 3.8.0 is required")
    endif()
    set(PETSC_INCLUDE_DIRS ${PETSC_INCLUDES})
-    if (MFEM_USE_SLEPC)
-      find_package(SLEPc REQUIRED config)
-      message(STATUS "Found SLEPc version ${SLEPC_VERSION}")
-    endif()
  endif()
 else()
-  set(PKGS_NEED_MPI SUPERLU PETSC SLEPC STRUMPACK PUMI)
+  set(PKGS_NEED_MPI SUPERLU PETSC STRUMPACK PUMI)
  foreach(PKG IN LISTS PKGS_NEED_MPI)
    if (MFEM_USE_${PKG})
      message(STATUS "Disabling package ${PKG} - requires MPI")
@@ -240,14 +206,12 @@ endif()

 # SUNDIALS
 if (MFEM_USE_SUNDIALS)
-  set(SUNDIALS_COMPONENTS CVODES ARKODE KINSOL NVector_Serial)
-  if (MFEM_USE_MPI)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Parallel NVector_MPIPlusX)
+  if (NOT MFEM_USE_MPI)
+    find_package(SUNDIALS REQUIRED NVector_Serial CVODE ARKODE KINSOL)
+  else()
+    find_package(SUNDIALS REQUIRED
+      NVector_Serial NVector_Parallel NVector_ParHyp CVODE ARKODE KINSOL)
  endif()
-  if (MFEM_USE_CUDA)
-    list(APPEND SUNDIALS_COMPONENTS NVector_Cuda)
-  endif()
-  find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()

 # Mesquite
@@ -297,10 +261,6 @@ if (MFEM_USE_CEED)
  find_package(libCEED REQUIRED)
 endif()

-if (MFEM_USE_AMGX)
-  find_package(AMGX REQUIRED)
-endif()
-
 if (MFEM_USE_CONDUIT)
    find_package(Conduit REQUIRED conduit relay blueprint )
 endif()
@@ -332,6 +292,22 @@ if (MFEM_USE_HIOP)
  # find_package updates HIOP_FOUND, HIOP_INCLUDE_DIRS, HIOP_LIBRARIES
 endif()

+# CUDA
+if (MFEM_USE_CUDA)
+   set(CMAKE_CUDA_STANDARD 11)
+   set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+   set(CMAKE_CUDA_EXTENSIONS OFF)
+   set(CMAKE_CUDA_FLAGS "-arch=${CUDA_ARCH} --expt-extended-lambda"
+       CACHE STRING "CUDA flags set for MFEM" FORCE)
+   if (MFEM_USE_MPI)
+      set(CUDA_CCBIN_COMPILER ${MPI_CXX_COMPILER})
+   else()
+      set(CUDA_CCBIN_COMPILER ${CMAKE_CXX_COMPILER})
+   endif()
+   string(APPEND CMAKE_CUDA_FLAGS " -ccbin ${CUDA_CCBIN_COMPILER}")
+   set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CUDA_CCBIN_COMPILER})
+endif()
+
 # OCCA
 if (MFEM_USE_OCCA)
   find_package(OCCA REQUIRED)
@@ -352,12 +328,6 @@ if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
 endif()

-if (MFEM_USE_MKL_CPARDISO)
-  if (MFEM_USE_MPI)
-   find_package(MKL_CPARDISO REQUIRED MKL_SEQUENTIAL MKL_LP64 MKL_MPI_WRAPPER)
-  endif()
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -382,9 +352,8 @@ endif()
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
 set(MFEM_TPLS MPI_CXX OPENMP BLAS LAPACK METIS HYPRE SuiteSparse SUNDIALS PETSC
-    SLEPC MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
-    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2
-    CUSPARSE MKL_CPARDISO AMGX)
+    MESQUITE SuperLUDist STRUMPACK AXOM CONDUIT Ginkgo GNUTLS GSLIB NETCDF
+    MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE ADIOS2)
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
 set(TPL_INCLUDE_DIRS "")
@@ -98,33 +98,26 @@ The MFEM source code has the following structure:
  │   └── web
  │       └── examples
  ├── examples
-  │   ├── amgx
  │   ├── ginkgo
  │   ├── hiop
  │   ├── petsc
  │   ├── pumi
  │   └── sundials
-  |   └── superlu
  ├── fem
  │   └── libceed
  ├── general
  ├── linalg
-  │   └── simd
  ├── mesh
  ├── miniapps
-  │   ├── adjoint
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
  │   ├── meshing
-  │   ├── navier
  │   ├── nurbs
  │   ├── performance
  │   ├── tools
  │   └── toys
  └── tests
-      ├── convergence
-      ├── par-mesh-format
      ├── scripts
      ├── unit
      │   ├── ...
@@ -344,10 +344,6 @@ MFEM_USE_SUPERLU = YES/NO
   SuperLURowLocMatrix a distributed CSR matrix class needed by SuperLU. When
   enabled, this option uses the SUPERLU_* library options, see below.

-MFEM_USE_SUPERLU5 = YES/NO
-   If SuperLU functionality is enabled, use the older 5.1.0 version rather than
-   the more recent 6+ versions.
-
 MFEM_USE_STRUMPACK = YES/NO
   Enable MFEM functionality based on the STRUMPACK sparse direct solver and
   preconditioner through the STRUMPACKSolver and STRUMPACKRowLocMatrix
@@ -360,11 +356,6 @@ MFEM_USE_GINKGO = YES/NO
   https://github.com/ginkgo-project/ginkgo. When enabled, the user can use
   Ginkgo's solvers and preconditioners as shown in examples/ginkgo/.

-MFEM_USE_AMGX = YES/NO
-   Enable MFEM functionality based on the AmgX multigrid library from NVIDIA.
-   Allows the user to use SparseMatrices and HypreParMatrices to solve linear
-   systems with the routines from the AmgX library.
-
 MFEM_USE_GNUTLS = YES/NO
   Enable secure socket support in class socketstream, using the auxiliary
   GnuTLS_* classes, based on the GnuTLS library. This option may be useful in
@@ -392,10 +383,6 @@ MFEM_USE_PETSC = YES/NO
   and other features based on the PETSc package. When enabled, this option uses
   the PETSC_* library options, see below.

-MFEM_USE_SLEPC = YES/NO
-   Enable MFEM eigensolvers based on the SLEPc package. When enabled, this
-   option uses the SLEPC_* library options, see below.
-
 MFEM_USE_MPFR = YES/NO
   MPFR is a library for multiple-precision floating-point computations. This
   option enables the use of MPFR in MFEM, e.g. for precise computation of 1D
@@ -409,12 +396,6 @@ MFEM_USE_SIDRE = YES/NO
   blueprint specification. When enabled, this option requires installation of
   HDF5 (see also MFEM_USE_NETCDF), Conduit and LLNL's axom project.

-MFEM_USE_SIMD = YES/NO
-   Enables the high performance templated classes to use architecture dependent
-   SIMD intrinsics instead of the generic implementation of class AutoSIMD in
-   linalg/simd/auto.hpp. This option should be combined with suitable
-   compiler options, such as -march=native, to enable optimal vectorization.
-
 MFEM_USE_CONDUIT = YES/NO
   Enables support for converting MFEM Mesh and Grid Function objects to and
   from Conduit Mesh Blueprint Descriptions (https://github.com/LLNL/conduit/)
@@ -445,8 +426,6 @@ MFEM_USE_PUMI = YES/NO
   data management system that is capable of handling general non-manifold
   models and effectively supports automated adaptive analysis. PUMI enables
   support for parallel unstructured mesh modifications in MFEM.
-   The develop branch of PUMI repository (https://github.com/SCOREC/core)
-   should be used for most updated features.

 MFEM_USE_UMPIRE = YES/NO
   Enables support for Umpire, a resource management library that allows the
@@ -495,13 +474,6 @@ MFEM_USE_CEED = YES/NO
   library for performant high-order operator evaluation developed by the Center
   for Efficient Exascale Discretizations in the Exascale Computing Project.

-MFEM_USE_MKL_CPARDISO = YES/NO
-   Enables the interface to MKL CPardiso: the Intel MKL Parallel Direct Sparse
-   Solver for Clusters. Make sure to set the correct values for MKL_MPI_WRAPPER
-   and MKL_LIBRARY_SUBDIR as shown in defaults.mk. If you configure MFEM with
-   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
-   OpenMP capabilities are disabled at link time.
-
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -551,14 +523,11 @@ The specific libraries and their options are:
  Beginning with MFEM v3.3, SUNDIALS v2.7.0 is supported.
  Beginning with MFEM v3.3.2, SUNDIALS v3.0.0 is also supported.
  Beginning with MFEM v4.1, only SUNDIALS v5.0.0+ is supported.
-  When MFEM_USE_CUDA is enabled, only SUNDIALS v5.4.0+ is supported.
  If MFEM_USE_MPI is enabled, we expect that SUNDIALS is built with support for
  both MPI and hypre.
-  If MFEM_USE_CUDA is enabled, we expect that SUNDIALS is built with support
-  for CUDA.
  URL: http://computation.llnl.gov/projects/sundials/sundials-software
  Options: SUNDIALS_OPT, SUNDIALS_LIB.
-  Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA suppport.
+  Versions: SUNDIALS >= 5.0.0.

 - Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
  URL: http://trilinos.org/oldsite/packages/mesquite
@@ -593,11 +562,6 @@ The specific libraries and their options are:
  URL: https://ginkgo-project.github.io
  Options: GINKGO_OPT (Not used), GINKGO_LIB.

- AmgX (optional), used when MFEM_USE_AMGX = YES.
-  URL: https://github.com/NVIDIA/AMGX
-  Options: AMGX_OPT, AMGX_LIB.
-  Versions: AmgX >= 2.1, older versions may work too.
-
 - GnuTLS (optional), used when MFEM_USE_GNUTLS = YES. On most Linux systems,
  GnuTLS is available as a development package, e.g. gnutls-devel. On Mac OS X,
  one can get the library through the Homebrew package manager (http://brew.sh).
@@ -625,12 +589,6 @@ The specific libraries and their options are:
  Options: PETSC_OPT, PETSC_LIB.
  Versions: PETSc >= 3.8.0.

- SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
-  uses some of the PETSc options when compiled.
-  URL: https://slepc.upv.es/
-  Options: SLEPC_OPT, SLEPC_LIB.
-  Versions: SLEPc >= 3.8.0.
-
 - Sidre (optional), part of LLNL's axom project, used when MFEM_USE_SIDRE = YES.
  Starting with MFEM v4.1, Axom version 0.3.1 or later is required.
  URL: https://github.com/LLNL/axom
@@ -651,9 +609,8 @@ The specific libraries and their options are:

 - PUMI (optional), used when MFEM_USE_PUMI = YES.
  URL: https://scorec.rpi.edu/pumi
-       https://github.com/SCOREC/core
  Options: PUMI_OPT, PUMI_LIB.
-  Versions: PUMI >= 2.2.3.
+  Versions: PUMI >= 2.2.0.

 - HiOp (optional), used when MFEM_USE_HIOP = YES.
  URL: https://github.com/LLNL/hiop
@@ -669,11 +626,6 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.5.

- MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
-  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
-  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
-  Versions: Intel MKL >= 2020.
-
 - CUDA (optional), used when MFEM_USE_CUDA = YES.
  URL: https://developer.nvidia.com/cuda-toolkit
  Options: CUDA_CXX, CUDA_ARCH, CUDA_OPT, CUDA_LIB.
@@ -686,13 +638,14 @@ The specific libraries and their options are:
 - OCCA (optional), used when MFEM_USE_OCCA = YES.
  URL: https://libocca.org
  Options: OCCA_DIR, OCCA_OPT, OCCA_LIB.
-  Versions: OCCA >= 1.1.0.
+  Versions: OCCA >= 1.0.9.

- libCEED (optional), used when MFEM_USE_CEED = YES.
+- libCEED (optional), used when MFEM_USE_CEED = YES. Requires libCEED v0.6
+  or later version, specifically, git-hash 3d05795 or later.
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.7.
+  Versions: libCEED >= 0.6.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.1, only RAJA v0.10.0+ is supported.
@@ -836,7 +789,6 @@ MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_STRUMPACK
 MFEM_USE_GINKGO
-MFEM_USE_AMGX
 MFEM_USE_GNUTLS
 MFEM_USE_NETCDF
 MFEM_USE_MPFR
@@ -885,12 +837,11 @@ The CMake build system adds auto-detection for the following packages/libraries:

 - HYPRE
 - METIS - The option MFEM_USE_METIS_5 is auto-detected.
- - ParMETIS
 - MESQUITE
 - SuiteSparse
 - SuperLUDist, STRUMPACK
+ - ParMETIS
 - Ginkgo
- - AMGX
 - GNUTLS - Extends the built-in CMake support, to search GNUTLS_DIR as well.
 - NETCDF
 - MPFR
@@ -25,8 +25,7 @@ This project distributes the sources of several external software products with
 their own respective licenses which can be found in their code and attached
 license files. These software products and their licenses are as follows:

-* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
-* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
 * Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
 * Picojson (fem/picojson.h) -- Custom 2-clause license
+* Catch++  (tests/unit/catch.hpp) -- Boost 1.0 license
 * Zstr (general/zstr.hpp) -- MIT license
@@ -244,10 +244,6 @@ IF (DEFINED TPL_ENABLE_PETSC)
  SET(MFEM_USE_PETSC ${TPL_ENABLE_PETSC} CACHE BOOL "Enable PETSc support." FORCE)
 ENDIF()

-IF (DEFINED TPL_ENABLE_SLEPC)
-  SET(MFEM_USE_SLEPC ${TPL_ENABLE_SLEPC} CACHE BOOL "Enable SLEPc support." FORCE)
-ENDIF()
-
 IF (DEFINED TPL_ENABLE_MPFR)
  SET(MFEM_USE_MPFR ${TPL_ENABLE_MPFR} CACHE BOOL "Enable MPFR usage." FORCE)
 ENDIF()
@@ -34,12 +34,10 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
 set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
-set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
 set(MFEM_USE_GNUTLS @MFEM_USE_GNUTLS@)
 set(MFEM_USE_GSLIB @MFEM_USE_GSLIB@)
 set(MFEM_USE_NETCDF @MFEM_USE_NETCDF@)
 set(MFEM_USE_PETSC @MFEM_USE_PETSC@)
-set(MFEM_USE_SLEPC @MFEM_USE_SLEPC@)
 set(MFEM_USE_MPFR @MFEM_USE_MPFR@)
 set(MFEM_USE_SIDRE @MFEM_USE_SIDRE@)
 set(MFEM_USE_CONDUIT @MFEM_USE_CONDUIT@)
@@ -49,7 +47,6 @@ set(MFEM_USE_OCCA @MFEM_USE_OCCA@)
 set(MFEM_USE_RAJA @MFEM_USE_RAJA@)
 set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
-set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
@@ -74,9 +74,6 @@
 // [Deprecated] Enable experimental OpenMP support. Requires MFEM_THREAD_SAFE.
 #cmakedefine MFEM_USE_LEGACY_OPENMP

-// Internal MFEM option: enable group/batch allocation for some small objects.
-#cmakedefine MFEM_USE_MEMALLOC
-
 // Enable MFEM functionality based on the Mesquite library.
 #cmakedefine MFEM_USE_MESQUITE

@@ -89,12 +86,12 @@
 // Enable MFEM functionality based on the STRUMPACK library.
 #cmakedefine MFEM_USE_STRUMPACK

+// Internal MFEM option: enable group/batch allocation for some small objects.
+#cmakedefine MFEM_USE_MEMALLOC
+
 // Enable functionality based on the Ginkgo library
 #cmakedefine MFEM_USE_GINKGO

-// Enable MFEM functionality based on the AmgX library
-#cmakedefine MFEM_USE_AMGX
-
 // Enable MFEM functionality based on the GnuTLS library
 #cmakedefine MFEM_USE_GNUTLS

@@ -107,15 +104,9 @@
 // Enable MFEM functionality based on the PETSc library
 #cmakedefine MFEM_USE_PETSC

-// Enable MFEM functionality based on the SLEPc library
-#cmakedefine MFEM_USE_SLEPC
-
 // Enable MFEM functionality based on the Sidre library
 #cmakedefine MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes
-#cmakedefine MFEM_USE_SIMD
-
 // Enable MFEM functionality based on Conduit
 #cmakedefine MFEM_USE_CONDUIT

@@ -159,7 +150,4 @@
 // library.
 #cmakedefine MFEM_USE_SIMMETRIX

-// Enable interface to the MKL CPardiso library.
-#cmakedefine MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER
@@ -38,19 +38,7 @@ if(NOT ADIOS2_FOUND)
  endif()

  find_path(ADIOS2_INCLUDE_DIR adios2.h ${ADIOS2_INCLUDE_OPTS})
-  
-  # adios2 version 2.5.0 
-  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS})
-  
-  # adios2 version 2.6.0 and onwards
-  if(NOT ADIOS2_LIBRARY)
-  	find_library(ADIOS2_CXX11_MPI_LIBRARY NAMES adios2_cxx11_mpi ${ADIOS2_LIBRARY_OPTS})
-  	find_library(ADIOS2_CXX11_LIBRARY NAMES adios2_cxx11 ${ADIOS2_LIBRARY_OPTS})
-  	set(ADIOS2_LIBRARY ${ADIOS2_CXX11_MPI_LIBRARY} ${ADIOS2_CXX11_LIBRARY})
-  	if(MFEM_USE_MPI)
-  	  add_definitions(-DADIOS2_USE_MPI)
-  	endif()
-  endif() 
+  find_library(ADIOS2_LIBRARY NAMES adios2 ${ADIOS2_LIBRARY_OPTS}) 

  include(FindPackageHandleStandardArgs)
  find_package_handle_standard_args(ADIOS2
@@ -1,20 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - AMGX_FOUND
-#   - AMGX_LIBRARIES
-#   - AMGX_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-set(AMGX_REQUIRED_LIBRARIES cusparse cusolver cublas nvToolsExt)
-mfem_find_package(AMGX AMGX AMGX_DIR "include" "amgx_c.h" "lib" "amgx"
-  "Paths to headers required by AMGX." "Libraries required by AMGX.")
@@ -1,32 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - MKL_CPARDISO_FOUND
-#   - MKL_CPARDISO_LIBRARIES
-#   - MKL_CPARDISO_INCLUDE_DIRS
-
-if(NOT MKL_MPI_WRAPPER_LIB)
-  message(FATAL_ERROR "MKL CPardiso enabled but no MKL MPI Wrapper lib specified")
-endif()
-
-if(NOT MKL_LIBRARY_DIR)
-  message(WARNING "Using default MKL library path. Double check the variable MKL_LIBRARY_DIR")
-  set(MKL_LIBRARY_DIR "lib")
-endif()
-
-include(MfemCmakeUtilities)
-mfem_find_package(MKL_CPARDISO MKL_CPARDISO
-    MKL_CPARDISO_DIR "include" mkl_cluster_sparse_solver.h ${MKL_LIBRARY_DIR} mkl_core
-  "Paths to headers required by MKL CPardiso." "Libraries required by MKL CPARDISO."
-  ADD_COMPONENT MKL_LP64 "include" "" ${MKL_LIBRARY_DIR} mkl_intel_lp64
-  ADD_COMPONENT MKL_SEQUENTIAL "include" "" ${MKL_LIBRARY_DIR} mkl_sequential
-  ADD_COMPONENT MKL_MPI_WRAPPER "include" "" ${MKL_LIBRARY_DIR} ${MKL_MPI_WRAPPER_LIB})
@@ -1,44 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Sets the following variables:
-#   - SLEPC_FOUND
-#   - SLEPC_INCLUDE_DIRS
-#   - SLEPC_LIBRARIES
-
-set(SLEPc_REQUIRED_PACKAGES "PETSC" CACHE STRING
-  "Additional packages required by SLEPc")
-
-include(MfemCmakeUtilities)
-mfem_find_package(SLEPc SLEPC SLEPC_DIR
-  "include" "slepceps.h"
-  "${PETSC_ARCH}/lib" "slepc" # add NAMES_PER_DIR?
-  "Paths to headers required by SLEPc."
-  "Libraries required by SLEPc."
-  ADD_COMPONENT "config" "${PETSC_ARCH}/include" "slepcconf.h" "" ""
-  CHECK_BUILD SLEPC_VERSION_OK TRUE
-"
-#include \"petsc.h\"
-#include \"slepceps.h\"
-int main()
-{
-  PetscErrorCode ierr;
-  int argc = 0;
-  char** argv = NULL;
-  ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL);
-  EPS eps;
-  ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr);
-  ierr = EPSDestroy(&eps); CHKERRQ(ierr);
-  ierr = SlepcFinalize(); CHKERRQ(ierr);
-  return 0;
-}
-"
-  )
@@ -20,15 +20,10 @@ mfem_find_package(SUNDIALS SUNDIALS SUNDIALS_DIR
  "Paths to headers required by SUNDIALS." "Libraries required by SUNDIALS."
  ADD_COMPONENT NVector_Serial
    "include" nvector/nvector_serial.h "lib" sundials_nvecserial
-  ADD_COMPONENT NVector_Cuda
-    "include" nvector/nvector_cuda.h "lib" sundials_nveccuda
-  ADD_COMPONENT NVector_ParHyp
-    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT NVector_Parallel
    "include" nvector/nvector_parallel.h "lib" sundials_nvecparallel
-  ADD_COMPONENT NVector_MPIPlusX
-    "include" nvector/nvector_mpiplusx.h "lib" sundials_nvecmpiplusx
+  ADD_COMPONENT NVector_ParHyp
+    "include" nvector/nvector_parhyp.h "lib" sundials_nvecparhyp
  ADD_COMPONENT CVODE "include" cvode/cvode.h "lib" sundials_cvode
-  ADD_COMPONENT CVODES "include" cvodes/cvodes.h "lib" sundials_cvodes
  ADD_COMPONENT ARKODE "include" arkode/arkode.h "lib" sundials_arkode
  ADD_COMPONENT KINSOL "include" kinsol/kinsol.h "lib" sundials_kinsol)
@@ -128,15 +128,7 @@ function(add_mfem_miniapp MFEM_EXE_NAME)
  if (MFEM_USE_CUDA)
    set_property(SOURCE ${MAIN_LIST} ${EXTRA_SOURCES_LIST}
      PROPERTY LANGUAGE CUDA)
-    if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12.0)
-      list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
-    else()
-      set(LIST_)
-      foreach(item IN LISTS EXTRA_OPTIONS_LIST)
-        list(APPEND LIST_ "-Xcompiler=${item}")
-      endforeach()
-      set(EXTRA_OPTIONS_LIST ${LIST_})
-    endif()
+    list(TRANSFORM EXTRA_OPTIONS_LIST PREPEND "-Xcompiler=")
  endif()

  # Actually add the executable
@@ -738,11 +730,10 @@ function(mfem_export_mk_files)
      MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
      MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_OPENMP MFEM_USE_LEGACY_OPENMP
      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
-      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GINKGO MFEM_USE_AMGX
-      MFEM_USE_GNUTLS MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC
-      MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_CONDUIT MFEM_USE_PUMI
-      MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA MFEM_USE_UMPIRE MFEM_USE_SIMD
-      MFEM_USE_ADIOS2)
+      MFEM_USE_SUPERLU MFEM_USE_STRUMPACK MFEM_USE_GNUTLS
+      MFEM_USE_GSLIB MFEM_USE_NETCDF MFEM_USE_PETSC MFEM_USE_MPFR MFEM_USE_SIDRE
+      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_CUDA MFEM_USE_OCCA MFEM_USE_RAJA
+      MFEM_USE_UMPIRE)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -752,7 +743,6 @@ function(mfem_export_mk_files)
  endforeach()
  # TODO: Add support for MFEM_USE_CUDA=YES
  set(MFEM_CXX ${CMAKE_CXX_COMPILER})
-  set(MFEM_HOST_CXX ${MFEM_CXX})
  set(MFEM_CPPFLAGS "")
  string(STRIP "${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
         MFEM_CXXFLAGS)
@@ -45,15 +45,9 @@
 #ifdef MFEM_USE_STRUMPACK
 #error Building with STRUMPACK (MFEM_USE_STRUMPACK=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_MKL_CPARDISO
-#error Building with MKL CPARDISO (MFEM_USE_MKL_CPARDISO=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PETSC
 #error Building with PETSc (MFEM_USE_PETSC=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
-#ifdef MFEM_USE_SLEPC
-#error Building with SLEPc (MFEM_USE_SLEPC=YES) requires MPI (MFEM_USE_MPI=YES)
-#endif
 #ifdef MFEM_USE_PUMI
 #error Building with PUMI (MFEM_USE_PUMI=YES) requires MPI (MFEM_USE_MPI=YES)
 #endif
@@ -93,7 +93,6 @@

 // Enable MFEM functionality based on the SuperLU library.
 // #define MFEM_USE_SUPERLU
-// #define MFEM_USE_SUPERLU5

 // Enable MFEM functionality based on the STRUMPACK library.
 // #define MFEM_USE_STRUMPACK
@@ -101,18 +100,12 @@
 // Enable MFEM features based on the Ginkgo library
 // #define MFEM_USE_GINKGO

-// Enable MFEM functionality based on the AmgX library.
-// #define MFEM_USE_AMGX
-
 // Enable secure socket streams based on the GNUTLS library
 // #define MFEM_USE_GNUTLS

 // Enable Sidre support
 // #define MFEM_USE_SIDRE

-// Enable the use of SIMD in the high performance templated classes
-// #define MFEM_USE_SIMD
-
 // Enable Conduit support
 // #define MFEM_USE_CONDUIT

@@ -122,9 +115,6 @@
 // Enable functionality based on the PETSc library
 // #define MFEM_USE_PETSC

-// Enable functionality based on the SLEPc library
-// #define MFEM_USE_SLEPC
-
 // Enable functionality based on the MPFR library.
 // #define MFEM_USE_MPFR

@@ -167,7 +157,4 @@
 // library.
 // #define MFEM_USE_SIMMETRIX

-// Enable interface to the MKL CPardiso library.
-// #define MFEM_USE_MKL_CPARDISO
-
 #endif // MFEM_CONFIG_HEADER
@@ -32,14 +32,11 @@ MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
 MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
-MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
 MFEM_USE_STRUMPACK     = @MFEM_USE_STRUMPACK@
 MFEM_USE_GINKGO        = @MFEM_USE_GINKGO@
-MFEM_USE_AMGX          = @MFEM_USE_AMGX@
 MFEM_USE_GNUTLS        = @MFEM_USE_GNUTLS@
 MFEM_USE_NETCDF        = @MFEM_USE_NETCDF@
 MFEM_USE_PETSC         = @MFEM_USE_PETSC@
-MFEM_USE_SLEPC         = @MFEM_USE_SLEPC@
 MFEM_USE_MPFR          = @MFEM_USE_MPFR@
 MFEM_USE_SIDRE         = @MFEM_USE_SIDRE@
 MFEM_USE_CONDUIT       = @MFEM_USE_CONDUIT@
@@ -52,13 +49,10 @@ MFEM_USE_RAJA          = @MFEM_USE_RAJA@
 MFEM_USE_OCCA          = @MFEM_USE_OCCA@
 MFEM_USE_CEED          = @MFEM_USE_CEED@
 MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
-MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
-MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
-MFEM_HOST_CXX  = @MFEM_HOST_CXX@
 MFEM_CPPFLAGS  = @MFEM_CPPFLAGS@
 MFEM_CXXFLAGS  = @MFEM_CXXFLAGS@
 MFEM_TPLFLAGS  = @MFEM_TPLFLAGS@
@@ -33,15 +33,12 @@ option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
 option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
-option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
 option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
 option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
-option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
 option(MFEM_USE_GNUTLS "Enable GNUTLS usage" OFF)
 option(MFEM_USE_GSLIB "Enable GSLIB usage" OFF)
 option(MFEM_USE_NETCDF "Enable NETCDF usage" OFF)
 option(MFEM_USE_PETSC "Enable PETSc support." OFF)
-option(MFEM_USE_SLEPC "Enable SLEPc support." OFF)
 option(MFEM_USE_MPFR "Enable MPFR usage." OFF)
 option(MFEM_USE_SIDRE "Enable Axom/Sidre usage" OFF)
 option(MFEM_USE_CONDUIT "Enable Conduit usage" OFF)
@@ -52,9 +49,7 @@ option(MFEM_USE_OCCA "Enable OCCA" OFF)
 option(MFEM_USE_RAJA "Enable RAJA" OFF)
 option(MFEM_USE_CEED "Enable CEED" OFF)
 option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
-option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
-option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)

 set(MFEM_MPI_NP 4 CACHE STRING "Number of processes used for MPI tests")

@@ -91,8 +86,6 @@ set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library."

 set(LIBUNWIND_DIR "" CACHE PATH "Path to Libunwind.")

-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
-# and modify cmake variables for hypre for sundials
 set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
    "Path to the SUNDIALS library.")
 # The following may be necessary, if SUNDIALS was built with KLU:
@@ -112,7 +105,7 @@ set(ParMETIS_DIR "${MFEM_DIR}/../parmetis-4.0.3" CACHE PATH
 set(ParMETIS_REQUIRED_PACKAGES "METIS" CACHE STRING
    "Additional packages required by ParMETIS.")

-set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_6.3.1" CACHE PATH
+set(SuperLUDist_DIR "${MFEM_DIR}/../SuperLU_DIST_5.1.0" CACHE PATH
    "Path to the SuperLU_DIST library.")
 # SuperLU_DIST may also depend on "OpenMP", depending on how it was compiled.
 set(SuperLUDist_REQUIRED_PACKAGES "MPI" "BLAS" "ParMETIS" CACHE STRING
@@ -148,8 +141,6 @@ set(ScaLAPACK_TARGET_NAMES scalapack)

 set(Ginkgo_DIR "${MFEM_DIR}/../ginkgo" CACHE PATH "Path to the Ginkgo library.")

-set(AMGX_DIR "${MFEM_DIR}/../amgx" CACHE PATH "Path to AmgX")
-
 set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")

 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
@@ -163,10 +154,6 @@ set(PETSC_DIR "${MFEM_DIR}/../petsc" CACHE PATH
    "Path to the PETSc main directory.")
 set(PETSC_ARCH "arch-linux2-c-debug" CACHE STRING "PETSc build architecture.")

-set(SLEPC_DIR "${MFEM_DIR}/../slepc" CACHE PATH
-    "Path to the SLEPc main directory.")
-set(SLEPC_ARCH "arch-linux2-c-debug" CACHE STRING "SLEPC build architecture.")
-
 set(MPFR_DIR "" CACHE PATH "Path to the MPFR library.")

 set(CONDUIT_DIR "${MFEM_DIR}/../conduit" CACHE PATH
@@ -185,10 +172,6 @@ set(HIOP_DIR "${MFEM_DIR}/../hiop/install" CACHE STRING
 set(HIOP_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
    "Packages that HiOp depends on.")

-set(MKL_CPARDISO_DIR "" CACHE STRING "MKL installation path.")
-set(MKL_MPI_WRAPPER_LIB "mkl_blacs_mpich_lp64" CACHE STRING "MKL MPI wrapper library")
-set(MKL_LIBRARY_DIR "" CACHE STRING "Custom library subdirectory")
-
 set(OCCA_DIR "${MFEM_DIR}/../occa" CACHE PATH "Path to OCCA")
 set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
@@ -120,14 +120,11 @@ MFEM_USE_SUNDIALS      = NO
 MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
-MFEM_USE_SUPERLU5      = NO
 MFEM_USE_STRUMPACK     = NO
 MFEM_USE_GINKGO        = NO
-MFEM_USE_AMGX          = NO
 MFEM_USE_GNUTLS        = NO
 MFEM_USE_NETCDF        = NO
 MFEM_USE_PETSC         = NO
-MFEM_USE_SLEPC         = NO
 MFEM_USE_MPFR          = NO
 MFEM_USE_SIDRE         = NO
 MFEM_USE_CONDUIT       = NO
@@ -140,9 +137,7 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_UMPIRE        = NO
-MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
-MFEM_USE_MKL_CPARDISO  = NO

 # Compile and link options for zlib.
 ZLIB_DIR =
@@ -192,19 +187,13 @@ OPENMP_LIB =
 POSIX_CLOCKS_LIB = -lrt

 # SUNDIALS library configuration
-# For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
-# and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
- -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = -Wl,-rpath,$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib64\
+ -lsundials_arkode -lsundials_cvode -lsundials_nvecserial -lsundials_kinsol

 ifeq ($(MFEM_USE_MPI),YES)
-   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
-endif
-ifeq ($(MFEM_USE_CUDA),YES)
-   SUNDIALS_LIB += -lsundials_nveccuda
+   SUNDIALS_LIB += -lsundials_nvecparhyp -lsundials_nvecparallel
 endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
@@ -223,15 +212,9 @@ SUITESPARSE_LIB = -Wl,-rpath,$(SUITESPARSE_DIR)/lib -L$(SUITESPARSE_DIR)/lib\
 -lsuitesparseconfig $(LIB_RT) $(METIS_LIB) $(LAPACK_LIB)

 # SuperLU library configuration
-ifeq ($(MFEM_USE_SUPERLU5),YES)
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0   
-else
-   SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_6.3.1
-   SUPERLU_OPT = -I$(SUPERLU_DIR)/include
-   SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib64 -L$(SUPERLU_DIR)/lib64 -lsuperlu_dist -lblas
-endif
+SUPERLU_DIR = @MFEM_DIR@/../SuperLU_DIST_5.1.0
+SUPERLU_OPT = -I$(SUPERLU_DIR)/SRC
+SUPERLU_LIB = -Wl,-rpath,$(SUPERLU_DIR)/lib -L$(SUPERLU_DIR)/lib -lsuperlu_dist_5.1.0

 # SCOTCH library configuration (required by STRUMPACK <= v2.1.0, optional in
 # STRUMPACK >= v2.2.0)
@@ -264,13 +247,7 @@ STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 # Ginkgo library configuration (currently not needed)
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo\
- -lginkgo_omp -lginkgo_cuda -lginkgo_reference
-
-# AmgX library configuration
-AMGX_DIR = @MFEM_DIR@/../amgx
-AMGX_OPT = -I$(AMGX_DIR)/include
-AMGX_LIB = -lcusparse -lcusolver -lcublas -lnvToolsExt -L$(AMGX_DIR)/lib -lamgx
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_DIR)/lib -L$(GINKGO_DIR)/lib -lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference

 # GnuTLS library configuration
 GNUTLS_OPT =
@@ -298,20 +275,6 @@ ifeq ($(PETSC_FOUND),YES)
      -L$(abspath $(PETSC_DIR))/lib -lpetsc $(PETSC_LIB)
 endif

-SLEPC_DIR := $(MFEM_DIR)/../slepc
-SLEPC_VARS := $(SLEPC_DIR)/lib/slepc/conf/slepc_variables
-SLEPC_FOUND := $(if $(wildcard $(SLEPC_VARS)),YES,)
-SLEPC_INC_VAR = SLEPC_INCLUDE
-SLEPC_LIB_VAR = SLEPC_EXTERNAL_LIB
-ifeq ($(SLEPC_FOUND),YES)
-   SLEPC_OPT := $(shell sed -n "s/$(SLEPC_INC_VAR) *= *//p" $(SLEPC_VARS))
-   # Some additional external libraries might be defined in this file
-   -include ${SLEPC_DIR}/${PETSC_ARCH}/lib/slepc/conf/slepcvariables
-   SLEPC_LIB := $(shell sed -n "s/$(SLEPC_LIB_VAR) *= *//p" $(SLEPC_VARS))
-   SLEPC_LIB := -Wl,-rpath,$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib\
-      -L$(abspath $(SLEPC_DIR))/$(PETSC_ARCH)/lib -lslepc $(SLEPC_LIB)
-endif
-
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr
@@ -360,9 +323,9 @@ GSLIB_DIR = @MFEM_DIR@/../gslib/build
 GSLIB_OPT = -I$(GSLIB_DIR)/include
 GSLIB_LIB = -L$(GSLIB_DIR)/lib -lgs

-# CUDA library configuration
+# CUDA library configuration (currently not needed)
 CUDA_OPT =
-CUDA_LIB = -lcusparse
+CUDA_LIB =

 # HIP library configuration (currently not needed)
 HIP_OPT =
@@ -391,15 +354,6 @@ UMPIRE_DIR = @MFEM_DIR@/../umpire
 UMPIRE_OPT = -I$(UMPIRE_DIR)/include
 UMPIRE_LIB = -L$(UMPIRE_DIR)/lib -lumpire

-# MKL CPardiso library configuration
-MKL_CPARDISO_DIR ?=
-MKL_MPI_WRAPPER ?= mkl_blacs_mpich_lp64
-MKL_LIBRARY_SUBDIR ?= lib
-MKL_CPARDISO_OPT = -I$(MKL_CPARDISO_DIR)/include
-MKL_CPARDISO_LIB = -Wl,-rpath,$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR)\
-                   -L$(MKL_CPARDISO_DIR)/$(MKL_LIBRARY_SUBDIR) -l$(MKL_MPI_WRAPPER)\
-                   -lmkl_intel_lp64 -lmkl_sequential -lmkl_core
-
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -78,14 +78,6 @@ groups_parallel=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 # All groups serial + parallel runs mixed in the same group:
 groups_all=(
@@ -115,14 +107,6 @@ groups_all=(
   "miniapps/electromagnetics"
   "joule.cpp"'
 #   "{volta,tesla,joule}.cpp"' # todo: multiline sample runs
-'"convergence"
-   "Convergence tests:"
-   "tests/convergence"
-   "diffusion.cpp"'
-'"par-mesh-format"
-   "Parallel mesh tests:"
-   "tests/par-mesh-format"
-   "ex1p.cpp"'
 )
 make_all="all"
 base_timeformat=$'real: %3Rs  user: %3Us  sys: %3Ss  %%cpu: %P'
@@ -396,15 +380,10 @@ function timed_run()
 # This function is used to execute the sample runs
 function go()
 {
-   # Strip leading and trailing spaces from $1 and store the result in cmd_line
-   shopt -s extglob
-   local cmd_line="${1##+( )}"
-   cmd_line="${cmd_line%%+( )}"
-   shopt -u extglob
-   eval local cmd=(${cmd_line})
+   local cmd=("$@")
   local res=""
   echo $sep
-   echo "<${group}>" "${cmd_line}"
+   echo "<${group}>" "${cmd[@]}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
      timed_run "${cmd[@]}"
@@ -416,15 +395,15 @@ function go()
   else
      res="${red}FAILED${none}"
   fi
-   printf "[${res}] <${group}> ${cmd_line}\n"
+   printf "[${res}] <${group}> ${cmd[*]}\n"
   if [ "${timing}" == "yes" ]; then
      printf "Run time: %s\n" "${timer}"
      timer=(${timer})
      timer="${timer[1]}"
-      printf -v line "[$res](%8s) ${cmd_line}" "$timer"
+      printf -v line "[$res](%8s) ${cmd[*]}" "$timer"
      summary=("${summary[@]}" "$line")
   else
-      summary=("${summary[@]}" "[${res}] ${cmd_line}")
+      summary=("${summary[@]}" "[${res}] ${cmd[*]}")
   fi
   echo $sep
 }
@@ -459,7 +438,7 @@ function go_group()
      fi
      for run in "${runs[@]}"; do
         if [ "${run}" == "" ]; then continue; fi
-         eval go \"\${run_prefix} \${run} \${run_suffix}\" $output
+         eval go \${run_prefix} \${run} \${run_suffix} $output
      done
   done
   ${make} clean-exec
@@ -525,7 +504,7 @@ function echo_run()
 {
   echo "   $@"
   { echo "   $@"; echo "$sep";
-     eval "$@"
+     "$@"
     echo "$sep"; } >> "$echo_log" 2>&1
 }

@@ -545,28 +524,6 @@ function build_all()
   echo_run ${make} config ${mfem_config} || exit 1
   echo_run ${make} ${make_j} || exit 1
   echo_run ${make} ${make_all} ${make_j} || exit 1
-   # Build groups in directories other than the directories built by 'make all':
-   for group_params in "${groups[@]}"; do
-      eval params=(${group_params})
-      group_dir="${params[2]}"
-      case "$group_dir" in
-         (examples*|miniapps*)
-            # Built by 'make all'
-            ;;
-         (*)
-            if [ "${mfem_dir}" != "${mfem_build_dir}" ]; then
-               echo_run mkdir -p "${group_dir}" || exit 1
-               echo_run cd "${group_dir}" || exit 1
-               echo_run cp -af "${mfem_dir}/${group_dir}/makefile" . || exit 1
-            else
-               echo_run cd "${group_dir}" || exit 1
-            fi
-            echo_run ${make} clean || exit 1
-            echo_run ${make} MFEM_DIR="${mfem_dir}" ${make_j} || exit 1
-            echo_run cd "${mfem_build_dir}" || exit 1
-            ;;
-      esac
-   done
 }

 # Function that runs all sample runs, given by the array variable "groups".
@@ -29,20 +29,8 @@
 #define MFEM_ALWAYS_INLINE
 #endif

-// --- MFEM_VECTORIZE_LOOP (disabled)
-#if (__cplusplus >= 201103L) && !defined(MFEM_DEBUG) && defined(__GNUC__)
-//#define MFEM_VECTORIZE_LOOP _Pragma("GCC ivdep")
-#define MFEM_VECTORIZE_LOOP
-#else
-#define MFEM_VECTORIZE_LOOP
-#endif
-
-// MFEM_TEMPLATE_BLOCK_SIZE is the block size used by the template matrix-matrix
-// multiply, Mult_AB, defined in tmatrix.hpp. This parameter will generally
-// require tuning to determine good value. It is probably highly influenced by
-// the SIMD width when Mult_AB is used with a SIMD type like AutoSIMD.
 #define MFEM_TEMPLATE_BLOCK_SIZE 4
-
+#define MFEM_SIMD_SIZE 32
 #define MFEM_TEMPLATE_ENABLE_SERIALIZE

 // #define MFEM_TEMPLATE_ELTRANS_HAS_NODE_DOFS
@@ -50,6 +38,11 @@
 // #define MFEM_TEMPLATE_FIELD_EVAL_DATA_HAS_DOFS
 #define MFEM_TEMPLATE_INTRULE_COEFF_PRECOMP

+// derived macros
+#define MFEM_ROUNDUP(val,base) ((((val)+(base)-1)/(base))*(base))
+#define MFEM_ALIGN_SIZE(size,type) \
+   MFEM_ROUNDUP(size,(MFEM_SIMD_SIZE)/sizeof(type))
+
 #ifdef MFEM_COUNT_FLOPS
 namespace mfem
 {
@@ -1,86 +0,0 @@
-SetFactory("OpenCASCADE");
-
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
-
-// Set the geometry order (1, 2, ..., 9)
-order = 3;
-
-// Set the element type (3 - triangles, 4 - quadrilaterals)
-type = 3;
-
-// Number of radial elements
-nrad = 2;
-
-// Number of azimuthal elements on inner arc
-nazm1 = 3;
-
-// Number of azimuthal elements on outer arc
-nazm2 = 5;
-
-// Note: Using type = 4 with nazm1 != nazm2 can lead to mixed meshes
-//       containing both triangles and quadrilaterals.
-
-// Inner and outer radii
-R1 = 1.0;
-R2 = 2.0;
-
-// Angular size of the sector
-Phi = Pi/3.0;
-
-Point(1) = {0.0, 0, 0, 1.0};
-Point(2) = {R1, 0, 0, 1.0};
-Point(3) = {R2, 0, 0, 1.0};
-Point(4) = {R1*Cos(Phi), R1*Sin(Phi), 0, 1.0};
-Point(5) = {R2*Cos(Phi), R2*Sin(Phi), 0, 1.0};
-Line(1) = {2, 3};
-Line(2) = {4, 5};
-Circle(3) = {2, 1, 4};
-Circle(4) = {3, 1, 5};
-Curve Loop(5) = {1, 4, -2, -3};
-Plane Surface(1) = {5};
-
-Transfinite Curve{1} = nrad+1;
-Transfinite Curve{2} = nrad+1;
-Transfinite Curve{3} = nazm1+1;
-Transfinite Curve{4} = nazm2+1;
-
-If (nazm1 == nazm2)
-   Transfinite Surface{1};
-EndIf
-
-If (type == 4)
-   Recombine Surface {1};
-EndIf
-
-// Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Line{1} = {2} Rotate{{0,0,1}, {0,0,0}, -Phi};
-EndIf
-
-// Tag surfaces and volumes with positive integers
-Physical Curve(1) = {3};
-Physical Curve(2) = {4};
-Physical Curve(3) = {1};
-Physical Curve(4) = {2};
-Physical Surface(1) = {1};
-
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
-
-// Generate 2D mesh
-Mesh 2;
-SetOrder order;
-Mesh.MshFileVersion = 2.2;
-
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-annulus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("annulus-sector-t%01g-o%01g.msh", type, order);
-EndIf
@@ -1,192 +0,0 @@
-$MeshFormat
-2.2 0 8
-$EndMeshFormat
-$Nodes
-136
-1 1 0 0
-2 2 0 0
-3 0.5000000000000001 0.8660254037844386 0
-4 1 1.732050807568877 0
-5 1.5 0 0
-6 1.166666666666667 0 0
-7 1.333333333333333 0 0
-8 1.666666666666667 0 0
-9 1.833333333333333 0 0
-10 0.7500000000000002 1.299038105676658 0
-11 0.5833333333333335 1.010362971081845 0
-12 0.6666666666666667 1.154700538379251 0
-13 0.8333333333333335 1.443375672974064 0
-14 0.9166666666666669 1.587713240271471 0
-15 0.9396926207859085 0.3420201433256683 0
-16 0.7660444431189786 0.6427876096865386 0
-17 0.993238357741943 0.1160929141252301 0
-18 0.9730448705798238 0.2306158707424401 0
-19 0.8936326403234125 0.4487991802004617 0
-20 0.8354878114129367 0.5495089780708056 0
-21 0.6862416378687343 0.7273736415730481 0
-22 0.597158591702787 0.8021231927550432 0
-23 1.956295201467611 0.4158233816355181 0
-24 1.827090915285202 0.8134732861515996 0
-25 1.618033988749896 1.175570504584944 0
-26 1.338261212717719 1.486289650954786 0
-27 1.995128100519648 0.1395129474882505 0
-28 1.980536137483141 0.278346201920131 0
-29 1.922523391876638 0.551274711633998 0
-30 1.879385241571817 0.6840402866513373 0
-31 1.765895185717855 0.9389431255717802 0
-32 1.696096192312853 1.059838528466408 0
-33 1.532088886237958 1.285575219373077 0
-34 1.438679600677305 1.389316740917992 0
-35 1.231322950651319 1.576021507213442 0
-36 1.118385806941496 1.658075145110082 0
-37 1.162276263405681 0.6710405135499813 0
-38 1.248615852873337 1.079531485311822 0
-39 1.559209616901855 0.5415673055003691 0
-40 1.478306597054007 0.8535007117539289 0
-41 0.9210953433941653 0.9653302893212266 0
-42 1.296548225291847 0.3150268220262836 0
-43 1.055002035226811 1.358510675893086 0
-44 1.704005774249187 0.2344032256041583 0
-45 0.6403651144647218 0.8991270322967013 0
-46 0.7807302289294435 0.9322286608089638 0
-47 0.864063562262777 1.07656622810637 0
-48 0.8070317811313885 1.187802166891514 0
-49 0.7236984477980553 1.043464599594108 0
-50 1.432182741763949 0.1050089406754279 0
-51 1.364365483527898 0.2100178813508558 0
-52 1.197698816861231 0.2100178813508558 0
-53 1.098849408430616 0.1050089406754279 0
-54 1.265516075097282 0.1050089406754278 0
-55 0.8177280765440409 0.7503018362314346 0
-56 0.869411709969103 0.8578160627763305 0
-57 0.7348318576552288 0.8316090412164392 0
-58 1.177596357123201 0.3240245957927452 0
-59 1.058644488954555 0.3330223695592067 0
-60 1.087610484537871 0.2205785356313179 0
-61 1.267619707955123 0.7318605796179638 0
-62 1.372963152504565 0.7926806456859463 0
-63 1.40174301566045 0.9288443029398934 0
-64 1.325179434266893 1.004187894125858 0
-65 1.219835989717452 0.9433678280578752 0
-66 1.191056126561566 0.8072041708039283 0
-67 1.296399571111008 0.8680242368719107 0
-68 1.532241943619239 0.645545107584889 0
-69 1.505274270336623 0.7495229096694089 0
-70 1.294587381237739 0.6278827775334439 0
-71 1.426898499069797 0.5847250415169065 0
-72 1.399930825787181 0.6887028436014264 0
-73 1.139442349713613 1.041464419981624 0
-74 1.030268846553889 1.003397354651425 0
-75 1.001488983398004 0.8672336973974781 0
-76 1.081882623401843 0.7691371054737297 0
-77 1.110662486557728 0.9053007627276766 0
-78 1.207033584034403 0.5523692830420821 0
-79 1.251790904663125 0.4336980525341829 0
-80 1.384102022495183 0.3905403165176455 0
-81 1.471655819698519 0.4660538110090073 0
-82 1.339344701866461 0.5092115470255447 0
-83 1.73779714915742 0.7228379592678562 0
-84 1.648503383029637 0.6322026323841127 0
-85 1.691571478423774 0.4996526642120855 0
-86 1.823933339945692 0.4577380229238018 0
-87 1.787039416783436 0.5922941012619014 0
-88 1.308379426102925 1.350703595740465 0
-89 1.278497639488131 1.215117540526144 0
-90 1.371755231498856 1.111544491736196 0
-91 1.494894610124376 1.14355749816057 0
-92 1.4064614465962 1.25147448186763 0
-93 1.013887168325833 0.451693600067106 0
-94 1.088081715865757 0.5613670568085436 0
-95 1.03019898997678 0.6616228789288338 0
-96 0.8981217165478794 0.6522052443076862 0
-97 0.9637989050473432 0.5564495572737495 0
-98 1.432367408277627 0.2881522898855752 0
-99 1.568186591263407 0.2612777577448668 0
-100 1.655740388466743 0.3367912522362286 0
-101 1.607475002684299 0.4391792788682989 0
-102 1.519921205480963 0.3636657843769371 0
-103 1.184077913657828 1.17252454883891 0
-104 1.119539974442319 1.265517612365998 0
-105 1.010366471282595 1.2274505470358 0
-106 0.9657309073383804 1.096390418178513 0
-107 1.074904410498104 1.134457483508712 0
-108 1.901335258083062 0.07813440853471942 0
-109 1.802670516166124 0.1562688170694388 0
-110 1.636003849499458 0.156268817069439 0
-111 1.568001924749729 0.07813440853471942 0
-112 1.734668591416396 0.07813440853471944 0
-113 0.8516673450756037 1.318862295748801 0
-114 0.9533346901512071 1.338686485820944 0
-115 1.03666802348454 1.48302405311835 0
-116 1.01833401174227 1.607537430343614 0
-117 0.9350006784089369 1.463199863046207 0
-118 1.710829475874804 0.8268157613523761 0
-119 1.594568036464405 0.8401582365531526 0
-120 1.621535709747021 0.7361804344686326 0
-121 1.52488239428597 0.9608573093642675 0
-122 1.571458191517933 1.068213906974606 0
-123 1.448318812892413 1.036200900550232 0
-124 0.908699126206992 1.207626356963657 0
-125 1.500184666513678 0.1831433492101474 0
-126 1.646765991694905 0.9507607885973723 0
-127 0.9498053499729417 0.7597194708525821 0
-128 1.132839036494479 0.4426958263006443 0
-129 1.149421761057113 1.40110366758032 0
-130 1.243841486887416 1.443696659267553 0
-131 1.134903597606542 1.530869109405537 0
-132 1.872198725728137 0.3553499962917315 0
-133 1.788102249988661 0.2948766109479449 0
-134 1.893223337417325 0.2174207916708467 0
-135 1.213959700272622 1.308110604053232 0
-136 1.739836864206217 0.3972646375800152 0
-$EndNodes
-$Elements
-38
-1 26 2 3 1 1 5 6 7
-2 26 2 3 1 5 2 8 9
-3 26 2 4 2 3 10 11 12
-4 26 2 4 2 10 4 13 14
-5 26 2 1 3 1 15 17 18
-6 26 2 1 3 15 16 19 20
-7 26 2 1 3 16 3 21 22
-8 26 2 2 4 2 23 27 28
-9 26 2 2 4 23 24 29 30
-10 26 2 2 4 24 25 31 32
-11 26 2 2 4 25 26 33 34
-12 26 2 2 4 26 4 35 36
-13 21 2 1 1 3 41 10 45 46 47 48 12 11 49
-14 21 2 1 1 5 42 1 50 51 52 53 6 7 54
-15 21 2 1 1 16 41 3 55 56 46 45 22 21 57
-16 21 2 1 1 1 42 15 53 52 58 59 18 17 60
-17 21 2 1 1 37 40 38 61 62 63 64 65 66 67
-18 21 2 1 1 39 40 37 68 69 62 61 70 71 72
-19 21 2 1 1 38 41 37 73 74 75 76 66 65 77
-20 21 2 1 1 37 42 39 78 79 80 81 71 70 82
-21 21 2 1 1 24 39 23 83 84 85 86 29 30 87
-22 21 2 1 1 26 38 25 88 89 90 91 33 34 92
-23 21 2 1 1 15 37 16 93 94 95 96 20 19 97
-24 21 2 1 1 42 44 39 98 99 100 101 81 80 102
-25 21 2 1 1 38 43 41 103 104 105 106 74 73 107
-26 21 2 1 1 2 44 5 108 109 110 111 8 9 112
-27 21 2 1 1 10 43 4 113 114 115 116 14 13 117
-28 21 2 1 1 24 40 39 118 119 69 68 84 83 120
-29 21 2 1 1 38 40 25 64 63 121 122 91 90 123
-30 21 2 1 1 41 43 10 106 105 114 113 48 47 124
-31 21 2 1 1 5 44 42 111 110 99 98 51 50 125
-32 21 2 1 1 25 40 24 122 121 119 118 31 32 126
-33 21 2 1 1 37 41 16 76 75 56 55 96 95 127
-34 21 2 1 1 15 42 37 59 58 79 78 94 93 128
-35 21 2 1 1 4 43 26 116 115 129 130 35 36 131
-36 21 2 1 1 23 44 2 132 133 109 108 27 28 134
-37 21 2 1 1 26 43 38 130 129 104 103 89 88 135
-38 21 2 1 1 39 44 23 101 100 133 132 86 85 136
-$EndElements
-$Periodic
-1
-1 1 2
-Affine 0.5000000000000001 0.8660254037844386 0 0 -0.8660254037844386 0.5000000000000001 0 0 0 0 1 0 0 0 0 1
-3
-5 10
-1 3
-2 4
-$EndPeriodic
@@ -1,141 +0,0 @@
-// Select periodic mesh by setting this to either 0 - standard, 1 - periodic
-periodic = 1;
-
-// Set the geometry order (1, 2, ..., 10 for tetrahedra or 9 for other types)
-order = 3;
-
-// Set the element type (4 - tetrahedra, 6 - wedges, 8 - hexahedra)
-type = 8;
-
-// Minor and major radii
-R1 = 1.0;
-R2 = 2.0;
-
-// Side length of interior square
-A1 = 0.8;
-
-// Angular size of the sector
-Phi = Pi/3.0;
-
-// Number of azimuthal elements
-nazm = 3;
-
-// Number of elements around a quarter of the circle
-narc = 2;
-
-// Number of elements between surface and interior square
-nshl = 1;
-
-lc = 0.5;
-a1 = A1 / Sqrt(2.0);
-
-Point(1) = {R2+R1, 0, 0, lc};
-Point(2) = {R2, 0, R1, lc};
-Point(3) = {R2-R1, 0, 0, lc};
-Point(4) = {R2, 0, -R1, lc};
-Point(5) = {R2, 0, 0, lc};
-Point(6) = {R2+a1, 0, 0, lc};
-Point(7) = {R2, 0, a1, lc};
-Point(8) = {R2-a1, 0, 0, lc};
-Point(9) = {R2, 0, -a1, lc};
-
-Circle(1) = {1,5,2};
-Circle(2) = {2,5,3};
-Circle(3) = {3,5,4};
-Circle(4) = {4,5,1};
-
-Line(5) = {6,1};
-Line(6) = {7,2};
-Line(7) = {8,3};
-Line(8) = {9,4};
-
-Line(9) = {6, 7};
-Line(10) = {7, 8};
-Line(11) = {8, 9};
-Line(12) = {9, 6};
-
-Line Loop(101) = {1, -6, -9, 5};
-Line Loop(102) = {2, -7, -10, 6};
-Line Loop(103) = {3, -8, -11, 7};
-Line Loop(104) = {4, -5, -12, 8};
-Line Loop(105) = {9, 10, 11, 12};
-
-Plane Surface(201) = {101};
-Plane Surface(202) = {102};
-Plane Surface(203) = {103};
-Plane Surface(204) = {104};
-Plane Surface(205) = {105};
-
-Transfinite Curve{1} = narc+1;
-Transfinite Curve{2} = narc+1;
-Transfinite Curve{3} = narc+1;
-Transfinite Curve{4} = narc+1;
-
-Transfinite Curve{5} = nshl+1;
-Transfinite Curve{6} = nshl+1;
-Transfinite Curve{7} = nshl+1;
-Transfinite Curve{8} = nshl+1;
-
-Transfinite Curve{9} = narc+1;
-Transfinite Curve{10} = narc+1;
-Transfinite Curve{11} = narc+1;
-Transfinite Curve{12} = narc+1;
-
-If (type == 8)
-   Recombine Surface {201};
-   Recombine Surface {202};
-   Recombine Surface {203};
-   Recombine Surface {204};
-   Recombine Surface {205};
-
-   Transfinite Surface {201} = {1,2,7,6};
-   Transfinite Surface {202} = {2,3,8,7};
-   Transfinite Surface {203} = {3,4,9,8};
-   Transfinite Surface {204} = {4,1,6,9};
-   Transfinite Surface {205} = {6,7,8,9};
-EndIf
-
-If (type == 4)
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm};
-}
-Else
-   Extrude { {0,0,1} , {0,0,0} , Phi} {
-      Surface{201,202,203,204,205}; Layers{nazm}; Recombine;
-}
-EndIf
-
-// Set a rotation periodicity constraint:
-If (periodic)
-   Periodic Surface{227} = {201} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{249} = {202} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{271} = {203} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{293} = {204} Rotate{{0,0,1}, {0,0,0}, Phi};
-   Periodic Surface{315} = {205} Rotate{{0,0,1}, {0,0,0}, Phi};
-EndIf
-
-// Tag surfaces and volumes with positive integers
-Physical Surface(1) = {201,202,203,204,205};
-Physical Surface(2) = {227,249,271,293,315};
-Physical Surface(3) = {214,236,258,280};
-Physical Volume(1) = {1,2,3,4,5};
-
-// Optimize the high-order mesh
-// See https://gmsh.info/doc/texinfo/gmsh.html#index-Mesh_002eHighOrderOptimize
-// Mesh.ElementOrder = order;
-// Mesh.HighOrderOptimize = 1;
-
-// Generate 3D mesh
-Mesh 3;
-SetOrder order;
-Mesh.MshFileVersion = 2.2;
-
-// Check the element quality (the Plugin may be called AnalyseCurvedMesh)
-// Plugin(AnalyseMeshQuality).JacobianDeterminant = 1;
-// Plugin(AnalyseMeshQuality).Run;
-
-If (periodic)
-   Save Sprintf("periodic-torus-sector-t%01g-o%01g.msh", type, order);
-Else
-   Save Sprintf("torus-sector-t%01g-o%01g.msh", type, order);
-EndIf
@@ -1,155 +0,0 @@
-MFEM NURBS mesh v1.0
-
-dimension
-2
-
-elements
-5
-1 3 0 3 7 4
-1 3 3 2 6 7
-1 3 2 1 5 6
-1 3 1 0 4 5
-1 3 2 8 9 1
-
-boundary
-10
-1 1 0 3
-2 1 3 2
-2 1 1 0
-2 1 2 8
-2 1 9 1
-3 1 7 4
-3 1 6 7
-3 1 5 6
-3 1 4 5
-4 1 8 9
-
-edges
-15
-0 0 4
-0 3 7
-0 1 5
-0 2 6
-1 0 3
-1 4 7
-2 3 2
-2 7 6
-2 1 0
-2 5 4
-1 2 1
-1 6 5
-1 8 9
-3 2 8
-3 1 9
-
-vertices
-10
-
-patches
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 5 1
-5 3.92523e-16 1
-5 -5 1
-2.47593 2.47593 1
-4.95187 6.06429e-16 0.707107
-2.47593 -2.47593 1
-0.424264 0.424264 1
-0.848528 1.03915e-16 0.707107
-0.424264 -0.424264 1
-0.353553 0.353553 1
-0.707107 8.65956e-17 0.707107
-0.353553 -0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 -5 1
-1.17757e-15 -5 1
-5 -5 1
-2.47593 -2.47593 1
-9.09644e-16 -4.95187 0.707107
-2.47593 -2.47593 1
-0.424264 -0.424264 1
-1.55872e-16 -0.848528 0.707107
-0.424264 -0.424264 1
-0.353553 -0.353553 1
-1.29893e-16 -0.707107 0.707107
-0.353553 -0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 -5 1
-5 -1.17757e-15 1
-5 5 1
-2.47593 -2.47593 1
-4.95187 -1.21286e-15 0.707107
-2.47593 2.47593 1
-0.424264 -0.424264 1
-0.848528 -2.07829e-16 0.707107
-0.424264 0.424264 1
-0.353553 -0.353553 1
-0.707107 -1.73191e-16 0.707107
-0.353553 0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 4 0 0 0 0.5 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 5 1
-3.92523e-16 5 1
-5 5 1
-2.47593 2.47593 1
-3.03215e-16 4.95187 0.707107
-2.47593 2.47593 1
-0.424264 0.424264 1
-5.19574e-17 0.848528 0.707107
-0.424264 0.424264 1
-0.353553 0.353553 1
-4.32978e-17 0.707107 0.707107
-0.353553 0.353553 1
-
-knotvectors
-2
-2 3 0 0 0 1 1 1
-2 3 0 0 0 1 1 1
-
-dimension
-2
-
-controlpoints_cartesian
-5 -5 1
-10 -5 1
-15 -5 1
-5 0 1
-10 0 1
-15 0 1
-5 5 1
-10 5 1
-15 5 1
@@ -1,118 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-20
-1 3 0 1 6 5
-1 3 1 2 7 6
-1 3 2 3 8 7
-1 3 3 4 9 8
-1 3 5 6 11 10
-1 2 6 7 11
-1 2 7 12 11
-1 2 7 8 13
-1 2 7 13 12
-1 3 8 9 14 13
-1 3 10 11 16 15
-1 2 11 12 17
-1 2 11 17 16
-1 2 12 13 17
-1 2 13 18 17
-1 3 13 14 19 18
-1 3 15 16 21 20
-1 3 16 17 22 21
-1 3 17 18 23 22
-1 3 18 19 24 23
-
-boundary
-16
-2 1 0 1
-2 1 1 2
-2 1 2 3
-2 1 3 4
-2 1 21 20
-2 1 22 21
-2 1 23 22
-2 1 24 23
-1 1 5 0
-1 1 10 5
-1 1 15 10
-1 1 20 15
-1 1 4 9
-1 1 9 14
-1 1 14 19
-1 1 19 24
-
-vertices
-25
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 0
-
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0.25
-0.5
-0.75
-1
-0
-0
-0
-0
-0
-0.25
-0.25
-0.25
-0.25
-0.25
-0.5
-0.5
-0.5
-0.5
-0.5
-0.75
-0.75
-0.75
-0.75
-0.75
-1
-1
-1
-1
-1
@@ -16,21 +16,36 @@ if (DOXYGEN_FOUND)
  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CodeDocumentation.conf.in
    ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf @ONLY)

+  if (UNIX)
+    # Only create symlinks if UNIX operating system
+    add_custom_target(doc
+      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
+      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+      COMMAND ${CMAKE_COMMAND} -E create_symlink
+      ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+      ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+      BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating API documentation with Doxygen to CodeDocumentation.html"
+      VERBATIM)

-  add_custom_target(doc
-    COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
-    COMMAND echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-    BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
-    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-    COMMENT "Generating API documentation with Doxygen to CodeDocumentation.html"
-    VERBATIM)
-
-  add_custom_target(clean-doc
-    COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
-    COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/warnings.log
-    COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
-    COMMENT "Removing API documentation"
-    VERBATIM)
+    add_custom_target(clean-doc
+      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.html
+      COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
+      COMMENT "Removing API documentation"
+      VERBATIM)

+  else (UNIX)
+    add_custom_target(doc
+      COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation.conf
+      BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation/html/index.html
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+      COMMENT "Generating API documentation with Doxygen to CodeDocumentation/html/index.html"
+      VERBATIM)

+    add_custom_target(clean-doc
+      COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_CURRENT_BINARY_DIR}/CodeDocumentation
+      COMMENT "Removing API documentation"
+      VERBATIM)
+  endif (UNIX)
 endif (DOXYGEN_FOUND)
@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "Finite element discretization library"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.

-PROJECT_LOGO           = web/logo-small.png
+PROJECT_LOGO           =

 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -746,7 +746,7 @@ WARN_FORMAT            = "$file:$line: $text"
 # messages should be written. If left blank the output is written to standard
 # error (stderr).

-WARN_LOGFILE           = warnings.log
+WARN_LOGFILE           =

 #---------------------------------------------------------------------------
 # Configuration options related to the input files
@@ -770,7 +770,6 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/examples/pumi \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/sundials \
-                         @MFEM_SOURCE_DIR@/miniapps/adjoint \			 
                         @MFEM_SOURCE_DIR@/miniapps/common \
                         @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                         @MFEM_SOURCE_DIR@/miniapps/gslib \
@@ -1471,7 +1470,7 @@ MATHJAX_FORMAT         = HTML-CSS
 # The default value is: http://cdn.mathjax.org/mathjax/latest.
 # This tag requires that the tag USE_MATHJAX is set to YES.

-MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_RELPATH        = https://cdn.llnl.gov/mathjax/2.7.2

 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
@@ -88,8 +88,8 @@ namespace mfem {
 * - <a class="el" href="ex24p_8cpp_source.html">Example 24p</a>: parallel mixed finite element spaces and interpolators
 * - <a class="el" href="ex25_8cpp_source.html">Example 25</a>: simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
 * - <a class="el" href="ex25p_8cpp_source.html">Example 25p</a>: parallel simulation of electromagnetic wave propagation using a Perfectly Matched Layer (PML)
- * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM
- * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM
+ * - <a class="el" href="ex26_8cpp_source.html">Example 26</a>: multigrid preconditioner for the Laplace problem using nodal H1 FEM 
+ * - <a class="el" href="ex26p_8cpp_source.html">Example 26p</a>: parallel multigrid preconditioner for the Laplace problem using nodal H1 FEM 
 *
 * <H4>SUNDIALS Examples</H4>
 * - Variants of Examples
@@ -101,9 +101,6 @@ namespace mfem {
 *   and
 *   <a class="el" href="sundials_2ex16p_8cpp_source.html">16p</a>
 *   demonstrating the use of MFEM's \link sundials.hpp SUNDIALS classes\endlink
- * - CVODES adjoint examples:
- *   <a class="el" href="cvsRoberts__ASAi__dns_8cpp_source.html">serial ODE system</a>,
- *   <a class="el" href="adjoint__advection__diffusion_8cpp_source.html">parallel advection-diffusion</a>
 *
 * <H4>PETSc Examples</H4>
 * - Variants of Examples
@@ -143,18 +140,15 @@ namespace mfem {
 * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
 * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
 * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
 * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
 * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces
 * - <a class="el" href="toroid_8cpp_source.html">Toroid</a>: generate simple toroidal meshes
 * - <a class="el" href="twist_8cpp_source.html">Twist</a>: generate simple periodic meshes
 * - <a class="el" href="minimal-surface_8cpp_source.html">Minimal Surface</a>: compute minimal surfaces, <a class="el" href="minimal-surface_8cpp_source.html">serial</a> and <a class="el" href="pminimal-surface_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="polar-nc_8cpp_source.html">Polar NC</a>: generate polar non-conforming meshes
 * - <a class="el" href="shaper_8cpp_source.html">Shaper</a>: resolve material interfaces by mesh refinement
 * - <a class="el" href="extruder_8cpp_source.html">Extruder</a>: extrude a low-dimensional mesh into a higher dimension
 * - <a class="el" href="mesh-explorer_8cpp_source.html">Mesh Explorer</a>: visualize and manipulate meshes
 * - <a class="el" href="mesh-optimizer_8cpp_source.html">Mesh Optimizer</a>: optimize high-order meshes, <a class="el" href="mesh-optimizer_8cpp_source.html">serial</a> and <a class="el" href="pmesh-optimizer_8cpp_source.html">parallel</a> versions
- * - <a class="el" href="trimmer_8cpp_source.html">Trimmer</a>: trim elements from existing meshes
 * - <a class="el" href="display-basis_8cpp_source.html">Display Basis</a>: visualize finite element basis functions
 * - <a class="el" href="get-values_8cpp_source.html">Get Values</a>: extract field values via DataCollection classes
 * - <a class="el" href="load-dc_8cpp_source.html">Load DC</a>: visualize fields saved via DataCollection classes
@@ -162,7 +156,7 @@ namespace mfem {
 * - <a class="el" href="lor-transfer_8cpp_source.html">LOR Transfer</a>: map functions between high-order and low-order refined spaces
 * - <a class="el" href="findpts_8cpp_source.html">Find Points</a>: evaluate grid function in physical space, <a class="el" href="findpts_8cpp_source.html">serial</a> and <a class="el" href="pfindpts_8cpp_source.html">parallel</a> versions
 * - <a class="el" href="field-diff_8cpp_source.html">Field Diff</a>: compare grid functions on different meshes
- * - <a class="el" href="field-interp_8cpp_source.html">Field Interp</a>: transfer a grid functions betwen meshes
+ * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
 * - <a class="el" href="miniapps_2performance_2ex1_8cpp_source.html">HPC Example 1</a>: high-performance nodal H1 FEM for the Laplace problem
 * - <a class="el" href="miniapps_2performance_2ex1p_8cpp_source.html">HPC Example 1p</a>: high-performance parallel nodal H1 FEM for the Laplace problem
 *
@@ -9,25 +9,18 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-SHELL = /bin/bash
 MFEM_DIR ?= ..
 DOXYGEN_CONF = CodeDocumentation.conf

-
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
-	@# Generate the html documentation
-	@doxygen $(DOXYGEN_CONF)
-	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
-	@cat warnings.log 1>&2
-	@# Generate the log of undocumented methods
-	@( cat $(DOXYGEN_CONF) ; echo "GENERATE_HTML=NO" ; echo "EXTRACT_ALL=NO" ; echo "WARN_LOGFILE=undoc.log" ; echo "QUIET=YES" ) | doxygen - &> /dev/null
+	doxygen $(DOXYGEN_CONF)
+	rm -f CodeDocumentation.html
+	ln -s CodeDocumentation/html/index.html CodeDocumentation.html

 clean:
 	rm -rf $(DOXYGEN_CONF) CodeDocumentation CodeDocumentation.html *~
-	rm -rf  undoc.log warnings.log

 $(DOXYGEN_CONF): $(MFEM_DIR)/doc/$(DOXYGEN_CONF).in
-	@sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
+	sed -e 's%@MFEM_SOURCE_DIR@%$(MFEM_DIR)%g' $(<) \
 	  > $(DOXYGEN_CONF)
-
@@ -64,6 +64,8 @@ if (MFEM_USE_MPI)
    ex25p.cpp
    ex26p.cpp
    ex27p.cpp
+    pa_oper.cpp
+    io_benchmark.cpp
    )
 endif()

@@ -91,7 +93,7 @@ foreach(SRC_FILE ${ALL_EXE_SRCS})
    add_test(NAME ${TEST_NAME}_ser
      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
+    add_test(NAME ${TEST_NAME}_np=4
      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
      ${MPIEXEC_PREFLAGS}
      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
@@ -101,22 +103,13 @@ endforeach()

 # If STRUMPACK is enabled, add a test run that uses it.
 if (MFEM_USE_STRUMPACK)
-  add_test(NAME ex11p_strumpack_np=${MFEM_MPI_NP}
+  add_test(NAME ex11p_strumpack_np=4
    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
    ${MPIEXEC_PREFLAGS}
    $<TARGET_FILE:ex11p> "-no-vis" "--strumpack"
    ${MPIEXEC_POSTFLAGS})
 endif()

-# If SuperLU_DIST is enabled, add a test run that uses it.
-if (MFEM_USE_SUPERLU)
-  add_test(NAME ex11p_superlu_np=${MFEM_MPI_NP}
-    COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-    ${MPIEXEC_PREFLAGS}
-    $<TARGET_FILE:ex11p> "-no-vis" "--superlu"
-    ${MPIEXEC_POSTFLAGS})
-endif()
-
 # Include the examples/sundials directory if SUNDIALS is enabled.
 if (MFEM_USE_SUNDIALS)
  add_subdirectory(sundials)
@@ -127,11 +120,6 @@ if (MFEM_USE_GINKGO)
  add_subdirectory(ginkgo)
 endif()

-# Include the examples/amgx directory if AmgX is enabled
-if (MFEM_USE_AMGX)
-  add_subdirectory(amgx)
-endif()
-
 # Include the examples/petsc directory if PETSc is enabled.
 if (MFEM_USE_PETSC)
  add_subdirectory(petsc)
@@ -145,3 +133,4 @@ endif()
 if (MFEM_USE_HIOP)
  add_subdirectory(hiop)
 endif()
+
@@ -1,79 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(AMGX_EXAMPLES_SRCS)
-list(APPEND AMGX_EXAMPLES_SRCS
-  ex1.cpp
-  )
-
-if (MFEM_USE_MPI)
-  list(APPEND AMGX_EXAMPLES_SRCS
-    ex1p.cpp
-    )
-endif()
-
-set(AMGX_JSON_FILES amg_pcg.json multi_gs.json precon.json)
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add targets to copy *.json files from the source directory
-foreach(JSON_FILE ${AMGX_JSON_FILES})
-  add_custom_command(OUTPUT ${JSON_FILE}
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-    ${CMAKE_CURRENT_SOURCE_DIR}/${JSON_FILE} ${JSON_FILE}
-    COMMENT "copy ${JSON_FILE}")
-endforeach()
-add_custom_target(copy_amgx_json_files DEPENDS ${AMGX_JSON_FILES}
-  COMMENT "Copying AMGX example json files ...")
-
-# Add "test_amgx" target, see below.
-add_custom_target(test_amgx
-  ${CMAKE_CTEST_COMMAND} -R amgx USES_TERMINAL)
-
-# Add one executable per cpp file, adding "amgx_" as prefix. Sets
-# "copy_amgx_json_files" as a prerequisite for the given examples. Also, sets
-# "test_amgx" as a target that depends on the given examples.
-set(PFX amgx_)
-add_mfem_examples(AMGX_EXAMPLES_SRCS ${PFX} copy_amgx_json_files test_amgx)
-
-# Testing.
-# The AMGX tests can be run separately using the target "test_amgx"
-# which builds the examples and runs:
-#   ctest -R amgx
-
-# Command line options for the tests.
-# Example 1/1p:
-set(EX1_TEST_OPTS)
-set(EX1P_TEST_OPTS)
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${AMGX_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on NVIDIA's multigrid library AmgX.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_AMGX = YES", see the top-level INSTALL file for details (version
-2.1 of AmgX is recommended, though older versions may work too.
-
-We recommend comparing the original example codes with the corresponding files
-in the current directory.
@@ -1,38 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "preconditioner": {
-            "print_grid_stats": 1,
-            "print_vis_data": 0,
-            "solver": "AMG",
-            "smoother": {
-                "scope": "jacobi",
-                "solver": "BLOCK_JACOBI",
-                "relaxation_factor": 0.7,
-                "monitor_residual": 0,
-                "print_solve_stats": 0
-            },
-            "print_solve_stats": 0,
-            "presweeps": 1,
-            "interpolator": "D2",
-            "max_row_sum" : 0.9,
-            "strength_threshold" : 0.25,
-            "max_iters": 2,
-            "monitor_residual": 0,
-            "store_res_history": 0,
-            "scope": "amg",
-            "max_levels": 100,
-            "cycle": "V",
-            "postsweeps": 1
-        },
-        "solver": "PCG",
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "max_iters": 100,
-        "monitor_residual": 1,
-        "convergence": "RELATIVE_MAX",
-        "scope": "main",
-        "tolerance": 1e-12,
-        "norm": "L2"
-    }
-}
@@ -1,255 +0,0 @@
-//                                MFEM Example 1
-//                              AmgX Modification
-//
-// Compile with: make ex1
-//
-// AmgX sample runs:
-//               ex1
-//               ex1 -d cuda
-//               ex1 --amgx-file multi_gs.json --amgx-solver
-//               ex1 --amgx-file precon.json --amgx-preconditioner
-//               ex1 --amgx-file multi_gs.json --amgx-solver -d cuda
-//               ex1 --amgx-file precon.json --amgx-preconditioner -d cuda
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   bool pa = false;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_lib = true;
-   bool amgx_solver = true;
-   const char* amgx_json_file = ""; // JSON file for AmgX
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
-                  "--no-amgx-lib", "Use AmgX in example.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_solver, "--amgx-solver", "--amgx-solver",
-                  "--amgx-preconditioner", "--amgx-preconditioner",
-                  "Configure AMGX as solver or preconditioner.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution. In this example we do
-   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
-   //    largest number that gives a final mesh with no more than 50,000
-   //    elements.
-   {
-      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (mesh.GetNodes())
-   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      cout << "Using isoparametric FEs: " << fec->Name() << endl;
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   //    In this example, the boundary conditions are defined by marking all
-   //    the boundary attributes from the mesh as essential (Dirichlet) and
-   //    converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
-   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
-   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 8. Define the solution vector x as a finite element grid function
-   //    corresponding to fespace. Initialize x with initial guess of zero,
-   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
-   x = 0.0;
-
-   // 9. Set up the bilinear form a(.,.) on the finite element space
-   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 10. Assemble the bilinear form and the corresponding linear system,
-   //     applying any necessary transformations such as: eliminating boundary
-   //     conditions, applying conforming constraints for non-conforming AMR,
-   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   cout << "Size of linear system: " << A->Height() << endl;
-
-   // 11. Solve the linear system A X = B.
-   if (pa)
-   {
-      // Jacobi preconditioning in partial assembly mode
-      if (UsesTensorBasis(fespace))
-      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 400, 1e-12, 0.0);
-      }
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
-   {
-      bool amgx_verbose = false;
-      AmgXSolver amgx(AmgXSolver::PRECONDITIONER, amgx_verbose);
-      amgx.SetOperator(*A.As<SparseMatrix>());
-      PCG(*A, amgx, B, X, 1, 200, 1e-12, 0.0);
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
-   {
-      AmgXSolver amgx;
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-      amgx.InitSerial();
-      amgx.SetOperator(*A.As<SparseMatrix>());
-
-      if (amgx_solver)
-      {
-         amgx.Mult(B,X);
-      }
-      else
-      {
-         PCG(*A.As<SparseMatrix>(), amgx, B, X, 3, 40, 1e-12, 0.0);
-      }
-   }
-   else
-   {
-#ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
-#else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
-#endif
-   }
-
-   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 13. Save the refined mesh and the solution. This output can be viewed later
-   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
-   ofstream mesh_ofs("refined.mesh");
-   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
-   ofstream sol_ofs("sol.gf");
-   sol_ofs.precision(8);
-   x.Save(sol_ofs);
-
-   // 14. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
-   }
-
-   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-
-   return 0;
-}
@@ -1,75 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/amgx/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_AMGX),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with AMGX)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial AMGX example
-PARALLEL_NAME := Parallel AMGX example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-%-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not build)
-
-clean: clean-build
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f .logamgx refined.mesh sol.gf mesh.* sol.*
@@ -1,24 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 10000,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0000001,
-        "print_solve_stats": 1,
-        "obtain_timings": 1,
-        "monitor_residual": 1,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}
@@ -1,21 +0,0 @@
-{
-    "config_version": 2,
-    "solver": {
-        "max_uncolored_percentage": 0.15,
-        "algorithm": "AGGREGATION",
-        "solver": "AMG",
-        "smoother": "MULTICOLOR_GS",
-        "presweeps": 1,
-        "symmetric_GS" : 1,
-        "selector": "SIZE_2",
-        "coarsest_sweeps": 10,
-        "max_iters": 2,
-        "postsweeps": 1,
-        "scope": "main",
-        "max_levels": 1000,
-        "matrix_coloring_scheme" : "MIN_MAX",
-        "tolerance": 0.0,
-        "norm": "L2",
-        "cycle": "V"
-    }
-}
@@ -9,8 +9,6 @@
 //               ex1 -m ../data/fichera.mesh
 //               ex1 -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/periodic-annulus-sector.msh
-//               ex1 -m ../data/periodic-torus-sector.msh
 //               ex1 -m ../data/square-disc-p2.vtk -o 2
 //               ex1 -m ../data/square-disc-p3.mesh -o 3
 //               ex1 -m ../data/square-disc-nurbs.mesh -o -1
@@ -34,9 +32,7 @@
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
-//             * ex1 -pa -d ceed-cuda
-//             * ex1 -pa -d ceed-hip
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -pa -d ceed-cuda
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -104,8 +100,8 @@ int main(int argc, char *argv[])
   // 3. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -113,10 +109,10 @@ int main(int argc, char *argv[])
   //    elements.
   {
      int ref_levels =
-         (int)floor(log(50000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(50000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

@@ -124,70 +120,66 @@ int main(int argc, char *argv[])
   //    Lagrange finite elements of the specified order. If order < 1, we
   //    instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (mesh.GetNodes())
+   else if (mesh->GetNodes())
   {
-      fec = mesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = mesh->GetNodes()->OwnFEC();
      cout << "Using isoparametric FEs: " << fec->Name() << endl;
   }
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   FiniteElementSpace fespace(&mesh, fec);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
+        << fespace->GetTrueVSize() << endl;

   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
+   if (mesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      Array<int> ess_bdr(mesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system, which in this case is (1,phi_i) where phi_i are
   //    the basis functions in the finite element fespace.
-   LinearForm b(&fespace);
+   LinearForm *b = new LinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 8. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x with initial guess of zero,
   //    which satisfies the boundary conditions.
-   GridFunction x(&fespace);
+   GridFunction x(fespace);
   x = 0.0;

   // 9. Set up the bilinear form a(.,.) on the finite element space
   //    corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //    domain integrator.
-   BilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   BilinearForm *a = new BilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
   //     applying any necessary transformations such as: eliminating boundary
   //     conditions, applying conforming constraints for non-conforming AMR,
   //     static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   cout << "Size of linear system: " << A->Height() << endl;

@@ -208,9 +200,9 @@ int main(int argc, char *argv[])
   }
   else // Jacobi preconditioning in partial assembly mode
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
+         OperatorJacobiSmoother M(*a, ess_tdof_list);
         PCG(*A, M, B, X, 1, 400, 1e-12, 0.0);
      }
      else
@@ -220,13 +212,13 @@ int main(int argc, char *argv[])
   }

   // 12. Recover the solution as a finite element grid function.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

   // 13. Save the refined mesh and the solution. This output can be viewed later
   //     using GLVis: "glvis -m refined.mesh -g sol.gf".
   ofstream mesh_ofs("refined.mesh");
   mesh_ofs.precision(8);
-   mesh.Print(mesh_ofs);
+   mesh->Print(mesh_ofs);
   ofstream sol_ofs("sol.gf");
   sol_ofs.precision(8);
   x.Save(sol_ofs);
@@ -238,14 +230,15 @@ int main(int argc, char *argv[])
      int  visport   = 19916;
      socketstream sol_sock(vishost, visport);
      sol_sock.precision(8);
-      sol_sock << "solution\n" << mesh << x << flush;
+      sol_sock << "solution\n" << *mesh << x << flush;
   }

   // 15. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete mesh;

   return 0;
 }
@@ -8,8 +8,6 @@
 //               mpirun -np 4 ex11p -m ../data/escher.mesh
 //               mpirun -np 4 ex11p -m ../data/fichera.mesh
 //               mpirun -np 4 ex11p -m ../data/fichera-mixed.mesh
-//               mpirun -np 4 ex11p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex11p -m ../data/periodic-torus-sector.msh -rs 1
 //               mpirun -np 4 ex11p -m ../data/toroid-wedge.mesh -o 2
 //               mpirun -np 4 ex11p -m ../data/square-disc-p2.vtk -o 2
 //               mpirun -np 4 ex11p -m ../data/square-disc-p3.mesh -o 3
@@ -72,7 +70,6 @@ int main(int argc, char *argv[])
   int seed = 75;
   bool slu_solver  = false;
   bool sp_solver = false;
-   bool cpardiso_solver = false;
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -96,10 +93,6 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
                  "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-#ifdef MFEM_USE_MKL_CPARDISO
-   args.AddOption(&cpardiso_solver, "-cpardiso", "--cpardiso", "-no-cpardiso",
-                  "--no-cpardiso", "Use the MKL CPardiso Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -241,7 +234,7 @@ int main(int argc, char *argv[])
   //    preconditioner for A to be used within the solver. Set the matrices
   //    which define the generalized eigenproblem A x = lambda M x.
   Solver * precond = NULL;
-   if (!slu_solver && !sp_solver && !cpardiso_solver)
+   if (!slu_solver && !sp_solver)
   {
      HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
      amg->SetPrintLevel(0);
@@ -273,19 +266,10 @@ int main(int argc, char *argv[])
         strumpack->SetFromCommandLine();
         precond = strumpack;
      }
-#endif
-#ifdef MFEM_USE_MKL_CPARDISO
-      if (cpardiso_solver)
-      {
-         auto cpardiso = new CPardisoSolver(A->GetComm());
-         cpardiso->SetMatrixType(CPardisoSolver::MatType::REAL_STRUCTURE_SYMMETRIC);
-         cpardiso->SetPrintLevel(1);
-         cpardiso->SetOperator(*A);
-         precond = cpardiso;
-      }
 #endif
   }

+
   HypreLOBPCG * lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
   lobpcg->SetNumModes(nev);
   lobpcg->SetRandomSeed(seed);
@@ -35,38 +35,6 @@
 using namespace std;
 using namespace mfem;

-class CustomSolverMonitor : public IterativeSolverMonitor
-{
-public:
-   CustomSolverMonitor(const ParMesh *m,
-                       ParGridFunction *f) :
-      pmesh(m),
-      pgf(f) {}
-
-   void MonitorSolution(int i, double norm, const Vector &x, bool final)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      int  num_procs, myid;
-
-      MPI_Comm_size(pmesh->GetComm(),&num_procs);
-      MPI_Comm_rank(pmesh->GetComm(),&myid);
-
-      pgf->SetFromTrueDofs(x);
-
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << *pmesh << *pgf
-               << "window_title 'Iteration no " << i << "'"
-               << "keys rRjlc\n" << flush;
-   }
-
-private:
-   const ParMesh *pmesh;
-   ParGridFunction *pgf;
-};
-
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
@@ -220,7 +188,6 @@ int main(int argc, char *argv[])
   }
   else
   {
-      CustomSolverMonitor monitor(pmesh, &x);
      GMRESSolver gmres(MPI_COMM_WORLD);
      gmres.SetAbsTol(0.0);
      gmres.SetRelTol(1e-12);
@@ -229,7 +196,6 @@ int main(int argc, char *argv[])
      gmres.SetPrintLevel(1);
      gmres.SetOperator(*A);
      gmres.SetPreconditioner(*amg);
-      gmres.SetMonitor(monitor);
      gmres.Mult(*B, *X);
   }
   delete amg;
@@ -88,6 +88,8 @@ private:
   Vector funval2;
   Vector nor;
   Vector fluxN;
+   IntegrationPoint eip1;
+   IntegrationPoint eip2;

 public:
   FaceIntegrator(RiemannSolver &rsolver_, const int dim);
@@ -416,24 +418,27 @@ void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
   {
      intorder++;
   }
-   const IntegrationRule *ir = &IntRules.Get(Tr.GetGeometryType(), intorder);
+   const IntegrationRule *ir = &IntRules.Get(Tr.FaceGeom, intorder);

   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);

-      Tr.SetAllIntPoints(&ip); // set face and element int. points
+      Tr.Loc1.Transform(ip, eip1);
+      Tr.Loc2.Transform(ip, eip2);

      // Calculate basis functions on both elements at the face
-      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
+      el1.CalcShape(eip1, shape1);
+      el2.CalcShape(eip2, shape2);

      // Interpolate elfun at the point
      elfun1_mat.MultTranspose(shape1, funval1);
      elfun2_mat.MultTranspose(shape2, funval2);

+      Tr.Face->SetIntPoint(&ip);
+
      // Get the normal vector and the flux on the face
-      CalcOrtho(Tr.Jacobian(), nor);
+      CalcOrtho(Tr.Face->Jacobian(), nor);
      const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);

      // Update max char speed
@@ -8,7 +8,6 @@
 //    ex19 -m ../data/beam-hex.mesh
 //    ex19 -m ../data/beam-tet.mesh
 //    ex19 -m ../data/beam-wedge.mesh
-//    ex19 -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -39,42 +38,6 @@
 using namespace std;
 using namespace mfem;

-class GeneralResidualMonitor : public IterativeSolverMonitor
-{
-public:
-   GeneralResidualMonitor(const std::string& prefix_, int print_lvl)
-      : prefix(prefix_)
-   {
-      print_level = print_lvl;
-   }
-
-   virtual void MonitorResidual(int it, double norm, const Vector &r, bool final);
-
-private:
-   const std::string prefix;
-   int print_level;
-   mutable double norm0;
-};
-
-void GeneralResidualMonitor::MonitorResidual(int it, double norm,
-                                             const Vector &r, bool final)
-{
-   if (print_level == 1 || (print_level == 3 && (final || it == 0)))
-   {
-      mfem::out << prefix << " iteration " << setw(2) << it
-                << " : ||r|| = " << norm;
-      if (it > 0)
-      {
-         mfem::out << ",  ||r||/||r_0|| = " << norm/norm0;
-      }
-      else
-      {
-         norm0 = norm;
-      }
-      mfem::out << '\n';
-   }
-}
-
 // Custom block preconditioner for the Jacobian of the incompressible nonlinear
 // elasticity operator. It has the form
 //
@@ -97,7 +60,7 @@ protected:
   Array<FiniteElementSpace *> spaces;

   // Offsets for extracting block vector segments
-   Array<int> &block_trueOffsets;
+   Array<int> &block_offsets;

   // Jacobian for block access
   BlockOperator *jacobian;
@@ -140,11 +103,9 @@ protected:

   // Newton solver for the hyperelastic operator
   NewtonSolver newton_solver;
-   GeneralResidualMonitor newton_monitor;

   // Solver for the Jacobian solve in the Newton method
   Solver *j_solver;
-   GeneralResidualMonitor j_monitor;

   // Preconditioner for the Jacobian
   Solver *j_prec;
@@ -153,7 +114,7 @@ protected:
   Coefficient &mu;

   // Block offsets for variable access
-   Array<int> &block_trueOffsets;
+   Array<int> &block_offsets;

 public:
   RubberOperator(Array<FiniteElementSpace *> &fes, Array<Array<int> *>&ess_bdr,
@@ -247,8 +208,8 @@ int main(int argc, char *argv[])
   spaces[0] = &R_space;
   spaces[1] = &W_space;

-   int R_size = R_space.GetTrueVSize();
-   int W_size = W_space.GetTrueVSize();
+   int R_size = R_space.GetVSize();
+   int W_size = W_space.GetVSize();

   // 6. Define the Dirichlet conditions (set to boundary attribute 1 and 2)
   Array<Array<int> *> ess_bdr(2);
@@ -272,13 +233,13 @@ int main(int argc, char *argv[])
   std::cout << "***********************************************************\n";

   // 8. Define the block structure of the solution vector (u then p)
-   Array<int> block_trueOffsets(3);
-   block_trueOffsets[0] = 0;
-   block_trueOffsets[1] = R_space.GetTrueVSize();
-   block_trueOffsets[2] = W_space.GetTrueVSize();
-   block_trueOffsets.PartialSum();
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = R_space.GetVSize();
+   block_offsets[2] = W_space.GetVSize();
+   block_offsets.PartialSum();

-   BlockVector xp(block_trueOffsets);
+   BlockVector xp(block_offsets);

   // 9. Define grid functions for the current configuration, reference
   //    configuration, final deformation, and pressure
@@ -287,8 +248,8 @@ int main(int argc, char *argv[])
   GridFunction x_def(&R_space);
   GridFunction p_gf(&W_space);

-   x_gf.MakeTRef(&R_space, xp.GetBlock(0), 0);
-   p_gf.MakeTRef(&W_space, xp.GetBlock(1), 0);
+   x_gf.MakeRef(&R_space, xp.GetBlock(0), 0);
+   p_gf.MakeRef(&W_space, xp.GetBlock(1), 0);

   VectorFunctionCoefficient deform(dim, InitialDeformation);
   VectorFunctionCoefficient refconfig(dim, ReferenceConfiguration);
@@ -297,19 +258,14 @@ int main(int argc, char *argv[])
   x_ref.ProjectCoefficient(refconfig);
   p_gf = 0.0;

-   x_gf.SetTrueVector();
-   p_gf.SetTrueVector();
-
   // 10. Initialize the incompressible neo-Hookean operator
-   RubberOperator oper(spaces, ess_bdr, block_trueOffsets,
+   RubberOperator oper(spaces, ess_bdr, block_offsets,
                       newton_rel_tol, newton_abs_tol, newton_iter, c_mu);

   // 11. Solve the Newton system
   oper.Solve(xp);

   // 12. Compute the final deformation
-   x_gf.SetFromTrueVector();
-   p_gf.SetFromTrueVector();
   subtract(x_gf, x_ref, x_def);

   // 13. Visualize the results if requested
@@ -355,7 +311,7 @@ int main(int argc, char *argv[])
 JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
                                               SparseMatrix &mass,
                                               Array<int> &offsets)
-   : Solver(offsets[2]), block_trueOffsets(offsets), pressure_mass(&mass)
+   : Solver(offsets[2]), block_offsets(offsets), pressure_mass(&mass)
 {
   fes.Copy(spaces);

@@ -387,18 +343,18 @@ JacobianPreconditioner::JacobianPreconditioner(Array<FiniteElementSpace *> &fes,
 void JacobianPreconditioner::Mult(const Vector &k, Vector &y) const
 {
   // Extract the blocks from the input and output vectors
-   Vector disp_in(k.GetData() + block_trueOffsets[0],
-                  block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_in(k.GetData() + block_trueOffsets[1],
-                  block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_in(k.GetData() + block_offsets[0],
+                  block_offsets[1]-block_offsets[0]);
+   Vector pres_in(k.GetData() + block_offsets[1],
+                  block_offsets[2]-block_offsets[1]);

-   Vector disp_out(y.GetData() + block_trueOffsets[0],
-                   block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector pres_out(y.GetData() + block_trueOffsets[1],
-                   block_trueOffsets[2]-block_trueOffsets[1]);
+   Vector disp_out(y.GetData() + block_offsets[0],
+                   block_offsets[1]-block_offsets[0]);
+   Vector pres_out(y.GetData() + block_offsets[1],
+                   block_offsets[2]-block_offsets[1]);

-   Vector temp(block_trueOffsets[1]-block_trueOffsets[0]);
-   Vector temp2(block_trueOffsets[1]-block_trueOffsets[0]);
+   Vector temp(block_offsets[1]-block_offsets[0]);
+   Vector temp2(block_offsets[1]-block_offsets[0]);

   // Perform the block elimination for the preconditioner
   mass_pcg->Mult(pres_in, pres_out);
@@ -453,9 +409,8 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
                               double abs_tol,
                               int iter,
                               Coefficient &c_mu)
-   : Operator(fes[0]->GetTrueVSize() + fes[1]->GetTrueVSize()),
-     newton_solver(), newton_monitor("Newton", 1),
-     j_monitor("  GMRES", 3), mu(c_mu), block_trueOffsets(offsets)
+   : Operator(fes[0]->GetVSize() + fes[1]->GetVSize()),
+     newton_solver(), mu(c_mu), block_offsets(offsets)
 {
   Array<Vector *> rhs(2);
   rhs = NULL; // Set all entries in the array
@@ -477,16 +432,12 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   a->AddDomainIntegrator(new MassIntegrator(one));
   a->Assemble();
   a->Finalize();
-
-   OperatorPtr op;
-   Array<int> p_ess_tdofs;
-   a->FormSystemMatrix(p_ess_tdofs, op);
   pressure_mass = a->LoseMat();
   delete a;

   // Initialize the Jacobian preconditioner
   JacobianPreconditioner *jac_prec =
-      new JacobianPreconditioner(fes, *pressure_mass, block_trueOffsets);
+      new JacobianPreconditioner(fes, *pressure_mass, block_offsets);
   j_prec = jac_prec;

   // Set up the Jacobian solver
@@ -495,8 +446,7 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   j_gmres->SetRelTol(1e-12);
   j_gmres->SetAbsTol(1e-12);
   j_gmres->SetMaxIter(300);
-   j_gmres->SetPrintLevel(-1);
-   j_gmres->SetMonitor(j_monitor);
+   j_gmres->SetPrintLevel(0);
   j_gmres->SetPreconditioner(*j_prec);
   j_solver = j_gmres;

@@ -504,8 +454,7 @@ RubberOperator::RubberOperator(Array<FiniteElementSpace *> &fes,
   newton_solver.iterative_mode = true;
   newton_solver.SetSolver(*j_solver);
   newton_solver.SetOperator(*this);
-   newton_solver.SetPrintLevel(-1);
-   newton_solver.SetMonitor(newton_monitor);
+   newton_solver.SetPrintLevel(1);
   newton_solver.SetRelTol(rel_tol);
   newton_solver.SetAbsTol(abs_tol);
   newton_solver.SetMaxIter(iter);
@@ -8,7 +8,6 @@
 //    mpirun -np 2 ex19p -m ../data/beam-hex.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-tet.mesh
 //    mpirun -np 2 ex19p -m ../data/beam-wedge.mesh
-//    mpirun -np 2 ex19p -m ../data/beam-quad-amr.mesh
 //
 // Description:  This examples solves a quasi-static incompressible nonlinear
 //               elasticity problem of the form 0 = H(x), where H is an
@@ -39,56 +38,6 @@
 using namespace std;
 using namespace mfem;

-class GeneralResidualMonitor : public IterativeSolverMonitor
-{
-public:
-   GeneralResidualMonitor(MPI_Comm comm, const std::string& prefix_,
-                          int print_lvl)
-      : prefix(prefix_)
-   {
-#ifndef MFEM_USE_MPI
-      print_level = print_lvl;
-#else
-      int rank;
-      MPI_Comm_rank(comm, &rank);
-      if (rank == 0)
-      {
-         print_level = print_lvl;
-      }
-      else
-      {
-         print_level = -1;
-      }
-#endif
-   }
-
-   virtual void MonitorResidual(int it, double norm, const Vector &r, bool final);
-
-private:
-   const std::string prefix;
-   int print_level;
-   mutable double norm0;
-};
-
-void GeneralResidualMonitor::MonitorResidual(int it, double norm,
-                                             const Vector &r, bool final)
-{
-   if (print_level == 1 || (print_level == 3 && (final || it == 0)))
-   {
-      mfem::out << prefix << " iteration " << setw(2) << it
-                << " : ||r|| = " << norm;
-      if (it > 0)
-      {
-         mfem::out << ",  ||r||/||r_0|| = " << norm/norm0;
-      }
-      else
-      {
-         norm0 = norm;
-      }
-      mfem::out << '\n';
-   }
-}
-
 // Custom block preconditioner for the Jacobian of the incompressible nonlinear
 // elasticity operator. It has the form
 //
@@ -154,11 +103,9 @@ protected:

   // Newton solver for the hyperelastic operator
   NewtonSolver newton_solver;
-   GeneralResidualMonitor newton_monitor;

   // Solver for the Jacobian solve in the Newton method
   Solver *j_solver;
-   GeneralResidualMonitor j_monitor;

   // Preconditioner for the Jacobian
   Solver *j_prec;
@@ -197,8 +144,10 @@ void InitialDeformation(const Vector &x, Vector &y);
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI
-   MPI_Session mpi;
-   const int myid = mpi.WorldRank();
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);

   // 2. Parse command-line options
   const char *mesh_file = "../data/beam-tet.mesh";
@@ -238,6 +187,7 @@ int main(int argc, char *argv[])
      {
         args.PrintUsage(cout);
      }
+      MPI_Finalize();
      return 1;
   }
   if (myid == 0)
@@ -397,6 +347,8 @@ int main(int argc, char *argv[])
   // 19. Free the used memory
   delete pmesh;

+   MPI_Finalize();
+
   return 0;
 }

@@ -470,11 +422,7 @@ void JacobianPreconditioner::SetOperator(const Operator &op)
   {
      HypreBoomerAMG *stiff_prec_amg = new HypreBoomerAMG();
      stiff_prec_amg->SetPrintLevel(0);
-
-      if (!spaces[0]->GetParMesh()->Nonconforming())
-      {
-         stiff_prec_amg->SetElasticityOptions(spaces[0]);
-      }
+      stiff_prec_amg->SetElasticityOptions(spaces[0]);

      stiff_prec = stiff_prec_amg;

@@ -511,10 +459,7 @@ RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
                               int iter,
                               Coefficient &c_mu)
   : Operator(fes[0]->TrueVSize() + fes[1]->TrueVSize()),
-     newton_solver(fes[0]->GetComm()),
-     newton_monitor(fes[0]->GetComm(), "Newton", 1),
-     j_monitor(fes[0]->GetComm(), "  GMRES", 3),
-     mu(c_mu), block_trueOffsets(trueOffsets)
+     newton_solver(fes[0]->GetComm()), mu(c_mu), block_trueOffsets(trueOffsets)
 {
   Array<Vector *> rhs(2);
   rhs = NULL; // Set all entries in the array
@@ -554,8 +499,7 @@ RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
   j_gmres->SetRelTol(1e-12);
   j_gmres->SetAbsTol(1e-12);
   j_gmres->SetMaxIter(300);
-   j_gmres->SetPrintLevel(-1);
-   j_gmres->SetMonitor(j_monitor);
+   j_gmres->SetPrintLevel(0);
   j_gmres->SetPreconditioner(*j_prec);
   j_solver = j_gmres;

@@ -563,8 +507,7 @@ RubberOperator::RubberOperator(Array<ParFiniteElementSpace *> &fes,
   newton_solver.iterative_mode = true;
   newton_solver.SetSolver(*j_solver);
   newton_solver.SetOperator(*this);
-   newton_solver.SetPrintLevel(-1);
-   newton_solver.SetMonitor(newton_monitor);
+   newton_solver.SetPrintLevel(1);
   newton_solver.SetRelTol(rel_tol);
   newton_solver.SetAbsTol(abs_tol);
   newton_solver.SetMaxIter(iter);
@@ -9,8 +9,6 @@
 //               mpirun -np 4 ex1p -m ../data/fichera.mesh
 //               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../data/periodic-torus-sector.msh
 //               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
 //               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
 //               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
@@ -32,9 +30,7 @@
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
-//             * mpirun -np 4 ex1p -pa -d ceed-cuda
-//             * mpirun -np 4 ex1p -pa -d ceed-hip
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -74,6 +70,7 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   bool visualization = true;
+   int nfiles = 1;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -90,6 +87,7 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&nfiles, "-nf", "--num-files", "Number of files to write.");
   args.Parse();
   if (!args.Good())
   {
@@ -113,8 +111,8 @@ int main(int argc, char *argv[])
   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -122,23 +120,23 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
   {
      int par_ref_levels = 2;
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh.UniformRefinement();
+         pmesh->UniformRefinement();
      }
   }

@@ -146,16 +144,13 @@ int main(int argc, char *argv[])
   //    use continuous Lagrange finite elements of the specified order. If
   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (pmesh.GetNodes())
+   else if (pmesh->GetNodes())
   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = pmesh->GetNodes()->OwnFEC();
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -164,10 +159,9 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec, 1, 0);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -178,44 +172,44 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
+   if (pmesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 9. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
+   ParLinearForm *b = new ParLinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 10. Define the solution vector x as a parallel finite element grid function
   //     corresponding to fespace. Initialize x with initial guess of zero,
   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
+   ParGridFunction x(fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
@@ -223,9 +217,9 @@ int main(int argc, char *argv[])
   Solver *prec = NULL;
   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
      }
   }
   else
@@ -243,22 +237,54 @@ int main(int argc, char *argv[])

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);
+
+   std::string filename("nranks_");
+   filename += to_string(num_procs);
+   filename += ".gf";
+   {
+      double t1;
+      t1 = MPI_Wtime();
+      x.Save(filename.c_str(), nfiles);
+      double t2 = MPI_Wtime();
+
+      if (myid == 0)
+      {
+         err << "elapsed write time: " << t2 - t1 << endl;
+      }
+   }
+   {
+      double t1;
+      t1 = MPI_Wtime();
+      ParGridFunction new_x(fespace, filename.c_str());
+      double t2 = MPI_Wtime();
+      if (myid == 0)
+      {
+         err << "elapsed read time: " << t2 - t1 << endl;
+      }
+      // new_x -= x;
+      // out << "GF difference: " << new_x.Norml1() << endl;
+   }

   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
+      //mesh_name << "mesh." << setfill('0') << setw(6) << myid;
+      sol_name << "sol." << num_procs << setfill('0') << setw(6) << myid;

+      //ofstream mesh_ofs(mesh_name.str().c_str());
+      //mesh_ofs.precision(8);
+      //pmesh->Print(mesh_ofs);
+      double t1 = MPI_Wtime();
      ofstream sol_ofs(sol_name.str().c_str());
      sol_ofs.precision(8);
      x.Save(sol_ofs);
+      double t2 = MPI_Wtime();
+      if (myid == 0)
+      {
+         err << t2 - t1 << endl;
+      }
   }

   // 16. Send the solution by socket to a GLVis server.
@@ -269,14 +295,16 @@ int main(int argc, char *argv[])
      socketstream sol_sock(vishost, visport);
      sol_sock << "parallel " << num_procs << " " << myid << "\n";
      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
+      sol_sock << "solution\n" << *pmesh << x << flush;
   }

   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
   MPI_Finalize();

   return 0;
@@ -6,20 +6,13 @@
 //               ex22 -m ../data/inline-tri.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 1
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa
 //               ex22 -m ../data/inline-quad.mesh -o 3 -p 2
 //               ex22 -m ../data/inline-tet.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 1
 //               ex22 -m ../data/inline-hex.mesh -o 2 -p 2
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa
 //               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               ex22 -m ../data/inline-quad.mesh -o 3 -p 1 -pa -d cuda
-//               ex22 -m ../data/inline-hex.mesh -o 2 -p 2 -pa -d cuda
-//               ex22 -m ../data/star.mesh -r 1 -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -83,8 +76,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -115,10 +106,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -148,18 +135,13 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes
   //    with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase resolution. In this example we do
+   // 3. Refine the mesh to increase resolution. In this example we do
   //    'ref_levels' of uniform refinement where the user specifies
   //    the number of levels with the '-r' option.
   for (int l = 0; l < ref_levels; l++)
@@ -167,7 +149,7 @@ int main(int argc, char *argv[])
      mesh->UniformRefinement();
   }

-   // 5. Define a finite element space on the mesh. Here we use continuous
+   // 4. Define a finite element space on the mesh. Here we use continuous
   //    Lagrange, Nedelec, or Raviart-Thomas finite elements of the specified
   //    order.
   if (dim == 1 && prob != 0 )
@@ -189,7 +171,7 @@ int main(int argc, char *argv[])
   cout << "Number of finite element unknowns: " << fespace->GetTrueVSize()
        << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined based on the type
   //    of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -201,12 +183,12 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side of
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side of
   //    the FEM linear system.
   ComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 8. Define the solution vector u as a complex finite element grid function
+   // 7. Define the solution vector u as a complex finite element grid function
   //    corresponding to fespace. Initialize u with initial guess of 1+0i or
   //    the exact solution if it is known.
   ComplexGridFunction u(fespace);
@@ -228,6 +210,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -280,7 +263,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 9. Set up the sesquilinear form a(.,.) on the finite element space
+   // 8. Set up the sesquilinear form a(.,.) on the finite element space
   //    corresponding to the damped harmonic oscillator operator of the
   //    appropriate type:
   //
@@ -299,7 +282,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   SesquilinearForm *a = new SesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -323,7 +305,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 9a. Set up the bilinear form for the preconditioner corresponding to the
+   // 8a. Set up the bilinear form for the preconditioner corresponding to the
   //     appropriate operator
   //
   //      0) A scalar H1 field
@@ -336,8 +318,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   BilinearForm *pcOp = new BilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-
   switch (prob)
   {
      case 0:
@@ -358,9 +338,9 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 10. Assemble the form and the corresponding linear system, applying any
-   //     necessary transformations such as: assembly, eliminating boundary
-   //     conditions, conforming constraints for non-conforming AMR, etc.
+   // 9. Assemble the form and the corresponding linear system, applying any
+   //    necessary transformations such as: assembly, eliminating boundary
+   //    conditions, conforming constraints for non-conforming AMR, etc.
   a->Assemble();
   pcOp->Assemble();

@@ -368,17 +348,28 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;

-   cout << "Size of linear system: " << A->Width() << endl << endl;
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

-   // 11. Define and apply a GMRES solver for AU=B with a block diagonal
+   {
+      ComplexSparseMatrix * Asp =
+         dynamic_cast<ComplexSparseMatrix*>(A.Ptr());
+
+      cout << "Size of linear system: "
+           << 2 * Asp->real().Width() << endl << endl;
+   }
+
+   // 10. Define and apply a GMRES solver for AU=B with a block diagonal
   //     preconditioner based on the appropriate sparse smoother.
   {
      Array<int> blockOffsets;
      blockOffsets.SetSize(3);
      blockOffsets[0] = 0;
-      blockOffsets[1] = A->Height() / 2;
-      blockOffsets[2] = A->Height() / 2;
+      blockOffsets[1] = PCOp.Ptr()->Height();
+      blockOffsets[2] = PCOp.Ptr()->Height();
      blockOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockOffsets);
@@ -386,31 +377,22 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      double s = 1.0;
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
+         case 0:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+         case 1:
+            pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
+            s = -1.0;
+            break;
+         case 2:
+            pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
+            break;
+
+         default: break; // This should be unreachable
      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->SetDiagonalPolicy(mfem::Operator::DIAG_ONE);
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 1:
-               pc_r = new GSSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            case 2:
-               pc_r = new DSmoother(*PCOp.As<SparseMatrix>());
-               break;
-            default:
-               break; // This should be unreachable
-         }
-      }
-      double s = (prob != 1) ? 1.0 : -1.0;
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
                                s:-s);
@@ -428,7 +410,7 @@ int main(int argc, char *argv[])
      gmres.Mult(B, U);
   }

-   // 12. Recover the solution as a finite element grid function and compute the
+   // 11. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a->RecoverFEMSolution(U, b, u);

@@ -460,7 +442,7 @@ int main(int argc, char *argv[])
      cout << endl;
   }

-   // 13. Save the refined mesh and the solution. This output can be viewed
+   // 12. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -475,7 +457,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 14. Send the solution by socket to a GLVis server.
+   // 13. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -534,7 +516,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Free the used memory.
+   // 14. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -7,19 +7,12 @@
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 3 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa
 //               mpirun -np 4 ex22p -m ../data/inline-tet.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 1
 //               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 2 -p 2
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa
 //               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0
 //
-// Device sample runs:
-//               mpirun -np 4 ex22p -m ../data/inline-quad.mesh -o 1 -p 1 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/inline-hex.mesh -o 1 -p 2 -pa -d cuda
-//               mpirun -np 4 ex22p -m ../data/star.mesh -o 2 -sigma 10.0 -pa -d cuda
-//
 // Description:  This example code demonstrates the use of MFEM to define and
 //               solve simple complex-valued linear systems. It implements three
 //               variants of a damped harmonic oscillator:
@@ -48,6 +41,7 @@
 //               We recommend viewing examples 1, 3 and 4 before viewing this
 //               example.

+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -90,8 +84,6 @@ int main(int argc, char *argv[])
   bool visualization = 1;
   bool herm_conv = true;
   bool exact_sol = true;
-   bool pa = false;
-   const char *device_config = "cpu";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -124,10 +116,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.Parse();
   if (!args.Good())
   {
@@ -164,24 +152,19 @@ int main(int argc, char *argv[])
   ComplexOperator::Convention conv =
      herm_conv ? ComplexOperator::HERMITIAN : ComplexOperator::BLOCK_SYMMETRIC;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors. We
+   // 3. Read the (serial) mesh from the given mesh file on all processors. We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution.
+   // 4. Refine the serial mesh on all processors to increase the resolution.
   for (int l = 0; l < ser_ref_levels; l++)
   {
      mesh->UniformRefinement();
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -191,7 +174,7 @@ int main(int argc, char *argv[])
      pmesh->UniformRefinement();
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use continuous Lagrange, Nedelec, or Raviart-Thomas finite elements of
   //    the specified order.
   if (dim == 1 && prob != 0 )
@@ -219,7 +202,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    based on the type of mesh and the problem type.
   Array<int> ess_tdof_list;
@@ -231,14 +214,14 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system.
   ParComplexLinearForm b(fespace, conv);
   b.Vector::operator=(0.0);

-   // 10. Define the solution vector u as a parallel complex finite element grid
-   //     function corresponding to fespace. Initialize u with initial guess of
-   //     1+0i or the exact solution if it is known.
+   // 9. Define the solution vector u as a parallel complex finite element grid
+   //    function corresponding to fespace. Initialize u with initial guess of
+   //    1+0i or the exact solution if it is known.
   ParComplexGridFunction u(fespace);
   ParComplexGridFunction * u_exact = NULL;
   if (exact_sol) { u_exact = new ParComplexGridFunction(fespace); }
@@ -258,6 +241,7 @@ int main(int argc, char *argv[])
   VectorConstantCoefficient zeroVecCoef(zeroVec);
   VectorConstantCoefficient oneVecCoef(oneVec);

+   u = 0.0;
   switch (prob)
   {
      case 0:
@@ -312,7 +296,7 @@ int main(int argc, char *argv[])
                 << "window_title 'Exact: Imaginary Part'" << flush;
   }

-   // 11. Set up the parallel sesquilinear form a(.,.) on the finite element
+   // 10. Set up the parallel sesquilinear form a(.,.) on the finite element
   //     space corresponding to the damped harmonic oscillator operator of the
   //     appropriate type:
   //
@@ -331,7 +315,6 @@ int main(int argc, char *argv[])
   ConstantCoefficient negMassCoef(omega_ * omega_ * epsilon_);

   ParSesquilinearForm *a = new ParSesquilinearForm(fespace, conv);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -355,7 +338,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 11a. Set up the parallel bilinear form for the preconditioner
+   // 10a. Set up the parallel bilinear form for the preconditioner
   //      corresponding to the appropriate operator
   //
   //      0) A scalar H1 field
@@ -368,7 +351,6 @@ int main(int argc, char *argv[])
   //         -Grad(a Div) - omega^2 b + omega c
   //
   ParBilinearForm *pcOp = new ParBilinearForm(fespace);
-   if (pa) { pcOp->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   switch (prob)
   {
      case 0:
@@ -389,7 +371,7 @@ int main(int argc, char *argv[])
      default: break; // This should be unreachable
   }

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, etc.
@@ -400,22 +382,30 @@ int main(int argc, char *argv[])
   Vector B, U;

   a->FormLinearSystem(ess_tdof_list, u, b, A, U, B);
+   u = 0.0;
+   U = 0.0;
+
+   OperatorHandle PCOp;
+   pcOp->FormSystemMatrix(ess_tdof_list, PCOp);

   if (myid == 0)
   {
+      ComplexHypreParMatrix * Ahyp =
+         dynamic_cast<ComplexHypreParMatrix*>(A.Ptr());
+
      cout << "Size of linear system: "
-           << 2 * fespace->GlobalTrueVSize() << endl << endl;
+           << 2 * Ahyp->real().GetGlobalNumRows() << endl << endl;
   }

-   // 13. Define and apply a parallel FGMRES solver for AU=B with a block
+   // 12. Define and apply a parallel FGMRES solver for AU=B with a block
   //     diagonal preconditioner based on the appropriate multigrid
   //     preconditioner from hypre.
   {
      Array<int> blockTrueOffsets;
      blockTrueOffsets.SetSize(3);
      blockTrueOffsets[0] = 0;
-      blockTrueOffsets[1] = A->Height() / 2;
-      blockTrueOffsets[2] = A->Height() / 2;
+      blockTrueOffsets[1] = PCOp.Ptr()->Height();
+      blockTrueOffsets[2] = PCOp.Ptr()->Height();
      blockTrueOffsets.PartialSum();

      BlockDiagonalPreconditioner BDP(blockTrueOffsets);
@@ -423,34 +413,25 @@ int main(int argc, char *argv[])
      Operator * pc_r = NULL;
      Operator * pc_i = NULL;

-      if (pa)
+      switch (prob)
      {
-         pc_r = new OperatorJacobiSmoother(*pcOp, ess_tdof_list);
-      }
-      else
-      {
-         OperatorHandle PCOp;
-         pcOp->FormSystemMatrix(ess_tdof_list, PCOp);
-         switch (prob)
-         {
-            case 0:
-               pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
-               break;
-            case 1:
+         case 0:
+            pc_r = new HypreBoomerAMG(*PCOp.As<HypreParMatrix>());
+            break;
+         case 1:
+            pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
+            break;
+         case 2:
+            if (dim == 2 )
+            {
               pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               break;
-            case 2:
-               if (dim == 2 )
-               {
-                  pc_r = new HypreAMS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               else
-               {
-                  pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
-               }
-               break;
-            default: break; // This should be unreachable
-         }
+            }
+            else
+            {
+               pc_r = new HypreADS(*PCOp.As<HypreParMatrix>(), fespace);
+            }
+            break;
+         default: break; // This should be unreachable
      }
      pc_i = new ScaledOperator(pc_r,
                                (conv == ComplexOperator::HERMITIAN) ?
@@ -468,7 +449,7 @@ int main(int argc, char *argv[])
      fgmres.SetPrintLevel(1);
      fgmres.Mult(B, U);
   }
-   // 14. Recover the parallel grid function corresponding to U. This is the
+   // 13. Recover the parallel grid function corresponding to U. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(U, b, u);

@@ -503,7 +484,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_r_name, sol_i_name;
@@ -523,7 +504,7 @@ int main(int argc, char *argv[])
      u.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -588,7 +569,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete a;
   delete u_exact;
   delete pcOp;
@@ -6,8 +6,6 @@
 //               ex24 -m ../data/square-disc.mesh -o 2
 //               ex24 -m ../data/beam-tet.mesh
 //               ex24 -m ../data/beam-hex.mesh -o 2 -pa
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               ex24 -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               ex24 -m ../data/escher.mesh
 //               ex24 -m ../data/escher.mesh -o 2
 //               ex24 -m ../data/fichera.mesh
@@ -25,16 +23,11 @@
 //               ex24 -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces. Using two different approaches, we project a gradient
+//               of a function in H^1 to H(curl). Other spaces and example
+//               computations are to be added in the future.
 //
-//               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
-//
-//               Using different approaches, we project the gradient, curl, or
-//               divergence to the appropriate space.
-//
-//               We recommend viewing examples 1, 3, and 5 before viewing this
+//               We recommend viewing examples 1 and 3 before viewing this
 //               example.

 #include "mfem.hpp"
@@ -46,19 +39,14 @@ using namespace mfem;

 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
-double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
   const char *mesh_file = "../data/beam-hex.mesh";
   int order = 1;
-   int prob = 0;
   bool static_cond = false;
   bool pa = false;
   const char *device_config = "cpu";
@@ -69,8 +57,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -88,7 +74,6 @@ int main(int argc, char *argv[])
      return 1;
   }
   args.PrintOptions(cout);
-   kappa = freq * M_PI;

   // 2. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -115,129 +100,72 @@ int main(int argc, char *argv[])
   }
   mesh->ReorientTetMesh();

-   // 5. Define a finite element space on the mesh. Here we use Nedelec or
-   //    Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *trial_fec = NULL;
-   FiniteElementCollection *test_fec = NULL;
+   // 5. Define a parallel finite element space on the parallel mesh. Here we
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
+   FiniteElementSpace *H1fespace = new FiniteElementSpace(mesh, H1fec);

-   if (prob == 0)
-   {
-      trial_fec = new H1_FECollection(order, dim);
-      test_fec = new ND_FECollection(order, dim);
-   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
-   else
-   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
-   }
+   int size = fespace->GetTrueVSize();
+   int H1size = H1fespace->GetTrueVSize();
+   cout << "Number of Nedelec finite element unknowns: " << size << endl;
+   cout << "Number of H1 finite element unknowns: " << H1size << endl;

-   FiniteElementSpace trial_fes(mesh, trial_fec);
-   FiniteElementSpace test_fes(mesh, test_fec);
-
-   int trial_size = trial_fes.GetTrueVSize();
-   int test_size = test_fes.GetTrueVSize();
-
-   if (prob == 0)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
-      cout << "Number of H1 finite element unknowns: " << trial_size << endl;
-   }
-   else if (prob == 1)
-   {
-      cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-      cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-           endl;
-   }
-   else
-   {
-      cout << "Number of Raviart-Thomas finite element unknowns: "
-           << trial_size << endl;
-      cout << "Number of L2 finite element unknowns: " << test_size << endl;
-   }
-
-   // 6. Define the solution vector as a finite element grid function
-   //    corresponding to the trial fespace.
-   GridFunction gftest(&test_fes);
-   GridFunction gftrial(&trial_fes);
-   GridFunction x(&test_fes);
+   // 6. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   GridFunction x(fespace);
   FunctionCoefficient p_coef(p_exact);
+   GridFunction p(H1fespace);
+   p.ProjectCoefficient(p_coef);
+   p.SetTrueVector();
+   p.SetFromTrueVector();
+
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
-   FunctionCoefficient divgradp_coef(div_gradp_exact);

-   if (prob == 0)
-   {
-      gftrial.ProjectCoefficient(p_coef);
-   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
-   else
-   {
-      gftrial.ProjectCoefficient(gradp_coef);
-   }
-
-   gftrial.SetTrueVector();
-   gftrial.SetFromTrueVector();
-
-   // 7. Set up the bilinear forms for L2 projection.
-   ConstantCoefficient one(1.0);
-   BilinearForm a(&test_fes);
-   MixedBilinearForm a_mixed(&trial_fes, &test_fes);
+   // 7. Set up the bilinear forms.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(1.0);
+   BilinearForm *a = new BilinearForm(fespace);
+   MixedBilinearForm *a_NDH1 = new MixedBilinearForm(H1fespace, fespace);
   if (pa)
   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a_mixed.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a_NDH1->SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }

-   if (prob == 0)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
-   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
-   else
-   {
-      a.AddDomainIntegrator(new MassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new VectorFEDivergenceIntegrator(one));
-   }
+   // First approach: L2 projection
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+   a_NDH1->AddDomainIntegrator(new MixedVectorGradientIntegrator(*muinv));

-   // 8. Assemble the bilinear form and the corresponding linear system,
-   //    applying any necessary transformations such as: eliminating boundary
-   //    conditions, applying conforming constraints for non-conforming AMR,
-   //    static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
+   // 8. Assemble the parallel bilinear form and the corresponding linear
+   //    system, applying any necessary transformations such as: parallel
+   //    assembly, eliminating boundary conditions, applying conforming
+   //    constraints for non-conforming AMR, static condensation, etc.
+   if (static_cond) { a->EnableStaticCondensation(); }

-   a.Assemble();
-   if (!pa) { a.Finalize(); }
+   a->Assemble();
+   if (!pa) { a->Finalize(); }

-   a_mixed.Assemble();
-   if (!pa) { a_mixed.Finalize(); }
+   a_NDH1->Assemble();
+   if (!pa) { a_NDH1->Finalize(); }

   if (pa)
   {
-      a_mixed.Mult(gftrial, x);
+      a_NDH1->Mult(p, x);
   }
   else
   {
-      SparseMatrix& mixed = a_mixed.SpMat();
-      mixed.Mult(gftrial, x);
+      SparseMatrix& NDH1 = a_NDH1->SpMat();
+      NDH1.Mult(p, x);
   }

   // 9. Define and apply a PCG solver for Ax = b with Jacobi preconditioner.
   {
-      GridFunction rhs(&test_fes);
+      GridFunction rhs(fespace);
      rhs = x;
      x = 0.0;

@@ -248,15 +176,15 @@ int main(int argc, char *argv[])
      if (pa)
      {
         Array<int> ess_tdof_list; // empty
-         OperatorJacobiSmoother Jacobi(a, ess_tdof_list);
+         OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);

-         cg.SetOperator(a);
+         cg.SetOperator(*a);
         cg.SetPreconditioner(Jacobi);
         cg.Mult(rhs, x);
      }
      else
      {
-         SparseMatrix& Amat = a.SpMat();
+         SparseMatrix& Amat = a->SpMat();
         DSmoother Jacobi(Amat);

         cg.SetOperator(Amat);
@@ -265,89 +193,33 @@ int main(int argc, char *argv[])
      }
   }

-   // 10. Compute the same field by applying a DiscreteInterpolator.
-   GridFunction discreteInterpolant(&test_fes);
-   DiscreteLinearOperator dlo(&trial_fes, &test_fes);
-   if (prob == 0)
-   {
-      dlo.AddDomainInterpolator(new GradientInterpolator());
-   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
-   else
-   {
-      dlo.AddDomainInterpolator(new DivergenceInterpolator());
-   }
+   // 10. Second approach: compute the same solution by applying
+   //     GradientInterpolator in H(curl).
+   DiscreteLinearOperator grad(H1fespace, fespace);
+   grad.AddDomainInterpolator(new GradientInterpolator());
+   grad.Assemble();

-   dlo.Assemble();
-   dlo.Mult(gftrial, discreteInterpolant);
+   GridFunction gradp(fespace);
+   grad.Mult(p, gradp);

-   // 11. Compute the projection of the exact field.
-   GridFunction exact_proj(&test_fes);
-   if (prob == 0)
-   {
-      exact_proj.ProjectCoefficient(gradp_coef);
-   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
-   else
-   {
-      exact_proj.ProjectCoefficient(divgradp_coef);
-   }
+   // 11. Compute the projection of the exact grad p.
+   GridFunction exact_gradp(fespace);
+   exact_gradp.ProjectCoefficient(gradp_coef);
+   exact_gradp.SetTrueVector();
+   exact_gradp.SetFromTrueVector();

-   exact_proj.SetTrueVector();
-   exact_proj.SetFromTrueVector();
-
-   // 12. Compute and print the L_2 norm of the error.
-   if (prob == 0)
+   // 12. Compute and print the L^2 norm of the error.
   {
      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
-      double errProj = exact_proj.ComputeL2Error(gradp_coef);
+      double errInterp = gradp.ComputeL2Error(gradp_coef);
+      double errProj = exact_gradp.ComputeL2Error(gradp_coef);

      cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl): "
-           "|| E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
+           "|| E_h - grad p ||_{L^2} = " << errSol << '\n' << endl;
      cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad p"
-           " ||_{L_2} = " << errInterp << '\n' << endl;
+           "||_{L^2} = " << errInterp << '\n' << endl;
      cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-           "||_{L_2} = " << errProj << '\n' << endl;
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in H(div): "
-           "|| E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
-           "||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
-           "||_{L_2} = " << errProj << '\n' << endl;
-   }
-   else
-   {
-      int order_quad = max(2, 2*order+1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i=0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      double errSol = x.ComputeL2Error(divgradp_coef, irs);
-      double errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
-      double errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);
-
-      cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
-           "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-      cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v "
-           "||_{L_2} = " << errInterp << '\n' << endl;
-      cout << " Projection f_h of exact div v in L_2: || f_h - div v "
-           "||_{L_2} = " << errProj << '\n' << endl;
+           "||_{L^2} = " << errProj << '\n' << endl;
   }

   // 13. Save the refined mesh and the solution. This output can be viewed
@@ -370,8 +242,14 @@ int main(int argc, char *argv[])
   }

   // 15. Free the used memory.
-   delete trial_fec;
-   delete test_fec;
+   delete a;
+   delete a_NDH1;
+   delete sigma;
+   delete muinv;
+   delete fespace;
+   delete H1fespace;
+   delete fec;
+   delete H1fec;
   delete mesh;

   return 0;
@@ -406,47 +284,3 @@ void gradp_exact(const Vector &x, Vector &f)
      if (x.Size() == 3) { f(2) = 0.0; }
   }
 }
-
-double div_gradp_exact(const Vector &x)
-{
-   if (dim == 3)
-   {
-      return -3.0 * sin(x(0)) * sin(x(1)) * sin(x(2));
-   }
-   else if (dim == 2)
-   {
-      return -2.0 * sin(x(0)) * sin(x(1));
-   }
-
-   return 0.0;
-}
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -6,8 +6,6 @@
 //               mpirun -np 4 ex24p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 1
-//               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -o 2 -pa -p 2
 //               mpirun -np 4 ex24p -m ../data/escher.mesh
 //               mpirun -np 4 ex24p -m ../data/escher.mesh -o 2
 //               mpirun -np 4 ex24p -m ../data/fichera.mesh
@@ -25,16 +23,11 @@
 //               mpirun -np 4 ex24p -m ../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code illustrates usage of mixed finite element
-//               spaces, with three variants:
+//               spaces. Using two different approaches, we project a gradient
+//               of a function in H^1 to H(curl). Other spaces and example
+//               computations are to be added in the future.
 //
-//               1) (grad p, u) for p in H^1 tested against u in H(curl)
-//               2) (curl v, u) for v in H(curl) tested against u in H(div), 3D
-//               3) (div v, q) for v in H(div) tested against q in L_2
-//
-//               Using different approaches, we project the gradient, curl, or
-//               divergence to the appropriate space.
-//
-//               We recommend viewing examples 1, 3, and 5 before viewing this
+//               We recommend viewing examples 1 and 3 before viewing this
 //               example.

 #include "mfem.hpp"
@@ -46,12 +39,8 @@ using namespace mfem;

 double p_exact(const Vector &x);
 void gradp_exact(const Vector &, Vector &);
-double div_gradp_exact(const Vector &x);
-void v_exact(const Vector &x, Vector &v);
-void curlv_exact(const Vector &x, Vector &cv);

 int dim;
-double freq = 1.0, kappa;

 int main(int argc, char *argv[])
 {
@@ -64,7 +53,6 @@ int main(int argc, char *argv[])
   // 2. Parse command-line options.
   const char *mesh_file = "../data/beam-hex.mesh";
   int order = 1;
-   int prob = 0;
   bool static_cond = false;
   bool pa = false;
   const char *device_config = "cpu";
@@ -75,8 +63,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&prob, "-p", "--problem-type",
-                  "Choose between 0: grad, 1: curl, 2: div");
   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
@@ -101,7 +87,6 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-   kappa = freq * M_PI;

   // 3. Enable hardware devices such as GPUs, and programming models such as
   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
@@ -144,137 +129,80 @@ int main(int argc, char *argv[])
   pmesh->ReorientTetMesh();

   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use Nedelec or Raviart-Thomas finite elements of the specified order.
-   FiniteElementCollection *trial_fec = NULL;
-   FiniteElementCollection *test_fec = NULL;
-
-   if (prob == 0)
-   {
-      trial_fec = new H1_FECollection(order, dim);
-      test_fec = new ND_FECollection(order, dim);
-   }
-   else if (prob == 1)
-   {
-      trial_fec = new ND_FECollection(order, dim);
-      test_fec = new RT_FECollection(order-1, dim);
-   }
-   else
-   {
-      trial_fec = new RT_FECollection(order-1, dim);
-      test_fec = new L2_FECollection(order-1, dim);
-   }
-
-   ParFiniteElementSpace trial_fes(pmesh, trial_fec);
-   ParFiniteElementSpace test_fes(pmesh, test_fec);
-
-   HYPRE_Int trial_size = trial_fes.GlobalTrueVSize();
-   HYPRE_Int test_size = test_fes.GlobalTrueVSize();
-
+   //    use the Nedelec finite elements of the specified order.
+   FiniteElementCollection *fec = new ND_FECollection(order, dim);
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
+   ParFiniteElementSpace *H1fespace = new ParFiniteElementSpace(pmesh, H1fec);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
+   HYPRE_Int H1size = H1fespace->GlobalTrueVSize();
   if (myid == 0)
   {
-      if (prob == 0)
-      {
-         cout << "Number of Nedelec finite element unknowns: " << test_size << endl;
-         cout << "Number of H1 finite element unknowns: " << trial_size << endl;
-      }
-      else if (prob == 1)
-      {
-         cout << "Number of Nedelec finite element unknowns: " << trial_size << endl;
-         cout << "Number of Raviart-Thomas finite element unknowns: " << test_size <<
-              endl;
-      }
-      else
-      {
-         cout << "Number of Raviart-Thomas finite element unknowns: "
-              << trial_size << endl;
-         cout << "Number of L2 finite element unknowns: " << test_size << endl;
-      }
+      cout << "Number of Nedelec finite element unknowns: " << size << endl;
+      cout << "Number of H1 finite element unknowns: " << H1size << endl;
   }

-   // 8. Define the solution vector as a parallel finite element grid function
-   //    corresponding to the trial fespace.
-   ParGridFunction gftest(&test_fes);
-   ParGridFunction gftrial(&trial_fes);
-   ParGridFunction x(&test_fes);
+   // 8. Define the solution vector x as a parallel finite element grid function
+   //    corresponding to fespace. Initialize x by projecting the exact
+   //    solution. Note that only values from the boundary edges will be used
+   //    when eliminating the non-homogeneous boundary condition to modify the
+   //    r.h.s. vector b.
+   ParGridFunction x(fespace);
   FunctionCoefficient p_coef(p_exact);
+   ParGridFunction p(H1fespace);
+   p.ProjectCoefficient(p_coef);
+   p.SetTrueVector();
+   p.SetFromTrueVector();
+
   VectorFunctionCoefficient gradp_coef(sdim, gradp_exact);
-   VectorFunctionCoefficient v_coef(sdim, v_exact);
-   VectorFunctionCoefficient curlv_coef(sdim, curlv_exact);
-   FunctionCoefficient divgradp_coef(div_gradp_exact);

-   if (prob == 0)
-   {
-      gftrial.ProjectCoefficient(p_coef);
-   }
-   else if (prob == 1)
-   {
-      gftrial.ProjectCoefficient(v_coef);
-   }
-   else
-   {
-      gftrial.ProjectCoefficient(gradp_coef);
-   }
-
-   gftrial.SetTrueVector();
-   gftrial.SetFromTrueVector();
-
-   // 9. Set up the parallel bilinear forms for L2 projection.
-   ConstantCoefficient one(1.0);
-   ParBilinearForm a(&test_fes);
-   ParMixedBilinearForm a_mixed(&trial_fes, &test_fes);
+   // 9. Set up the parallel bilinear forms.
+   Coefficient *muinv = new ConstantCoefficient(1.0);
+   Coefficient *sigma = new ConstantCoefficient(1.0);
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   ParMixedBilinearForm *a_NDH1 = new ParMixedBilinearForm(H1fespace, fespace);
   if (pa)
   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a_mixed.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      a_NDH1->SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }

-   if (prob == 0)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorGradientIntegrator(one));
-   }
-   else if (prob == 1)
-   {
-      a.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new MixedVectorCurlIntegrator(one));
-   }
-   else
-   {
-      a.AddDomainIntegrator(new MassIntegrator(one));
-      a_mixed.AddDomainIntegrator(new VectorFEDivergenceIntegrator(one));
-   }
+   // First approach: L2 projection
+   a->AddDomainIntegrator(new VectorFEMassIntegrator(*sigma));
+   a_NDH1->AddDomainIntegrator(new MixedVectorGradientIntegrator(*muinv));

   // 10. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
+   if (static_cond) { a->EnableStaticCondensation(); }

-   a.Assemble();
-   if (!pa) { a.Finalize(); }
+   a->Assemble();
+   if (!pa) { a->Finalize(); }

-   a_mixed.Assemble();
-   if (!pa) { a_mixed.Finalize(); }
+   a_NDH1->Assemble();
+   if (!pa) { a_NDH1->Finalize(); }

-   Vector B(test_fes.GetTrueVSize());
-   Vector X(test_fes.GetTrueVSize());
+   Vector B(fespace->GetTrueVSize());
+   Vector X(fespace->GetTrueVSize());

   if (pa)
   {
-      ParLinearForm b(&test_fes); // used as a vector
-      a_mixed.Mult(gftrial, b); // process-local multiplication
-      b.ParallelAssemble(B);
+      ParLinearForm *b = new ParLinearForm(fespace); // used as a vector
+      a_NDH1->Mult(p, *b); // process-local multiplication
+      b->ParallelAssemble(B);
+      delete b;
   }
   else
   {
-      HypreParMatrix *mixed = a_mixed.ParallelAssemble();
+      HypreParMatrix *NDH1 = a_NDH1->ParallelAssemble();

-      Vector P(trial_fes.GetTrueVSize());
-      gftrial.GetTrueDofs(P);
+      Vector P(H1fespace->GetTrueVSize());
+      p.GetTrueDofs(P);

-      mixed->Mult(P,B);
+      NDH1->Mult(P,B);

-      delete mixed;
+      delete NDH1;
   }

   // 11. Define and apply a parallel PCG solver for AX=B with Jacobi
@@ -284,9 +212,9 @@ int main(int argc, char *argv[])
      Array<int> ess_tdof_list; // empty

      OperatorPtr A;
-      a.FormSystemMatrix(ess_tdof_list, A);
+      a->FormSystemMatrix(ess_tdof_list, A);

-      OperatorJacobiSmoother Jacobi(a, ess_tdof_list);
+      OperatorJacobiSmoother Jacobi(*a, ess_tdof_list);

      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-12);
@@ -299,7 +227,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      HypreParMatrix *Amat = a.ParallelAssemble();
+      HypreParMatrix *Amat = a->ParallelAssemble();
      HypreDiagScale Jacobi(*Amat);
      HyprePCG pcg(*Amat);
      pcg.SetTol(1e-12);
@@ -314,97 +242,35 @@ int main(int argc, char *argv[])

   x.SetFromTrueDofs(X);

-   // 12. Compute the same field by applying a DiscreteInterpolator.
-   ParGridFunction discreteInterpolant(&test_fes);
-   ParDiscreteLinearOperator dlo(&trial_fes, &test_fes);
-   if (prob == 0)
-   {
-      dlo.AddDomainInterpolator(new GradientInterpolator());
-   }
-   else if (prob == 1)
-   {
-      dlo.AddDomainInterpolator(new CurlInterpolator());
-   }
-   else
-   {
-      dlo.AddDomainInterpolator(new DivergenceInterpolator());
-   }
+   // 12. Second approach: compute the same solution by applying
+   //     GradientInterpolator in H(curl).
+   ParDiscreteLinearOperator grad(H1fespace, fespace);
+   grad.AddDomainInterpolator(new GradientInterpolator());
+   grad.Assemble();

-   dlo.Assemble();
-   dlo.Mult(gftrial, discreteInterpolant);
+   ParGridFunction gradp(fespace);
+   grad.Mult(p, gradp);

-   // 13. Compute the projection of the exact field.
-   ParGridFunction exact_proj(&test_fes);
-   if (prob == 0)
-   {
-      exact_proj.ProjectCoefficient(gradp_coef);
-   }
-   else if (prob == 1)
-   {
-      exact_proj.ProjectCoefficient(curlv_coef);
-   }
-   else
-   {
-      exact_proj.ProjectCoefficient(divgradp_coef);
-   }
+   // 13. Compute the projection of the exact grad p.
+   ParGridFunction exact_gradp(fespace);
+   exact_gradp.ProjectCoefficient(gradp_coef);
+   exact_gradp.SetTrueVector();
+   exact_gradp.SetFromTrueVector();

-   exact_proj.SetTrueVector();
-   exact_proj.SetFromTrueVector();
-
-   // 14. Compute and print the L_2 norm of the error.
-   if (prob == 0)
+   // 14. Compute and print the L^2 norm of the error.
   {
      double errSol = x.ComputeL2Error(gradp_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(gradp_coef);
-      double errProj = exact_proj.ComputeL2Error(gradp_coef);
+      double errInterp = gradp.ComputeL2Error(gradp_coef);
+      double errProj = exact_gradp.ComputeL2Error(gradp_coef);

      if (myid == 0)
      {
-         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in H(curl)"
-              ": || E_h - grad p ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - grad"
-              " p ||_{L_2} = " << errInterp << '\n' << endl;
+         cout << "\n Solution of (E_h,v) = (grad p_h,v) for E_h and v in "
+              "H(curl): || E_h - grad p ||_{L^2} = " << errSol << '\n' << endl;
+         cout << " Gradient interpolant E_h = grad p_h in H(curl): || E_h - "
+              "grad p ||_{L^2} = " << errInterp << '\n' << endl;
         cout << " Projection E_h of exact grad p in H(curl): || E_h - grad p "
-              "||_{L_2} = " << errProj << '\n' << endl;
-      }
-   }
-   else if (prob == 1)
-   {
-      double errSol = x.ComputeL2Error(curlv_coef);
-      double errInterp = discreteInterpolant.ComputeL2Error(curlv_coef);
-      double errProj = exact_proj.ComputeL2Error(curlv_coef);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (E_h,w) = (curl v_h,w) for E_h and w in "
-              "H(div): || E_h - curl v ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Curl interpolant E_h = curl v_h in H(div): || E_h - curl v "
-              "||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection E_h of exact curl v in H(div): || E_h - curl v "
-              "||_{L_2} = " << errProj << '\n' << endl;
-      }
-   }
-   else
-   {
-      int order_quad = max(2, 2*order+1);
-      const IntegrationRule *irs[Geometry::NumGeom];
-      for (int i=0; i < Geometry::NumGeom; ++i)
-      {
-         irs[i] = &(IntRules.Get(i, order_quad));
-      }
-
-      double errSol = x.ComputeL2Error(divgradp_coef, irs);
-      double errInterp = discreteInterpolant.ComputeL2Error(divgradp_coef, irs);
-      double errProj = exact_proj.ComputeL2Error(divgradp_coef, irs);
-
-      if (myid == 0)
-      {
-         cout << "\n Solution of (f_h,q) = (div v_h,q) for f_h and q in L_2: "
-              "|| f_h - div v ||_{L_2} = " << errSol << '\n' << endl;
-         cout << " Divergence interpolant f_h = div v_h in L_2: || f_h - div v"
-              " ||_{L_2} = " << errInterp << '\n' << endl;
-         cout << " Projection f_h of exact div v in L_2: || f_h - div v "
-              "||_{L_2} = " << errProj << '\n' << endl;
+              "||_{L^2} = " << errProj << '\n' << endl;
      }
   }

@@ -436,8 +302,14 @@ int main(int argc, char *argv[])
   }

   // 17. Free the used memory.
-   delete trial_fec;
-   delete test_fec;
+   delete a;
+   delete a_NDH1;
+   delete sigma;
+   delete muinv;
+   delete fespace;
+   delete H1fespace;
+   delete fec;
+   delete H1fec;
   delete pmesh;

   MPI_Finalize();
@@ -474,47 +346,3 @@ void gradp_exact(const Vector &x, Vector &f)
      if (x.Size() == 3) { f(2) = 0.0; }
   }
 }
-
-double div_gradp_exact(const Vector &x)
-{
-   if (dim == 3)
-   {
-      return -3.0 * sin(x(0)) * sin(x(1)) * sin(x(2));
-   }
-   else if (dim == 2)
-   {
-      return -2.0 * sin(x(0)) * sin(x(1));
-   }
-
-   return 0.0;
-}
-
-void v_exact(const Vector &x, Vector &v)
-{
-   if (dim == 3)
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(2));
-      v(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      v(0) = sin(kappa * x(1));
-      v(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { v(2) = 0.0; }
-   }
-}
-
-void curlv_exact(const Vector &x, Vector &cv)
-{
-   if (dim == 3)
-   {
-      cv(0) = -kappa * cos(kappa * x(2));
-      cv(1) = -kappa * cos(kappa * x(0));
-      cv(2) = -kappa * cos(kappa * x(1));
-   }
-   else
-   {
-      cv = 0.0;
-   }
-}
@@ -82,27 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -368,19 +365,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -392,22 +389,27 @@ int main(int argc, char *argv[])
   //     applying any necessary transformations such as: assembly, eliminating
   //     boundary conditions, applying conforming constraints for
   //     non-conforming AMR, etc.
-   a.Assemble(0);
+   a.Assemble();

-   OperatorPtr A;
+   OperatorHandle Ah;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

-   // 13. Solve using a direct or an iterative solver
+   // 13. Transform to monolithic SparseMatrix
+   SparseMatrix *A = Ah.As<ComplexSparseMatrix>()->GetSystemMatrix();
+
+   cout << "Size of linear system: " << A->Height() << endl;
+
+   // 14. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUITESPARSE
   {
-      ComplexUMFPackSolver csolver(*A.As<ComplexSparseMatrix>());
-      csolver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      csolver.SetPrintLevel(1);
-      csolver.Mult(B, X);
+      UMFPackSolver  solver(*A);
+      solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+      solver.Mult(B, X);
   }
 #else
-   // 13a. Set up the Bilinear form a(.,.) for the preconditioner
+
+   // 14a. Set up the Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
   //    Domain:   1/mu (Curl E, Curl F) + omega^2 * epsilon (E,F)
@@ -422,23 +424,23 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

      prec.Assemble();

-      OperatorPtr PCOpAh;
+      OperatorHandle PCOpAh;
      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);

-      // 13b. Define and apply a GMRES solver for AU=B with a block diagonal
+      // 14b. Define and apply a GMRES solver for AU=B with a block diagonal
      //      preconditioner based on the Gauss-Seidel sparse smoother.
      Array<int> offsets(3);
      offsets[0] = 0;
@@ -465,15 +467,17 @@ int main(int argc, char *argv[])
   }
 #endif

-   // 14. Recover the solution as a finite element grid function and compute the
+   // 15. Recover the solution as a finite element grid function and compute the
   //     errors if the exact solution is known.
   a.RecoverFEMSolution(X, b, x);

   // If exact is known compute the error
   if (exact_known)
   {
+      ComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -502,7 +506,7 @@ int main(int argc, char *argv[])
           << sqrt(L2Error_Re*L2Error_Re + L2Error_Im*L2Error_Im) << "\n\n";
   }

-   // 15. Save the refined mesh and the solution. This output can be viewed
+   // 16. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m mesh -g sol".
   {
      ofstream mesh_ofs("ex25.mesh");
@@ -517,7 +521,7 @@ int main(int argc, char *argv[])
      x.imag().Save(sol_i_ofs);
   }

-   // 16. Send the solution by socket to a GLVis server.
+   // 17. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      // Define visualization keys for GLVis (see GLVis documentation)
@@ -568,7 +572,8 @@ int main(int argc, char *argv[])
      }
   }

-   // 17. Free the used memory.
+   // 18. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -766,7 +771,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -777,13 +782,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -794,13 +800,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -811,13 +818,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector &D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -831,18 +839,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector &D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -855,18 +864,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -879,13 +889,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector &D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -82,27 +82,24 @@ public:
 };

 // Class for returning the PML coefficients of the bilinear form
-class PMLDiagMatrixCoefficient : public VectorCoefficient
+class PMLMatrixCoefficient : public MatrixCoefficient
 {
 private:
   CartesianPML * pml = nullptr;
-   void (*Function)(const Vector &, CartesianPML * , Vector &);
+   void (*Function)(const Vector &, CartesianPML * , DenseMatrix &);
 public:
-   PMLDiagMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
-                                              Vector &),
-                            CartesianPML * pml_)
-      : VectorCoefficient(dim), pml(pml_), Function(F)
+   PMLMatrixCoefficient(int dim, void(*F)(const Vector &, CartesianPML *,
+                                          DenseMatrix &),
+                        CartesianPML * pml_)
+      : MatrixCoefficient(dim), pml(pml_), Function(F)
   {}
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &K, ElementTransformation &T,
+   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip)
   {
      double x[3];
      Vector transip(x, 3);
      T.Transform(ip, transip);
-      K.SetSize(vdim);
+      K.SetSize(height, width);
      (*Function)(transip, pml, K);
   }
 };
@@ -119,13 +116,13 @@ void source(const Vector &x, Vector & f);

 // Functions for computing the necessary coefficients after PML stretching.
 // J is the Jacobian matrix of the stretching function
-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D);
-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D);
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M);
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M);

 Array2D<double> comp_domain_bdr;
 Array2D<double> domain_bdr;
@@ -396,19 +393,19 @@ int main(int argc, char *argv[])
   a.AddDomainIntegrator(new VectorFEMassIntegrator(restr_omeg),NULL);

   int cdim = (dim == 2) ? 1 : dim;
-   PMLDiagMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
-   PMLDiagMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
-   ScalarVectorProductCoefficient c1_Re(muinv,pml_c1_Re);
-   ScalarVectorProductCoefficient c1_Im(muinv,pml_c1_Im);
-   VectorRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
-   VectorRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);
+   PMLMatrixCoefficient pml_c1_Re(cdim,detJ_inv_JT_J_Re, pml);
+   PMLMatrixCoefficient pml_c1_Im(cdim,detJ_inv_JT_J_Im, pml);
+   ScalarMatrixProductCoefficient c1_Re(muinv,pml_c1_Re);
+   ScalarMatrixProductCoefficient c1_Im(muinv,pml_c1_Im);
+   MatrixRestrictedCoefficient restr_c1_Re(c1_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c1_Im(c1_Im,attrPML);

-   PMLDiagMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
-   PMLDiagMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
-   ScalarVectorProductCoefficient c2_Re(omeg,pml_c2_Re);
-   ScalarVectorProductCoefficient c2_Im(omeg,pml_c2_Im);
-   VectorRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
-   VectorRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);
+   PMLMatrixCoefficient pml_c2_Re(dim, detJ_JT_J_inv_Re,pml);
+   PMLMatrixCoefficient pml_c2_Im(dim, detJ_JT_J_inv_Im,pml);
+   ScalarMatrixProductCoefficient c2_Re(omeg,pml_c2_Re);
+   ScalarMatrixProductCoefficient c2_Im(omeg,pml_c2_Im);
+   MatrixRestrictedCoefficient restr_c2_Re(c2_Re,attrPML);
+   MatrixRestrictedCoefficient restr_c2_Im(c2_Im,attrPML);

   // Integrators inside the PML region
   a.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_Re),
@@ -422,15 +419,21 @@ int main(int argc, char *argv[])
   //     constraints for non-conforming AMR, etc.
   a.Assemble();

-   OperatorPtr Ah;
+   OperatorHandle Ah;
   Vector B, X;
   a.FormLinearSystem(ess_tdof_list, x, b, Ah, X, B);

-   // 15. Solve using a direct or an iterative solver
+   // 15. Transform to monolithic HypreParMatrix
+   HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+
+   if (myid == 0)
+   {
+      cout << "Size of linear system: " << A->GetGlobalNumRows() << endl;
+   }
+
+   // 16. Solve using a direct or an iterative solver
 #ifdef MFEM_USE_SUPERLU
   {
-      // Transform to monolithic HypreParMatrix
-      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
      SuperLURowLocMatrix SA(*A);
      SuperLUSolver superlu(MPI_COMM_WORLD);
      superlu.SetPrintStatistics(false);
@@ -438,9 +441,9 @@ int main(int argc, char *argv[])
      superlu.SetColumnPermutation(superlu::PARMETIS);
      superlu.SetOperator(SA);
      superlu.Mult(B, X);
-      delete A;
   }
 #else
+
   // 16a. Set up the parallel Bilinear form a(.,.) for the preconditioner
   //
   //    In Comp
@@ -456,20 +459,20 @@ int main(int argc, char *argv[])
      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_muinv));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_absomeg));

-      PMLDiagMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
-      ScalarVectorProductCoefficient c1_abs(muinv,pml_c1_abs);
-      VectorRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);
+      PMLMatrixCoefficient pml_c1_abs(cdim,detJ_inv_JT_J_abs, pml);
+      ScalarMatrixProductCoefficient c1_abs(muinv,pml_c1_abs);
+      MatrixRestrictedCoefficient restr_c1_abs(c1_abs,attrPML);

-      PMLDiagMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
-      ScalarVectorProductCoefficient c2_abs(absomeg,pml_c2_abs);
-      VectorRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);
+      PMLMatrixCoefficient pml_c2_abs(dim, detJ_JT_J_inv_abs,pml);
+      ScalarMatrixProductCoefficient c2_abs(absomeg,pml_c2_abs);
+      MatrixRestrictedCoefficient restr_c2_abs(c2_abs,attrPML);

      prec.AddDomainIntegrator(new CurlCurlIntegrator(restr_c1_abs));
      prec.AddDomainIntegrator(new VectorFEMassIntegrator(restr_c2_abs));

      prec.Assemble();

-      OperatorPtr PCOpAh;
+      OperatorHandle PCOpAh;
      prec.FormSystemMatrix(ess_tdof_list, PCOpAh);

      // 16b. Define and apply a parallel GMRES solver for AU=B with a block
@@ -493,7 +496,7 @@ int main(int argc, char *argv[])
      gmres.SetMaxIter(2000);
      gmres.SetRelTol(1e-5);
      gmres.SetAbsTol(0.0);
-      gmres.SetOperator(*Ah);
+      gmres.SetOperator(*A);
      gmres.SetPreconditioner(BlockAMS);
      gmres.Mult(B, X);
   }
@@ -506,8 +509,10 @@ int main(int argc, char *argv[])
   // If exact is known compute the error
   if (exact_known)
   {
+      ParComplexGridFunction x_gf(fespace);
      VectorFunctionCoefficient E_ex_Re(dim, E_exact_Re);
      VectorFunctionCoefficient E_ex_Im(dim, E_exact_Im);
+      x_gf.ProjectCoefficient(E_ex_Re, E_ex_Im);
      int order_quad = max(2, 2 * order + 1);
      const IntegrationRule *irs[Geometry::NumGeom];
      for (int i = 0; i < Geometry::NumGeom; ++i)
@@ -624,6 +629,7 @@ int main(int argc, char *argv[])
   }

   // 20. Free the used memory.
+   delete A;
   delete pml;
   delete fespace;
   delete fec;
@@ -822,7 +828,7 @@ void E_bdr_data_Im(const Vector &x, Vector &E)
   }
 }

-void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -833,13 +839,14 @@ void detJ_JT_J_inv_Re(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).real();
+      M(i, i) = (det / pow(dxs[i], 2)).real();
   }
 }

-void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -850,13 +857,14 @@ void detJ_JT_J_inv_Im(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = (det / pow(dxs[i], 2)).imag();
+      M(i, i) = (det / pow(dxs[i], 2)).imag();
   }
 }

-void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -867,13 +875,14 @@ void detJ_JT_J_inv_abs(const Vector &x, CartesianPML * pml, Vector & D)
      det *= dxs[i];
   }

+   M = 0.0;
   for (int i = 0; i < dim; ++i)
   {
-      D(i) = abs(det / pow(dxs[i], 2));
+      M(i, i) = abs(det / pow(dxs[i], 2));
   }
 }

-void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det(1.0, 0.0);
@@ -887,18 +896,19 @@ void detJ_inv_JT_J_Re(const Vector &x, CartesianPML * pml, Vector & D)
   // in the 2D case the coefficient is scalar 1/det(J)
   if (dim == 2)
   {
-      D = (1.0 / det).real();
+      M = (1.0 / det).real();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).real();
+         M(i, i) = (pow(dxs[i], 2) / det).real();
      }
   }
 }

-void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -911,18 +921,19 @@ void detJ_inv_JT_J_Im(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = (1.0 / det).imag();
+      M = (1.0 / det).imag();
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = (pow(dxs[i], 2) / det).imag();
+         M(i, i) = (pow(dxs[i], 2) / det).imag();
      }
   }
 }

-void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)
+void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, DenseMatrix &M)
 {
   vector<complex<double>> dxs(dim);
   complex<double> det = 1.0;
@@ -935,13 +946,14 @@ void detJ_inv_JT_J_abs(const Vector &x, CartesianPML * pml, Vector & D)

   if (dim == 2)
   {
-      D = abs(1.0 / det);
+      M = abs(1.0 / det);
   }
   else
   {
+      M = 0.0;
      for (int i = 0; i < dim; ++i)
      {
-         D(i) = abs(pow(dxs[i], 2) / det);
+         M(i, i) = abs(pow(dxs[i], 2) / det);
      }
   }
 }
@@ -60,7 +60,6 @@ int main(int argc, char *argv[])
   bool static_cond = false;
   bool visualization = 1;
   bool amg_elast = 0;
-   bool reorder_space = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -76,8 +75,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&reorder_space, "-nodes", "--by-nodes", "-vdim", "--by-vdim",
-                  "Use byNODES ordering of vector space instead of byVDIM");
   args.Parse();
   if (!args.Good())
   {
@@ -159,14 +156,7 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order, dim);
-      if (reorder_space)
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byNODES);
-      }
-      else
-      {
-         fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
-      }
+      fespace = new ParFiniteElementSpace(pmesh, fec, dim, Ordering::byVDIM);
   }
   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
@@ -259,7 +249,7 @@ int main(int argc, char *argv[])
   }
   else
   {
-      amg->SetSystemsOptions(dim, reorder_space);
+      amg->SetSystemsOptions(dim);
   }
   HyprePCG *pcg = new HyprePCG(A);
   pcg->SetTol(1e-8);
@@ -284,6 +274,13 @@ int main(int argc, char *argv[])
      pmesh->SetNodalFESpace(fespace);
   }

+   {
+      x.Save("ex2p.gf", 1);
+      ParGridFunction new_x(fespace, "ex2p.gf");
+      new_x -= x;
+      out << "GF difference: " << new_x.Norml1() << endl;
+   }
+
   // 16. Save in parallel the displaced mesh and the inverted solution (which
   //     gives the backward displacements to the original grid). This output
   //     can be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
@@ -6,7 +6,6 @@
 //               ex4 -m ../data/star.mesh
 //               ex4 -m ../data/beam-tet.mesh
 //               ex4 -m ../data/beam-hex.mesh
-//               ex4 -m ../data/beam-hex.mesh -o 2 -pa
 //               ex4 -m ../data/escher.mesh
 //               ex4 -m ../data/fichera.mesh -o 2 -hb
 //               ex4 -m ../data/fichera-q2.vtk
@@ -21,12 +20,6 @@
 //               ex4 -m ../data/fichera-amr.mesh -o 2 -sc
 //               ex4 -m ../data/star-surf.mesh -o 1
 //
-// Device sample runs:
-//               ex4 -m ../data/star.mesh -pa -d cuda
-//               ex4 -m ../data/star.mesh -pa -d raja-cuda
-//               ex4 -m ../data/star.mesh -pa -d raja-omp
-//               ex4 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D H(div) diffusion
 //               problem corresponding to the second order definite equation
 //               -grad(alpha div F) + beta F = f with boundary condition F dot n
@@ -62,8 +55,6 @@ int main(int argc, char *argv[])
   bool set_bc = true;
   bool static_cond = false;
   bool hybridization = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -79,10 +70,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -95,19 +82,14 @@ int main(int argc, char *argv[])
   args.PrintOptions(cout);
   kappa = freq * M_PI;

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume, as well as
   //    periodic meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 25,000
   //    elements.
@@ -120,14 +102,14 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use the
+   // 4. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, fec);
   cout << "Number of finite element unknowns: "
        << fespace->GetTrueVSize() << endl;

-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 5. Determine the list of true (i.e. conforming) essential boundary dofs.
   //    In this example, the boundary conditions are defined by marking all
   //    the boundary attributes from the mesh as essential (Dirichlet) and
   //    converting them to a list of true dofs.
@@ -139,7 +121,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 7. Set up the linear form b(.) which corresponds to the right-hand side
+   // 6. Set up the linear form b(.) which corresponds to the right-hand side
   //    of the FEM linear system, which in this case is (f,phi_i) where f is
   //    given by the function f_exact and phi_i are the basis functions in the
   //    finite element fespace.
@@ -148,7 +130,7 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
   b->Assemble();

-   // 8. Define the solution vector x as a finite element grid function
+   // 7. Define the solution vector x as a finite element grid function
   //    corresponding to fespace. Initialize x by projecting the exact
   //    solution. Note that only values from the boundary faces will be used
   //    when eliminating the non-homogeneous boundary condition to modify the
@@ -157,17 +139,16 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient F(sdim, F_exact);
   x.ProjectCoefficient(F);

-   // 9. Set up the bilinear form corresponding to the H(div) diffusion operator
+   // 8. Set up the bilinear form corresponding to the H(div) diffusion operator
   //    grad alpha div + beta I, by adding the div-div and the mass domain
   //    integrators.
   Coefficient *alpha = new ConstantCoefficient(1.0);
   Coefficient *beta  = new ConstantCoefficient(1.0);
   BilinearForm *a = new BilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));

-   // 10. Assemble the bilinear form and the corresponding linear system,
+   // 9. Assemble the bilinear form and the corresponding linear system,
   //    applying any necessary transformations such as: eliminating boundary
   //    conditions, applying conforming constraints for non-conforming AMR,
   //    static condensation, hybridization, etc.
@@ -186,47 +167,32 @@ int main(int argc, char *argv[])
   }
   a->Assemble();

-   OperatorPtr A;
+   SparseMatrix A;
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   cout << "Size of linear system: " << A->Height() << endl;
+   cout << "Size of linear system: " << A.Height() << endl;

-   // 11. Solve the linear system A X = B.
-   if (!pa)
-   {
 #ifndef MFEM_USE_SUITESPARSE
-      // Use a simple symmetric Gauss-Seidel preconditioner with PCG.
-      GSSmoother M((SparseMatrix&)(*A));
-      PCG(*A, M, B, X, 1, 10000, 1e-20, 0.0);
+   // 10. Define a simple symmetric Gauss-Seidel preconditioner and use it to
+   //     solve the system A X = B with PCG.
+   GSSmoother M(A);
+   PCG(A, M, B, X, 1, 10000, 1e-20, 0.0);
 #else
-      // If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
-      UMFPackSolver umf_solver;
-      umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-      umf_solver.SetOperator(*A);
-      umf_solver.Mult(B, X);
+   // 10. If compiled with SuiteSparse support, use UMFPACK to solve the system.
+   UMFPackSolver umf_solver;
+   umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   umf_solver.SetOperator(A);
+   umf_solver.Mult(B, X);
 #endif
-   }
-   else // Jacobi preconditioning in partial assembly mode
-   {
-      if (UsesTensorBasis(*fespace))
-      {
-         OperatorJacobiSmoother M(*a, ess_tdof_list);
-         PCG(*A, M, B, X, 1, 10000, 1e-20, 0.0);
-      }
-      else
-      {
-         CG(*A, B, X, 1, 10000, 1e-20, 0.0);
-      }
-   }

-   // 12. Recover the solution as a finite element grid function.
+   // 11. Recover the solution as a finite element grid function.
   a->RecoverFEMSolution(X, *b, x);

-   // 13. Compute and print the L^2 norm of the error.
+   // 12. Compute and print the L^2 norm of the error.
   cout << "\n|| F_h - F ||_{L^2} = " << x.ComputeL2Error(F) << '\n' << endl;

-   // 14. Save the refined mesh and the solution. This output can be viewed
+   // 13. Save the refined mesh and the solution. This output can be viewed
   //     later using GLVis: "glvis -m refined.mesh -g sol.gf".
   {
      ofstream mesh_ofs("refined.mesh");
@@ -237,7 +203,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 15. Send the solution by socket to a GLVis server.
+   // 14. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -247,7 +213,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *mesh << x << flush;
   }

-   // 16. Free the used memory.
+   // 15. Free the used memory.
   delete hfes;
   delete hfec;
   delete a;
@@ -269,7 +235,7 @@ void F_exact(const Vector &p, Vector &F)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   F(0) = cos(kappa*x)*sin(kappa*y);
   F(1) = cos(kappa*y)*sin(kappa*x);
@@ -286,7 +252,7 @@ void f_exact(const Vector &p, Vector &f)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   double temp = 1 + 2*kappa*kappa;

@@ -6,7 +6,6 @@
 //               mpirun -np 4 ex4p -m ../data/star.mesh
 //               mpirun -np 4 ex4p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex4p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex4p -m ../data/escher.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/fichera.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/fichera-q2.vtk
@@ -16,17 +15,10 @@
 //               mpirun -np 4 ex4p -m ../data/periodic-square.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/periodic-cube.mesh -no-bc
 //               mpirun -np 4 ex4p -m ../data/amr-quad.mesh
-//               mpirun -np 3 ex4p -m ../data/amr-quad.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -sc
 //               mpirun -np 4 ex4p -m ../data/amr-hex.mesh -o 2 -hb
 //               mpirun -np 4 ex4p -m ../data/star-surf.mesh -o 3 -hb
 //
-// Device sample runs:
-//               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d cuda
-//               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-cuda
-//               mpirun -np 4 ex4p -m ../data/star.mesh -pa -d raja-omp
-//               mpirun -np 4 ex4p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D H(div) diffusion
 //               problem corresponding to the second order definite equation
 //               -grad(alpha div F) + beta F = f with boundary condition F dot n
@@ -68,8 +60,6 @@ int main(int argc, char *argv[])
   bool set_bc = true;
   bool static_cond = false;
   bool hybridization = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -85,10 +75,6 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&hybridization, "-hb", "--hybridization", "-no-hb",
                  "--no-hybridization", "Enable hybridization.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -108,19 +94,14 @@ int main(int argc, char *argv[])
   }
   kappa = freq * M_PI;

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume, as well as periodic meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
   //    more than 1,000 elements.
@@ -133,7 +114,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted. Tetrahedral
   //    meshes need to be reoriented before we can define high-order Nedelec
@@ -149,7 +130,7 @@ int main(int argc, char *argv[])
   }
   pmesh->ReorientTetMesh();

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *fec = new RT_FECollection(order-1, dim);
   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
@@ -159,7 +140,7 @@ int main(int argc, char *argv[])
      cout << "Number of finite element unknowns: " << size << endl;
   }

-   // 8. Determine the list of true (i.e. parallel conforming) essential
+   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
@@ -171,7 +152,7 @@ int main(int argc, char *argv[])
      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 9. Set up the parallel linear form b(.) which corresponds to the
+   // 8. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (f,phi_i) where f is given by the function f_exact and phi_i are the
   //    basis functions in the finite element fespace.
@@ -180,7 +161,7 @@ int main(int argc, char *argv[])
   b->AddDomainIntegrator(new VectorFEDomainLFIntegrator(f));
   b->Assemble();

-   // 10. Define the solution vector x as a parallel finite element grid function
+   // 9. Define the solution vector x as a parallel finite element grid function
   //    corresponding to fespace. Initialize x by projecting the exact
   //    solution. Note that only values from the boundary faces will be used
   //    when eliminating the non-homogeneous boundary condition to modify the
@@ -189,17 +170,16 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient F(sdim, F_exact);
   x.ProjectCoefficient(F);

-   // 11. Set up the parallel bilinear form corresponding to the H(div)
+   // 10. Set up the parallel bilinear form corresponding to the H(div)
   //     diffusion operator grad alpha div + beta I, by adding the div-div and
   //     the mass domain integrators.
   Coefficient *alpha = new ConstantCoefficient(1.0);
   Coefficient *beta  = new ConstantCoefficient(1.0);
   ParBilinearForm *a = new ParBilinearForm(fespace);
-   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   a->AddDomainIntegrator(new DivDivIntegrator(*alpha));
   a->AddDomainIntegrator(new VectorFEMassIntegrator(*beta));

-   // 12. Assemble the parallel bilinear form and the corresponding linear
+   // 11. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation,
@@ -219,43 +199,41 @@ int main(int argc, char *argv[])
   }
   a->Assemble();

-   OperatorPtr A;
+   HypreParMatrix A;
   Vector B, X;
   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

-   if (myid == 0 && !pa)
+   HYPRE_Int glob_size = A.GetGlobalNumRows();
+   if (myid == 0)
   {
-      cout << "Size of linear system: "
-           << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
+      cout << "Size of linear system: " << glob_size << endl;
   }

-   // 13. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
+   // 12. Define and apply a parallel PCG solver for A X = B with the 2D AMS or
   //     the 3D ADS preconditioners from hypre. If using hybridization, the
-   //     system is preconditioned with hypre's BoomerAMG. In the partial
-   //     assembly case, use Jacobi preconditioning.
-   Solver *prec = NULL;
-   CGSolver *pcg = new CGSolver(MPI_COMM_WORLD);
-   pcg->SetOperator(*A);
+   //     system is preconditioned with hypre's BoomerAMG.
+   HypreSolver *prec = NULL;
+   CGSolver *pcg = new CGSolver(A.GetComm());
+   pcg->SetOperator(A);
   pcg->SetRelTol(1e-12);
-   pcg->SetMaxIter(2000);
+   pcg->SetMaxIter(500);
   pcg->SetPrintLevel(1);
-   if (hybridization) { prec = new HypreBoomerAMG(*A.As<HypreParMatrix>()); }
-   else if (pa) { prec = new OperatorJacobiSmoother(*a, ess_tdof_list); }
+   if (hybridization) { prec = new HypreBoomerAMG(A); }
   else
   {
      ParFiniteElementSpace *prec_fespace =
         (a->StaticCondensationIsEnabled() ? a->SCParFESpace() : fespace);
-      if (dim == 2)   { prec = new HypreAMS(*A.As<HypreParMatrix>(), prec_fespace); }
-      else            { prec = new HypreADS(*A.As<HypreParMatrix>(), prec_fespace); }
+      if (dim == 2)   { prec = new HypreAMS(A, prec_fespace); }
+      else            { prec = new HypreADS(A, prec_fespace); }
   }
   pcg->SetPreconditioner(*prec);
   pcg->Mult(B, X);

-   // 14. Recover the parallel grid function corresponding to X. This is the
+   // 13. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
   a->RecoverFEMSolution(X, *b, x);

-   // 15. Compute and print the L^2 norm of the error.
+   // 14. Compute and print the L^2 norm of the error.
   {
      double err = x.ComputeL2Error(F);
      if (myid == 0)
@@ -264,7 +242,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 16. Save the refined mesh and the solution in parallel. This output can
+   // 15. Save the refined mesh and the solution in parallel. This output can
   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
   {
      ostringstream mesh_name, sol_name;
@@ -280,7 +258,7 @@ int main(int argc, char *argv[])
      x.Save(sol_ofs);
   }

-   // 17. Send the solution by socket to a GLVis server.
+   // 16. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -291,7 +269,7 @@ int main(int argc, char *argv[])
      sol_sock << "solution\n" << *pmesh << x << flush;
   }

-   // 18. Free the used memory.
+   // 17. Free the used memory.
   delete pcg;
   delete prec;
   delete hfes;
@@ -317,7 +295,7 @@ void F_exact(const Vector &p, Vector &F)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   F(0) = cos(kappa*x)*sin(kappa*y);
   F(1) = cos(kappa*y)*sin(kappa*x);
@@ -334,7 +312,7 @@ void f_exact(const Vector &p, Vector &f)

   double x = p(0);
   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+   // double z = (dim == 3) ? p(2) : 0.0;

   double temp = 1 + 2*kappa*kappa;

@@ -4,19 +4,11 @@
 //
 // Sample runs:  ex5 -m ../data/square-disc.mesh
 //               ex5 -m ../data/star.mesh
-//               ex5 -m ../data/star.mesh -pa
 //               ex5 -m ../data/beam-tet.mesh
 //               ex5 -m ../data/beam-hex.mesh
-//               ex5 -m ../data/beam-hex.mesh -pa
 //               ex5 -m ../data/escher.mesh
 //               ex5 -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               ex5 -m ../data/star.mesh -pa -d cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-cuda
-//               ex5 -m ../data/star.mesh -pa -d raja-omp
-//               ex5 -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -55,8 +47,6 @@ int main(int argc, char *argv[])
   // 1. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;

   OptionsParser args(argc, argv);
@@ -64,10 +54,6 @@ int main(int argc, char *argv[])
                  "Mesh file to use.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -79,18 +65,13 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   // 2. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   device.Print();
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   // 2. Read the mesh from the given mesh file. We can handle triangular,
   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
   //    the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 4. Refine the mesh to increase the resolution. In this example we do
+   // 3. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
@@ -103,7 +84,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 5. Define a finite element space on the mesh. Here we use the
+   // 4. Define a finite element space on the mesh. Here we use the
   //    Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -111,7 +92,7 @@ int main(int argc, char *argv[])
   FiniteElementSpace *R_space = new FiniteElementSpace(mesh, hdiv_coll);
   FiniteElementSpace *W_space = new FiniteElementSpace(mesh, l2_coll);

-   // 6. Define the BlockStructure of the problem, i.e. define the array of
+   // 5. Define the BlockStructure of the problem, i.e. define the array of
   //    offsets for each variable. The last component of the Array is the sum
   //    of the dimensions of each block.
   Array<int> block_offsets(3); // number of variables + 1
@@ -126,7 +107,7 @@ int main(int argc, char *argv[])
   std::cout << "dim(R+W) = " << block_offsets.Last() << "\n";
   std::cout << "***********************************************************\n";

-   // 7. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 6. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -136,28 +117,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 8. Allocate memory (x, rhs) for the analytical solution and the right hand
+   // 7. Allocate memory (x, rhs) for the analytical solution and the right hand
   //    side.  Define the GridFunction u,p for the finite element solution and
   //    linear forms fform and gform for the right hand side.  The data
   //    allocated by x and rhs are passed as a reference to the grid functions
   //    (u,p) and the linear forms (fform, gform).
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
+   BlockVector x(block_offsets), rhs(block_offsets);

   LinearForm *fform(new LinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);

   LinearForm *gform(new LinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);

-   // 9. Assemble the finite element matrices for the Darcy operator
+   // 8. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -168,103 +146,55 @@ int main(int argc, char *argv[])
   BilinearForm *mVarf(new BilinearForm(R_space));
   MixedBilinearForm *bVarf(new MixedBilinearForm(R_space, W_space));

-   if (pa) { mVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(k));
   mVarf->Assemble();
-   if (!pa) { mVarf->Finalize(); }
+   mVarf->Finalize();
+   SparseMatrix &M(mVarf->SpMat());

-   if (pa) { bVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
   bVarf->Assemble();
-   if (!pa) { bVarf->Finalize(); }
+   bVarf->Finalize();
+   SparseMatrix & B(bVarf->SpMat());
+   B *= -1.;
+   SparseMatrix *BT = Transpose(B);

-   BlockOperator darcyOp(block_offsets);
+   BlockMatrix darcyMatrix(block_offsets);
+   darcyMatrix.SetBlock(0,0, &M);
+   darcyMatrix.SetBlock(0,1, BT);
+   darcyMatrix.SetBlock(1,0, &B);

-   TransposeOperator *Bt = NULL;
-
-   if (pa)
-   {
-      Bt = new TransposeOperator(bVarf);
-
-      darcyOp.SetBlock(0,0, mVarf);
-      darcyOp.SetBlock(0,1, Bt, -1.0);
-      darcyOp.SetBlock(1,0, bVarf, -1.0);
-   }
-   else
-   {
-      SparseMatrix &M(mVarf->SpMat());
-      SparseMatrix &B(bVarf->SpMat());
-      B *= -1.;
-      Bt = new TransposeOperator(&B);
-
-      darcyOp.SetBlock(0,0, &M);
-      darcyOp.SetBlock(0,1, Bt);
-      darcyOp.SetBlock(1,0, &B);
-   }
-
-   // 10. Construct the operators for preconditioner
+   // 9. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
   //
   //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
   //     pressure Schur Complement
-   SparseMatrix *MinvBt = NULL;
-   Vector Md(mVarf->Height());
+   SparseMatrix *MinvBt = Transpose(B);
+   Vector Md(M.Height());
+   M.GetDiag(Md);
+   for (int i = 0; i < Md.Size(); i++)
+   {
+      MinvBt->ScaleRow(i, 1./Md(i));
+   }
+   SparseMatrix *S = Mult(B, *MinvBt);

-   BlockDiagonalPreconditioner darcyPrec(block_offsets);
   Solver *invM, *invS;
-   SparseMatrix *S = NULL;
-
-   if (pa)
-   {
-      mVarf->AssembleDiagonal(Md);
-      auto Md_host = Md.HostRead();
-      Vector invMd(mVarf->Height());
-      for (int i=0; i<mVarf->Height(); ++i)
-      {
-         invMd(i) = 1.0 / Md_host[i];
-      }
-
-      Vector BMBt_diag(bVarf->Height());
-      bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-      Array<int> ess_tdof_list;  // empty
-
-      invM = new OperatorJacobiSmoother(Md, ess_tdof_list);
-      invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
-   }
-   else
-   {
-      SparseMatrix &M(mVarf->SpMat());
-      M.GetDiag(Md);
-
-      SparseMatrix &B(bVarf->SpMat());
-      MinvBt = Transpose(B);
-
-      for (int i = 0; i < Md.Size(); i++)
-      {
-         MinvBt->ScaleRow(i, 1./Md(i));
-      }
-
-      S = Mult(B, *MinvBt);
-
-      invM = new DSmoother(M);
-
+   invM = new DSmoother(M);
 #ifndef MFEM_USE_SUITESPARSE
-      invS = new GSSmoother(*S);
+   invS = new GSSmoother(*S);
 #else
-      invS = new UMFPackSolver(*S);
+   invS = new UMFPackSolver(*S);
 #endif
-   }

   invM->iterative_mode = false;
   invS->iterative_mode = false;

+   BlockDiagonalPreconditioner darcyPrec(block_offsets);
   darcyPrec.SetDiagonalBlock(0, invM);
   darcyPrec.SetDiagonalBlock(1, invS);

-   // 11. Solve the linear system with MINRES.
+   // 10. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
   int maxIter(1000);
   double rtol(1.e-6);
@@ -276,12 +206,11 @@ int main(int argc, char *argv[])
   solver.SetAbsTol(atol);
   solver.SetRelTol(rtol);
   solver.SetMaxIter(maxIter);
-   solver.SetOperator(darcyOp);
+   solver.SetOperator(darcyMatrix);
   solver.SetPreconditioner(darcyPrec);
   solver.SetPrintLevel(1);
   x = 0.0;
   solver.Mult(rhs, x);
-   if (device.IsEnabled()) { x.HostRead(); }
   chrono.Stop();

   if (solver.GetConverged())
@@ -292,7 +221,7 @@ int main(int argc, char *argv[])
                << " iterations. Residual norm is " << solver.GetFinalNorm() << ".\n";
   std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";

-   // 12. Create the grid functions u and p. Compute the L2 error norms.
+   // 11. Create the grid functions u and p. Compute the L2 error norms.
   GridFunction u, p;
   u.MakeRef(R_space, x.GetBlock(0), 0);
   p.MakeRef(W_space, x.GetBlock(1), 0);
@@ -312,7 +241,7 @@ int main(int argc, char *argv[])
   std::cout << "|| u_h - u_ex || / || u_ex || = " << err_u / norm_u << "\n";
   std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";

-   // 13. Save the mesh and the solution. This output can be viewed later using
+   // 12. Save the mesh and the solution. This output can be viewed later using
   //     GLVis: "glvis -m ex5.mesh -g sol_u.gf" or "glvis -m ex5.mesh -g
   //     sol_p.gf".
   {
@@ -329,13 +258,13 @@ int main(int argc, char *argv[])
      p.Save(p_ofs);
   }

-   // 14. Save data in the VisIt format
+   // 13. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5", mesh);
   visit_dc.RegisterField("velocity", &u);
   visit_dc.RegisterField("pressure", &p);
   visit_dc.Save();

-   // 15. Save data in the ParaView format
+   // 14. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5", mesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -347,7 +276,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",&p);
   paraview_dc.Save();

-   // 16. Send the solution by socket to a GLVis server.
+   // 15. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -360,14 +289,14 @@ int main(int argc, char *argv[])
      p_sock << "solution\n" << *mesh << p << "window_title 'Pressure'" << endl;
   }

-   // 17. Free the used memory.
+   // 16. Free the used memory.
   delete fform;
   delete gform;
   delete invM;
   delete invS;
   delete S;
-   delete Bt;
   delete MinvBt;
+   delete BT;
   delete mVarf;
   delete bVarf;
   delete W_space;
@@ -4,19 +4,11 @@
 //
 // Sample runs:  mpirun -np 4 ex5p -m ../data/square-disc.mesh
 //               mpirun -np 4 ex5p -m ../data/star.mesh
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa
 //               mpirun -np 4 ex5p -m ../data/beam-tet.mesh
 //               mpirun -np 4 ex5p -m ../data/beam-hex.mesh
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa
 //               mpirun -np 4 ex5p -m ../data/escher.mesh
 //               mpirun -np 4 ex5p -m ../data/fichera.mesh
 //
-// Device sample runs:
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-cuda
-//               mpirun -np 4 ex5p -m ../data/star.mesh -r 2 -pa -d raja-omp
-//               mpirun -np 4 ex5p -m ../data/beam-hex.mesh -pa -d cuda
-//
 // Description:  This example code solves a simple 2D/3D mixed Darcy problem
 //               corresponding to the saddle point system
 //                                 k*u + grad p = f
@@ -62,28 +54,19 @@ int main(int argc, char *argv[])

   // 2. Parse command-line options.
   const char *mesh_file = "../data/star.mesh";
-   int ref_levels = -1;
   int order = 1;
   bool par_format = false;
-   bool pa = false;
-   const char *device_config = "cpu";
   bool visualization = 1;
   bool adios2 = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Finite element order (polynomial degree).");
   args.AddOption(&par_format, "-pf", "--parallel-format", "-sf",
                  "--serial-format",
                  "Format to use when saving the results for VisIt.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -105,34 +88,26 @@ int main(int argc, char *argv[])
      args.PrintOptions(cout);
   }

-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
+   // 3. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();

-   // 5. Refine the serial mesh on all processors to increase the resolution. In
+   // 4. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 10,000 elements, unless the user specifies it as input.
+   //    more than 10,000 elements.
   {
-      if (ref_levels == -1)
-      {
-         ref_levels = (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
-      }
-
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
      }
   }

-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
+   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
@@ -145,7 +120,7 @@ int main(int argc, char *argv[])
      }
   }

-   // 7. Define a parallel finite element space on the parallel mesh. Here we
+   // 6. Define a parallel finite element space on the parallel mesh. Here we
   //    use the Raviart-Thomas finite elements of the specified order.
   FiniteElementCollection *hdiv_coll(new RT_FECollection(order, dim));
   FiniteElementCollection *l2_coll(new L2_FECollection(order, dim));
@@ -165,7 +140,7 @@ int main(int argc, char *argv[])
      std::cout << "***********************************************************\n";
   }

-   // 8. Define the two BlockStructure of the problem.  block_offsets is used
+   // 7. Define the two BlockStructure of the problem.  block_offsets is used
   //    for Vector based on dof (like ParGridFunction or ParLinearForm),
   //    block_trueOffstes is used for Vector based on trueDof (HypreParVector
   //    for the rhs and solution of the linear system).  The offsets computed
@@ -182,7 +157,7 @@ int main(int argc, char *argv[])
   block_trueOffsets[2] = W_space->TrueVSize();
   block_trueOffsets.PartialSum();

-   // 9. Define the coefficients, analytical solution, and rhs of the PDE.
+   // 8. Define the coefficients, analytical solution, and rhs of the PDE.
   ConstantCoefficient k(1.0);

   VectorFunctionCoefficient fcoeff(dim, fFun);
@@ -192,30 +167,25 @@ int main(int argc, char *argv[])
   VectorFunctionCoefficient ucoeff(dim, uFun_ex);
   FunctionCoefficient pcoeff(pFun_ex);

-   // 10. Define the parallel grid function and parallel linear forms, solution
-   //     vector and rhs.
-   MemoryType mt = device.GetMemoryType();
-   BlockVector x(block_offsets, mt), rhs(block_offsets, mt);
-   BlockVector trueX(block_trueOffsets, mt), trueRhs(block_trueOffsets, mt);
+   // 9. Define the parallel grid function and parallel linear forms, solution
+   //    vector and rhs.
+   BlockVector x(block_offsets), rhs(block_offsets);
+   BlockVector trueX(block_trueOffsets), trueRhs(block_trueOffsets);

   ParLinearForm *fform(new ParLinearForm);
   fform->Update(R_space, rhs.GetBlock(0), 0);
   fform->AddDomainIntegrator(new VectorFEDomainLFIntegrator(fcoeff));
   fform->AddBoundaryIntegrator(new VectorFEBoundaryFluxLFIntegrator(fnatcoeff));
   fform->Assemble();
-   fform->SyncAliasMemory(rhs);
   fform->ParallelAssemble(trueRhs.GetBlock(0));
-   trueRhs.GetBlock(0).SyncAliasMemory(trueRhs);

   ParLinearForm *gform(new ParLinearForm);
   gform->Update(W_space, rhs.GetBlock(1), 0);
   gform->AddDomainIntegrator(new DomainLFIntegrator(gcoeff));
   gform->Assemble();
-   gform->SyncAliasMemory(rhs);
   gform->ParallelAssemble(trueRhs.GetBlock(1));
-   trueRhs.GetBlock(1).SyncAliasMemory(trueRhs);

-   // 11. Assemble the finite element matrices for the Darcy operator
+   // 10. Assemble the finite element matrices for the Darcy operator
   //
   //                            D = [ M  B^T ]
   //                                [ B   0  ]
@@ -226,93 +196,44 @@ int main(int argc, char *argv[])
   ParBilinearForm *mVarf(new ParBilinearForm(R_space));
   ParMixedBilinearForm *bVarf(new ParMixedBilinearForm(R_space, W_space));

-   HypreParMatrix *M = NULL;
-   HypreParMatrix *B = NULL;
+   HypreParMatrix *M, *B;

-   if (pa) { mVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   mVarf->AddDomainIntegrator(new VectorFEMassIntegrator(k));
   mVarf->Assemble();
-   if (!pa) { mVarf->Finalize(); }
+   mVarf->Finalize();
+   M = mVarf->ParallelAssemble();

-   if (pa) { bVarf->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   bVarf->AddDomainIntegrator(new VectorFEDivergenceIntegrator);
   bVarf->Assemble();
-   if (!pa) { bVarf->Finalize(); }
+   bVarf->Finalize();
+   B = bVarf->ParallelAssemble();
+   (*B) *= -1;
+
+   HypreParMatrix *BT = B->Transpose();

   BlockOperator *darcyOp = new BlockOperator(block_trueOffsets);
+   darcyOp->SetBlock(0,0, M);
+   darcyOp->SetBlock(0,1, BT);
+   darcyOp->SetBlock(1,0, B);

-   Array<int> empty_tdof_list;  // empty
-   OperatorPtr opM, opB;
-
-   TransposeOperator *Bt = NULL;
-
-   if (pa)
-   {
-      mVarf->FormSystemMatrix(empty_tdof_list, opM);
-      bVarf->FormRectangularSystemMatrix(empty_tdof_list, empty_tdof_list, opB);
-      Bt = new TransposeOperator(opB.Ptr());
-
-      darcyOp->SetBlock(0,0, opM.Ptr());
-      darcyOp->SetBlock(0,1, Bt, -1.0);
-      darcyOp->SetBlock(1,0, opB.Ptr(), -1.0);
-   }
-   else
-   {
-      M = mVarf->ParallelAssemble();
-      B = bVarf->ParallelAssemble();
-      (*B) *= -1;
-      Bt = new TransposeOperator(B);
-
-      darcyOp->SetBlock(0,0, M);
-      darcyOp->SetBlock(0,1, Bt);
-      darcyOp->SetBlock(1,0, B);
-   }
-
-   // 12. Construct the operators for preconditioner
+   // 11. Construct the operators for preconditioner
   //
   //                 P = [ diag(M)         0         ]
   //                     [  0       B diag(M)^-1 B^T ]
   //
   //     Here we use Symmetric Gauss-Seidel to approximate the inverse of the
   //     pressure Schur Complement.
-   HypreParMatrix *MinvBt = NULL;
-   HypreParVector *Md = NULL;
-   HypreParMatrix *S = NULL;
-   Vector Md_PA;
-   Solver *invM, *invS;
+   HypreParMatrix *MinvBt = B->Transpose();
+   HypreParVector *Md = new HypreParVector(MPI_COMM_WORLD, M->GetGlobalNumRows(),
+                                           M->GetRowStarts());
+   M->GetDiag(*Md);

-   if (pa)
-   {
-      Md_PA.SetSize(R_space->GetTrueVSize());
-      mVarf->AssembleDiagonal(Md_PA);
-      auto Md_host = Md_PA.HostRead();
-      Vector invMd(Md_PA.Size());
-      for (int i=0; i<Md_PA.Size(); ++i)
-      {
-         invMd(i) = 1.0 / Md_host[i];
-      }
+   MinvBt->InvScaleRows(*Md);
+   HypreParMatrix *S = ParMult(B, MinvBt);

-      Vector BMBt_diag(W_space->GetTrueVSize());
-      bVarf->AssembleDiagonal_ADAt(invMd, BMBt_diag);
-
-      Array<int> ess_tdof_list;  // empty
-
-      invM = new OperatorJacobiSmoother(Md_PA, ess_tdof_list);
-      invS = new OperatorJacobiSmoother(BMBt_diag, ess_tdof_list);
-   }
-   else
-   {
-      Md = new HypreParVector(MPI_COMM_WORLD, M->GetGlobalNumRows(),
-                              M->GetRowStarts());
-      M->GetDiag(*Md);
-
-      MinvBt = B->Transpose();
-      MinvBt->InvScaleRows(*Md);
-      S = ParMult(B, MinvBt);
-
-      invM = new HypreDiagScale(*M);
-      invS = new HypreBoomerAMG(*S);
-   }
+   HypreSolver *invM, *invS;
+   invM = new HypreDiagScale(*M);
+   invS = new HypreBoomerAMG(*S);

   invM->iterative_mode = false;
   invS->iterative_mode = false;
@@ -322,9 +243,9 @@ int main(int argc, char *argv[])
   darcyPr->SetDiagonalBlock(0, invM);
   darcyPr->SetDiagonalBlock(1, invS);

-   // 13. Solve the linear system with MINRES.
+   // 12. Solve the linear system with MINRES.
   //     Check the norm of the unpreconditioned residual.
-   int maxIter(pa ? 1000 : 500);
+   int maxIter(500);
   double rtol(1.e-6);
   double atol(1.e-10);

@@ -339,7 +260,6 @@ int main(int argc, char *argv[])
   solver.SetPrintLevel(verbose);
   trueX = 0.0;
   solver.Mult(trueRhs, trueX);
-   if (device.IsEnabled()) { trueX.HostRead(); }
   chrono.Stop();

   if (verbose)
@@ -353,7 +273,7 @@ int main(int argc, char *argv[])
      std::cout << "MINRES solver took " << chrono.RealTime() << "s. \n";
   }

-   // 14. Extract the parallel grid function corresponding to the finite element
+   // 13. Extract the parallel grid function corresponding to the finite element
   //     approximation X. This is the local solution on each processor. Compute
   //     L2 error norms.
   ParGridFunction *u(new ParGridFunction);
@@ -381,7 +301,7 @@ int main(int argc, char *argv[])
      std::cout << "|| p_h - p_ex || / || p_ex || = " << err_p / norm_p << "\n";
   }

-   // 15. Save the refined mesh and the solution in parallel. This output can be
+   // 14. Save the refined mesh and the solution in parallel. This output can be
   //     viewed later using GLVis: "glvis -np <np> -m mesh -g sol_*".
   {
      ostringstream mesh_name, u_name, p_name;
@@ -402,7 +322,7 @@ int main(int argc, char *argv[])
      p->Save(p_ofs);
   }

-   // 16. Save data in the VisIt format
+   // 15. Save data in the VisIt format
   VisItDataCollection visit_dc("Example5-Parallel", pmesh);
   visit_dc.RegisterField("velocity", u);
   visit_dc.RegisterField("pressure", p);
@@ -411,7 +331,7 @@ int main(int argc, char *argv[])
                      DataCollection::PARALLEL_FORMAT);
   visit_dc.Save();

-   // 17. Save data in the ParaView format
+   // 16. Save data in the ParaView format
   ParaViewDataCollection paraview_dc("Example5P", pmesh);
   paraview_dc.SetPrefixPath("ParaView");
   paraview_dc.SetLevelsOfDetail(order);
@@ -423,7 +343,7 @@ int main(int argc, char *argv[])
   paraview_dc.RegisterField("pressure",p);
   paraview_dc.Save();

-   // 18. Optionally output a BP (binary pack) file using ADIOS2. This can be
+   // 17. Optionally output a BP (binary pack) file using ADIOS2. This can be
   //     visualized with the ParaView VTX reader.
 #ifdef MFEM_USE_ADIOS2
   if (adios2)
@@ -443,7 +363,7 @@ int main(int argc, char *argv[])
   }
 #endif

-   // 19. Send the solution by socket to a GLVis server.
+   // 18. Send the solution by socket to a GLVis server.
   if (visualization)
   {
      char vishost[] = "localhost";
@@ -463,7 +383,7 @@ int main(int argc, char *argv[])
             << endl;
   }

-   // 20. Free the used memory.
+   // 19. Free the used memory.
   delete fform;
   delete gform;
   delete u;
@@ -475,7 +395,7 @@ int main(int argc, char *argv[])
   delete S;
   delete Md;
   delete MinvBt;
-   delete Bt;
+   delete BT;
   delete B;
   delete M;
   delete mVarf;
@@ -20,7 +20,7 @@
 //               ex6 -pa -d occa-cuda
 //               ex6 -pa -d raja-omp
 //               ex6 -pa -d ceed-cpu
-//             * ex6 -pa -d ceed-cuda
+//               * ex6 -pa -d ceed-cuda
 //               ex6 -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -108,11 +108,7 @@ int main(int argc, char *argv[])
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   BilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   LinearForm b(&fespace);

   ConstantCoefficient one(1.0);
@@ -203,10 +199,9 @@ int main(int argc, char *argv[])
         umf_solver.Mult(B, X);
 #endif
      }
-      else // Diagonal preconditioning in partial assembly mode.
+      else // No preconditioning for now in partial assembly mode.
      {
-         OperatorJacobiSmoother M(a, ess_tdof_list);
-         PCG(*A, M, B, X, 3, 2000, 1e-12, 0.0);
+         CG(*A, B, X, 3, 2000, 1e-12, 0.0);
      }

      // 18. After solving the linear system, reconstruct the solution as a
@@ -20,7 +20,7 @@
 //               mpirun -np 4 ex6p -pa -d occa-cuda
 //               mpirun -np 4 ex6p -pa -d raja-omp
 //               mpirun -np 4 ex6p -pa -d ceed-cpu
-//             * mpirun -np 4 ex6p -pa -d ceed-cuda
+//               * mpirun -np 4 ex6p -pa -d ceed-cuda
 //               mpirun -np 4 ex6p -pa -d ceed-cuda:/gpu/cuda/shared
 //
 // Description:  This is a version of Example 1 with a simple adaptive mesh
@@ -129,11 +129,7 @@ int main(int argc, char *argv[])
   //    the Laplace problem -\Delta u = 1. We don't assemble the discrete
   //    problem yet, this will be done in the main loop.
   ParBilinearForm a(&fespace);
-   if (pa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      a.SetDiagonalPolicy(Operator::DIAG_ONE);
-   }
+   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
   ParLinearForm b(&fespace);

   ConstantCoefficient one(1.0);
@@ -224,26 +220,17 @@ int main(int argc, char *argv[])

      // 17. Solve the linear system A X = B.
      //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-      //     * With partial assembly, use a diagonal preconditioner.
-      Solver *M = NULL;
-      if (pa)
-      {
-         M = new OperatorJacobiSmoother(a, ess_tdof_list);
-      }
-      else
-      {
-         HypreBoomerAMG *amg = new HypreBoomerAMG;
-         amg->SetPrintLevel(0);
-         M = amg;
-      }
+      //     * With partial assembly, use no preconditioner, for now.
+      HypreBoomerAMG *amg = NULL;
+      if (!pa) { amg = new HypreBoomerAMG; amg->SetPrintLevel(0); }
      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-6);
      cg.SetMaxIter(2000);
      cg.SetPrintLevel(3); // print the first and the last iterations only
-      cg.SetPreconditioner(*M);
+      if (amg) { cg.SetPreconditioner(*amg); }
      cg.SetOperator(*A);
      cg.Mult(B, X);
-      delete M;
+      delete amg;

      // 18. Switch back to the host and extract the parallel grid function
      //     corresponding to the finite element approximation X. This is the
@@ -19,12 +19,8 @@
 //
 // Device sample runs:
 //    ex9 -pa
-//    ex9 -ea
-//    ex9 -fa
 //    ex9 -pa -m ../data/periodic-cube.mesh
 //    ex9 -pa -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -146,8 +142,6 @@ int main(int argc, char *argv[])
   int ref_levels = 2;
   int order = 3;
   bool pa = false;
-   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -172,10 +166,6 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -279,16 +269,6 @@ int main(int argc, char *argv[])
      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }
-   else if (ea)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
@@ -448,19 +428,20 @@ int main(int argc, char *argv[])
 FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
 {
+   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
+   if (pa)
   {
      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
      M_solver.SetOperator(M);
      dg_solver = NULL;
   }
+   else
+   {
+      M_prec = new DSmoother(M.SpMat());
+      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
+      M_solver.SetOperator(M.SpMat());
+   }
   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
@@ -16,16 +16,11 @@
 //    mpirun -np 4 ex9p -m ../data/disc-nurbs.mesh -p 2 -rp 1 -dt 0.005 -tf 9
 //    mpirun -np 4 ex9p -m ../data/periodic-square.mesh -p 3 -rp 2 -dt 0.0025 -tf 9 -vs 20
 //    mpirun -np 4 ex9p -m ../data/periodic-cube.mesh -p 0 -o 2 -rp 1 -dt 0.01 -tf 8
-//    mpirun -np 3 ex9p -m ../data/amr-hex.mesh -p 1 -rs 1 -rp 0 -dt 0.005 -tf 0.5
 //
 // Device sample runs:
 //    mpirun -np 4 ex9p -pa
-//    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh
 //    mpirun -np 4 ex9p -pa -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../data/periodic-cube.mesh -d cuda
 //
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
@@ -166,8 +161,6 @@ int main(int argc, char *argv[])
   int par_ref_levels = 0;
   int order = 3;
   bool pa = false;
-   bool ea = false;
-   bool fa = false;
   const char *device_config = "cpu";
   int ode_solver_type = 4;
   double t_final = 10.0;
@@ -195,10 +188,6 @@ int main(int argc, char *argv[])
                  "Order (degree) of the finite elements.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
@@ -330,17 +319,6 @@ int main(int argc, char *argv[])
      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
   }
-   else if (ea)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
   m->AddDomainIntegrator(new MassIntegrator);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
@@ -577,21 +555,28 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
     M_solver(_M.ParFESpace()->GetComm()),
     z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
+   bool pa = _M.GetAssemblyLevel()==AssemblyLevel::PARTIAL;
+
+   if (pa)
   {
      M.Reset(&_M, false);
      K.Reset(&_K, false);
   }
+   else
+   {
+      M.Reset(_M.ParallelAssemble(), true);
+      K.Reset(_K.ParallelAssemble(), true);
+   }

   M_solver.SetOperator(*M);

   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
+   if (pa)
+   {
+      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
+      dg_solver = NULL;
+   }
+   else
   {
      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
@@ -600,11 +585,6 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,

      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }

   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Ginkgo high-performance linear algebra library
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM for solving nonlinear constrained optimization problems, including
@@ -1,13 +1,37 @@
 //                       MFEM Example 1 - Parallel Version
-//                              AmgX Modification
 //
 // Compile with: make ex1p
 //
-// AmgX sample runs:
-//               mpirun -np 4 ex1p
-//               mpirun -np 4 ex1p -d cuda
-//               mpirun -np 10 ex1p --amgx-file amg_pcg.json --amgx-mpi-teams
-//               mpirun -np 4 ex1p --amgx-file amg_pcg.json
+// Sample runs:  mpirun -np 4 ex1p -m ../data/square-disc.mesh
+//               mpirun -np 4 ex1p -m ../data/star.mesh
+//               mpirun -np 4 ex1p -m ../data/star-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/escher.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera.mesh
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed.mesh
+//               mpirun -np 4 ex1p -m ../data/toroid-wedge.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-p2.vtk -o 2
+//               mpirun -np 4 ex1p -m ../data/square-disc-p3.mesh -o 3
+//               mpirun -np 4 ex1p -m ../data/square-disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/star-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/disc-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/pipe-nurbs.mesh -o -1
+//               mpirun -np 4 ex1p -m ../data/ball-nurbs.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/fichera-mixed-p2.mesh -o 2
+//               mpirun -np 4 ex1p -m ../data/star-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/square-disc-surf.mesh
+//               mpirun -np 4 ex1p -m ../data/inline-segment.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-quad.mesh
+//               mpirun -np 4 ex1p -m ../data/amr-hex.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh
+//               mpirun -np 4 ex1p -m ../data/mobius-strip.mesh -o -1 -sc
+//
+// Device sample runs:
+//               mpirun -np 4 ex1p -pa -d cuda
+//               mpirun -np 4 ex1p -pa -d occa-cuda
+//               mpirun -np 4 ex1p -pa -d raja-omp
+//               mpirun -np 4 ex1p -pa -d ceed-cpu
+//               mpirun -np 4 ex1p -pa -d ceed-cuda
+//               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -27,6 +51,7 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include "mpi.h"

 using namespace std;
 using namespace mfem;
@@ -40,16 +65,15 @@ int main(int argc, char *argv[])
   MPI_Comm_rank(MPI_COMM_WORLD, &myid);

   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
+   // const char *mesh_file = "../data/star.mesh";
+   const char *mesh_file = "../data/square-disc.mesh";
   int order = 1;
   bool static_cond = false;
   bool pa = false;
   const char *device_config = "cpu";
-   bool visualization = true;
-   bool amgx_lib = true;
-   bool amgx_mpi_teams = false;
-   const char* amgx_json_file = ""; // JSON file for AmgX
-   int ndevices = 1;
+   bool visualization = false;
+   int nfiles = 1;
+   // const char *out_file = "0_0.gf";

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -61,20 +85,14 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&amgx_lib, "-amgx", "--amgx-lib", "-no-amgx",
-                  "--no-amgx-lib", "Use AmgX in example.");
-   args.AddOption(&amgx_json_file, "--amgx-file", "--amgx-file",
-                  "AMGX solver config file (overrides --amgx-solver, --amgx-verbose)");
-   args.AddOption(&amgx_mpi_teams, "--amgx-mpi-teams", "--amgx-mpi-teams",
-                  "--amgx-mpi-gpu-exclusive", "--amgx-mpi-gpu-exclusive",
-                  "Create MPI teams when using AmgX to load balance between ranks and GPUs.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
-   args.AddOption(&ndevices, "-nd","--gpus-per-node-in-teams-mode",
-                  "Number of GPU devices per node (Only used if amgx_mpi_teams is true).");
+   args.AddOption(&nfiles, "-nf", "--num-files", "Number of files to write.");
+   // args.AddOption(&out_file, "-o", "--outfile",
+   //                "Name of file to write.");

   args.Parse();
   if (!args.Good())
@@ -99,8 +117,8 @@ int main(int argc, char *argv[])
   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -108,23 +126,23 @@ int main(int argc, char *argv[])
   //    more than 10,000 elements.
   {
      int ref_levels =
-         (int)floor(log(10000./mesh.GetNE())/log(2.)/dim);
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
-         mesh.UniformRefinement();
+         mesh->UniformRefinement();
      }
   }

   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
   //    this mesh further in parallel to increase the resolution. Once the
   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
   {
      int par_ref_levels = 2;
      for (int l = 0; l < par_ref_levels; l++)
      {
-         pmesh.UniformRefinement();
+         pmesh->UniformRefinement();
      }
   }

@@ -132,16 +150,13 @@ int main(int argc, char *argv[])
   //    use continuous Lagrange finite elements of the specified order. If
   //    order < 1, we instead use an isoparametric/isogeometric space.
   FiniteElementCollection *fec;
-   bool delete_fec;
   if (order > 0)
   {
      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
   }
-   else if (pmesh.GetNodes())
+   else if (pmesh->GetNodes())
   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
+      fec = pmesh->GetNodes()->OwnFEC();
      if (myid == 0)
      {
         cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -150,10 +165,9 @@ int main(int argc, char *argv[])
   else
   {
      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
+   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec, 1, 0);
+   HYPRE_Int size = fespace->GlobalTrueVSize();
   if (myid == 0)
   {
      cout << "Number of finite element unknowns: " << size << endl;
@@ -164,157 +178,114 @@ int main(int argc, char *argv[])
   //    by marking all the boundary attributes from the mesh as essential
   //    (Dirichlet) and converting them to a list of true dofs.
   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
+   if (pmesh->bdr_attributes.Size())
   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      Array<int> ess_bdr(pmesh->bdr_attributes.Max());
      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

   // 9. Set up the parallel linear form b(.) which corresponds to the
   //    right-hand side of the FEM linear system, which in this case is
   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
+   ParLinearForm *b = new ParLinearForm(fespace);
   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
+   b->AddDomainIntegrator(new DomainLFIntegrator(one));
+   b->Assemble();

   // 10. Define the solution vector x as a parallel finite element grid function
   //     corresponding to fespace. Initialize x with initial guess of zero,
   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
+   ParGridFunction x(fespace);
   x = 0.0;

   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
+   ParBilinearForm *a = new ParBilinearForm(fespace);
+   if (pa) { a->SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   a->AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
   //     system, applying any necessary transformations such as: parallel
   //     assembly, eliminating boundary conditions, applying conforming
   //     constraints for non-conforming AMR, static condensation, etc.
-   if (static_cond) { a.EnableStaticCondensation(); }
-   a.Assemble();
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();

   OperatorPtr A;
   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+   a->FormLinearSystem(ess_tdof_list, x, *b, A, X, B);

   // 13. Solve the linear system A X = B.
   //     * With full assembly, use the BoomerAMG preconditioner from hypre.
-   //     * If AmgX is available solve using amg preconditioner.
   //     * With partial assembly, use Jacobi smoothing, for now.
   Solver *prec = NULL;
   if (pa)
   {
-      if (UsesTensorBasis(fespace))
+      if (UsesTensorBasis(*fespace))
      {
-         prec = new OperatorJacobiSmoother(a, ess_tdof_list);
+         prec = new OperatorJacobiSmoother(*a, ess_tdof_list);
      }
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") == 0)
-   {
-      MFEM_VERIFY(!amgx_mpi_teams,
-                  "Please add JSON file to try AmgX with MPI teams mode");
-
-      bool amgx_verbose = false;
-      prec = new AmgXSolver(MPI_COMM_WORLD, AmgXSolver::PRECONDITIONER,
-                            amgx_verbose);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-
-   }
-   else if (amgx_lib && strcmp(amgx_json_file,"") != 0)
-   {
-      AmgXSolver amgx;
-      amgx.ReadParameters(amgx_json_file, AmgXSolver::EXTERNAL);
-
-      if (amgx_mpi_teams)
-      {
-         // Forms MPI teams to load balance between MPI ranks and GPUs
-         amgx.InitMPITeams(MPI_COMM_WORLD, ndevices);
-      }
-      else
-      {
-         // Assumes each MPI rank is paired with a GPU
-         amgx.InitExclusiveGPU(MPI_COMM_WORLD);
-      }
-
-      amgx.SetOperator(*A.As<HypreParMatrix>());
-      amgx.Mult(B, X);
-
-      // Release MPI communicators and resources created by AmgX
-      amgx.Finalize();
   }
   else
   {
      prec = new HypreBoomerAMG;
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(1);
-      if (prec) { cg.SetPreconditioner(*prec); }
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
   }
+   CGSolver cg(MPI_COMM_WORLD);
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   if (prec) { cg.SetPreconditioner(*prec); }
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete prec;

   // 14. Recover the parallel grid function corresponding to X. This is the
   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
+   a->RecoverFEMSolution(X, *b, x);

-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
+   std::string filename = to_string(num_procs) + "_" + to_string(nfiles) + "_";
   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
+      double t1;
+      t1 = MPI_Wtime();
+      x.Save(filename.c_str(), nfiles);
+      double t2 = MPI_Wtime();

-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
+      double write_time = t2 - t1;
+      double average_write_time;
+      MPI_Reduce(&write_time, &average_write_time, 1,
+               MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+      if (myid == 0)
+      {
+         std::cout << "Average write time: " << average_write_time / num_procs << " for "
+                   << nfiles << " files and "  << num_procs << " ranks\n";
+      }
   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
+      double t1;
+      t1 = MPI_Wtime();
+      ParGridFunction temp_gf(fespace, filename.c_str());
+      double t2 = MPI_Wtime();
+
+      double read_time = t2 - t1;
+      double average_read_time;
+      MPI_Reduce(&read_time, &average_read_time, 1,
+                 MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+      if (myid == 0)
+      {
+         std::cout << "Average read time: " << average_read_time / num_procs << " for "
+                   << nfiles << " files and "  << num_procs << " ranks\n";
+      }
   }

   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
+   delete a;
+   delete b;
+   delete fespace;
+   if (order > 0) { delete fec; }
+   delete pmesh;
+
   MPI_Finalize();

   return 0;
@@ -48,12 +48,6 @@ endif
 ifeq ($(MFEM_USE_GINKGO),YES)
   SUBDIRS += ginkgo
 endif
-ifeq ($(MFEM_USE_AMGX),YES)
-   SUBDIRS += amgx
-endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-   SUBDIRS += superlu
-endif

 SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
 SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
@@ -120,16 +114,6 @@ ex11p-test-strumpack: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), STRUMPACK example,--strumpack)
 test-par-YES: ex11p-test-strumpack
 endif
-ifeq ($(MFEM_USE_SUPERLU),YES)
-ex11p-test-superlu: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), SuperLU_DIST example,--superlu)
-test-par-YES: ex11p-test-superlu
-endif
-ifeq ($(MFEM_USE_MKL_CPARDISO),YES)
-ex11p-test-cpardiso: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
-test-par-YES: ex11p-test-cpardiso
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -0,0 +1,906 @@
+//                                MFEM Example 9
+//
+// Compile with: make serial_nogpu
+//
+// Description:  This code solves the time-dependent advection-diffusion
+//               equation:
+//               \frac(\partial u}{\partial t}
+//                                 = \mathbf{a} \cdot \Nabla u - \nu \Nabla^2 u
+//               where a is a given advection velocity, \nu is the diffusion
+//               parameter, and u0(x) = u(0,x) is a given initial condition.
+//
+//               The demonstrates explicit time marching with H1 elements of
+//               arbitrary order. Periodic boundary conditions are used through
+//               periodic meshes. GLVis can be used for visualization of a
+//               time-evolving solution.
+
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+
+#include "mfem.hpp"
+#include "mpi.h"
+
+using namespace std;
+using namespace mfem;
+
+/** A time-dependent operator for the right-hand side of the ODE. The weak
+    form of du/dt = -a.grad(u) + nu Delta(u) is M du/dt = K u + b, where M and
+    K are the mass and advection-diffusion matrices, and b describes the flow
+    on the boundary. This can be written as a general ODE,
+    du/dt = M^{-1} (K u + b), and this class is used to evaluate the right-hand
+    side. */
+class AdvectionDiffusionEvolution : public mfem::TimeDependentOperator
+{
+public:
+   /// \param[in] M - bilinear form for mass matrix
+   /// \param[in] K - bilinear form for stiffness matrix
+   /// \param[in] b - load vector
+   AdvectionDiffusionEvolution(mfem::BilinearForm &M, mfem::BilinearForm &K,
+                               const mfem::Vector &b);
+
+   /// Perform the action of the operator: y = k = f(x, t), where k solves
+   /// Compute k = M^-1(Kx + l)
+   void Mult(const mfem::Vector &x, mfem::Vector &y) const override;
+
+   /// Solve the implicit equation: k = f(x + dt k, t), for the unknown k at
+   /// the current time t. 
+   void ImplicitSolve(const double dt, const mfem::Vector &x,
+                      mfem::Vector &k) override;
+
+   virtual ~AdvectionDiffusionEvolution();
+
+private:
+   mfem::BilinearForm &M, &K;
+   const mfem::Vector &b;
+   /// solver for inverting mass matrix for explicit time-marching
+   std::unique_ptr<mfem::Solver> M_prec;
+   mfem::CGSolver M_solver;
+   /// solver for implicit time-marching
+   mfem::GSSmoother prec;
+   mfem::GMRESSolver linear_solver;
+   mfem::NewtonSolver newton;
+
+   mutable mfem::Vector z;
+
+   /// pointer-to-implementation idiom
+   /// Hides implementation details of this operator
+   class SystemOperator;
+   /// Operator that combines the linear spatial discretization with
+   /// the load vector into one operator used for implicit solves
+   std::unique_ptr<SystemOperator> combined_oper;
+
+   /// sets the state and dt for the combined operator
+   /// \param[in] dt - time increment
+   /// \param[in] x - the current state
+   void setOperParameters(double dt, const mfem::Vector *x);
+
+};
+
+class PAJacobianOperator : public mfem::Operator
+{
+public:
+   PAJacobianOperator(mfem::ParBilinearForm &_mass,
+                      mfem::ParBilinearForm &_stiff);
+
+   /// Compute r = J@k = M@k + dt*K@k
+   /// \param[in] k - dx/dt
+   /// \param[out] r - J@k = M@k + dt*K@k
+   void Mult(const mfem::Vector &k, mfem::Vector &r) const override;
+      
+   /// Set current dt values - needed to compute action of Jacobian.
+   void setParameters(double dt);
+
+private:
+   mfem::ParBilinearForm &mass;
+   mfem::ParBilinearForm &stiff;
+
+   double dt;
+};
+
+class ParSystemOperator : public mfem::Operator
+{
+public:
+   /// Nonlinear operator of the form that combines the mass, res, stiff,
+   /// and load elements for implicit/explicit ODE integration
+   /// \param[in] ess_bdr - array of boundaries attributes marked essential
+   /// \param[in] mass - bilinear form for mass matrix (not owned)
+   /// \param[in] res - nonlinear residual operator (not owned)
+   /// \param[in] stiff - bilinear form for stiffness matrix (not owned)
+   /// \param[in] load - load vector (not owned)
+   /// \param[in] a - used to move the spatial residual to the rhs
+   ParSystemOperator(mfem::ParBilinearForm &_mass,
+                     mfem::ParBilinearForm &_stiff);
+
+   /// Compute r = M@k + K@(x+dt*k)
+   /// (with `@` denoting matrix-vector multiplication)
+   /// \param[in] k - dx/dt 
+   /// \param[out] r - the residual
+   /// \note the signs on each operator must be accounted for elsewhere
+   void Mult(const mfem::Vector &k, mfem::Vector &r) const override;
+
+   /// Compute J = M + dt * K
+   /// \param[in] k - dx/dt 
+   mfem::Operator &GetGradient(const mfem::Vector &k) const override;
+
+   /// Set current dt and x values - needed to compute action and Jacobian.
+   void setParameters(double _dt, const mfem::Vector *_x);
+
+   ~ParSystemOperator();
+
+private:
+   mfem::ParBilinearForm &mass;
+   mfem::ParBilinearForm &stiff;
+   mutable mfem::HypreParMatrix *jacobian, *stiff_jacobian;
+
+   double dt;
+   const mfem::Vector *x;
+
+   mutable mfem::Vector work, work2;
+
+   std::unique_ptr<PAJacobianOperator> pa_jac;
+
+};
+
+/** A time-dependent operator for the right-hand side of the ODE. The weak
+    form of du/dt = -a.grad(u) + nu Delta(u) is M du/dt = K u + b, where M and
+    K are the mass and advection-diffusion matrices, and b describes the flow
+    on the boundary. This can be written as a general ODE,
+    du/dt = M^{-1} (K u + b), and this class is used to evaluate the right-hand
+    side. */
+class ParAdvectionDiffusionEvolution : public mfem::TimeDependentOperator
+{
+public:
+   /// \param[in] M - parallel bilinear form for mass matrix
+   /// \param[in] K - parallel bilinear form for stiffness matrix
+   ParAdvectionDiffusionEvolution(mfem::ParBilinearForm &M,
+                                  mfem::ParBilinearForm &K);
+
+   /// Perform the action of the operator: y = k = f(x, t), where k solves
+   /// Compute k = M^-1(Kx + l)
+   void Mult(const mfem::Vector &x, mfem::Vector &y) const override;
+
+   /// Solve the implicit equation: k = f(x + dt k, t), for the unknown k at
+   /// the current time t. 
+   void ImplicitSolve(const double dt, const mfem::Vector &x,
+                      mfem::Vector &k) override;
+
+   virtual ~ParAdvectionDiffusionEvolution();
+
+private:
+   mfem::OperatorHandle M_;
+   mfem::ParBilinearForm &M, &K;
+   /// solver for inverting mass matrix for explicit time-marching
+   std::unique_ptr<mfem::Solver> M_prec;
+   mfem::CGSolver M_solver;
+   /// solver for implicit time-marching
+   mfem::Solver *prec;
+   mfem::GMRESSolver linear_solver;
+   mfem::NewtonSolver newton;
+
+   mfem::Vector diag;
+   mutable mfem::Vector z, work, work2;
+
+   /// pointer-to-implementation idiom
+   /// Hides implementation details of this operator
+   /// Operator that combines the linear spatial discretization with
+   /// the load vector into one operator used for implicit solves
+   std::unique_ptr<ParSystemOperator> combined_oper;
+
+   /// sets the state and dt for the combined operator
+   /// \param[in] dt - time increment
+   /// \param[in] x - the current state
+   void setOperParameters(double dt, const mfem::Vector *x);
+
+};
+
+// Choice for the problem setup. The fluid velocity, initial condition and
+// inflow boundary condition are chosen based on this parameter.
+int problem;
+
+// Velocity coefficient
+void velocity_function(const Vector &X, Vector &v);
+
+// Initial condition
+double u0_function(const Vector &X);
+
+// Inflow boundary condition
+double inflow_function(const Vector &X, const double t);
+
+// Mesh bounding box
+Vector bb_min, bb_max;
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI.
+   int num_procs, myid;
+   MPI_Init(&argc, &argv);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
+
+   // 2. Parse command-line options.
+   problem = 3;
+   const char *mesh_file = "../data/periodic-square.mesh";
+   int ser_ref_levels = 0;
+   int par_ref_levels = 0;
+   int order = 3;
+   const char *device_config = "cpu";
+   int ode_solver_type = 22;
+   double t_final = 3 * 2*M_PI;
+   double dt = 0.01;
+   bool glvis = false;
+   bool paraview = false;
+   int vis_steps = 5;
+
+   double nu_val = 0.001;
+
+   int precision = 8;
+   cout.precision(precision);
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&problem, "-p", "--problem",
+                  "Problem setup to use. See options in velocity_function().");
+   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
+                  "Number of times to refine the mesh uniformly in serial.");
+   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
+                  "Number of times to refine the mesh uniformly in parallel.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&device_config, "-d", "--device",
+                  "Device configuration string, see Device::Configure().");
+   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
+                  "ODE solver: 1 - Forward Euler,\n\t"
+                  "            2 - RK2 SSP, 3 - RK3 SSP, 4 - RK4, 6 - RK6.");
+   args.AddOption(&t_final, "-tf", "--t-final",
+                  "Final time; start time is 0.");
+   args.AddOption(&dt, "-dt", "--time-step",
+                  "Time step.");
+   args.AddOption(&glvis, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
+                  "--no-paraview-datafiles",
+                  "Save data files for ParaView (paraview.org) visualization.");
+   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
+                  "Visualize every n-th timestep.");
+   args.AddOption(&nu_val, "-nu", "--nu-value",
+                  "Value for \nu, the parameter that controls diffusion.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      std::cout << "Num ranks: " << num_procs << "\n";
+      args.PrintOptions(cout);
+   }
+   Device device(device_config);
+   if (myid == 0) { device.Print(); }
+
+   // 3. Read the serial mesh from the given mesh file on all processors. We can
+   //    handle geometrically periodic meshes in this code.
+   Mesh *mesh = new Mesh(mesh_file, 1, 1);
+   int dim = mesh->Dimension();
+
+   // 5. Refine the mesh in serial to increase the resolution. In this example
+   //    we do 'ser_ref_levels' of uniform refinement, where 'ser_ref_levels' is
+   //    a command-line parameter. If the mesh is of NURBS type, we convert it
+   //    to a (piecewise-polynomial) high-order mesh.
+   for (int lev = 0; lev < ser_ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+   mesh->GetBoundingBox(bb_min, bb_max, max(order, 1));
+
+   // 6. Define the parallel mesh by a partitioning of the serial mesh. Refine
+   //    this mesh further in parallel to increase the resolution. Once the
+   //    parallel mesh is defined, the serial mesh can be deleted.
+   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
+   delete mesh;
+   for (int lev = 0; lev < par_ref_levels; lev++)
+   {
+      pmesh->UniformRefinement();
+   }
+
+   // 7. Define the finite element space of the given
+   //    polynomial order on the refined mesh.
+   H1_FECollection fec(order, dim, BasisType::GaussLobatto);
+   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);
+
+   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of unknowns: " << global_vSize << endl;
+   }
+
+   // 8. Set up and assemble the bilinear and linear forms corresponding to the
+   //    CG discretization.
+   /// negative to move the diffusion terms to the right side
+   ConstantCoefficient nu(-nu_val);
+   ConstantCoefficient one(1.0);
+   VectorFunctionCoefficient velocity(dim, velocity_function);
+   FunctionCoefficient u0(u0_function);
+
+   ParBilinearForm *m_pa = new ParBilinearForm(fes);
+   ParBilinearForm *k_pa = new ParBilinearForm(fes);
+   m_pa->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+   k_pa->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+
+   /// create mass matrix
+   m_pa->AddDomainIntegrator(new MassIntegrator(one));
+   /// add advection terms to stiffness matrix
+   k_pa->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   /// add diffusion terms to stiffness matrix
+   k_pa->AddDomainIntegrator(new DiffusionIntegrator(nu));
+
+   m_pa->Assemble();
+   int skip_zeros = 0;
+   k_pa->Assemble(skip_zeros);
+   m_pa->Finalize();
+   k_pa->Finalize(skip_zeros);
+
+   ParBilinearForm *m = new ParBilinearForm(fes);
+   ParBilinearForm *k = new ParBilinearForm(fes);
+   /// create mass matrix
+   m->AddDomainIntegrator(new MassIntegrator);
+   /// add advection terms to stiffness matrix
+   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
+   /// add diffusion terms to stiffness matrix
+   k->AddDomainIntegrator(new DiffusionIntegrator(nu));
+
+   m->Assemble();
+   k->Assemble(skip_zeros);
+   m->Finalize();
+   k->Finalize(skip_zeros);
+
+
+   ParGridFunction *u = new ParGridFunction(fes);
+   u->UseDevice(true);
+   u->ProjectCoefficient(u0);
+
+
+   HypreParVector *U = u->GetTrueDofs();
+
+   ParSystemOperator pso(*m, *k);
+   ParSystemOperator pso_pa(*m_pa, *k_pa);
+
+   pso.setParameters(dt, U);
+   pso_pa.setParameters(dt, U);
+
+   MPI_Barrier(MPI_COMM_WORLD);
+   mfem::Vector pso_r(U->Size());
+   double t1 = MPI_Wtime();
+   pso.Mult(*U, pso_r);
+   double t2 = MPI_Wtime();
+   double fa_mult_time = t2 - t1;
+   double average_fa_mult_time;
+   MPI_Reduce(&fa_mult_time, &average_fa_mult_time, 1,
+              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+   if (myid == 0)
+      std::cout << "FA Mult time: " << average_fa_mult_time / num_procs << endl;
+
+   MPI_Barrier(MPI_COMM_WORLD);
+   mfem::Vector pso_pa_r(U->Size());
+   double t3 = MPI_Wtime();
+   pso_pa.Mult(*U, pso_pa_r);
+   double t4 = MPI_Wtime();
+   double pa_mult_time = t4 - t3;
+   double average_pa_mult_time;
+   MPI_Reduce(&pa_mult_time, &average_pa_mult_time, 1,
+              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+   if (myid == 0)
+      std::cout << "FA Mult time: " << average_pa_mult_time / num_procs << endl;
+   
+   double local_mult_speedup = (t2-t1) / (t4-t3);
+   double global_mult_speedup;
+   MPI_Reduce(&local_mult_speedup, &global_mult_speedup, 1,
+              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+
+   if (myid == 0)
+      std::cout << "PA mult speedup: " << global_mult_speedup / num_procs << endl;
+
+   mfem::Vector diff_r(pso_pa_r);
+   diff_r -= pso_r;
+   // std::cout << "r diff: " << diff_r.Norml2() << std::endl;
+
+   mfem::Operator &pso_jac = pso.GetGradient(*U);
+   mfem::Operator &pso_pa_jac = pso_pa.GetGradient(*U);
+
+   MPI_Barrier(MPI_COMM_WORLD);
+   mfem::Vector pso_jac_r(U->Size());
+   double t5 = MPI_Wtime();
+   pso_jac.Mult(*U, pso_jac_r);
+   double t6 = MPI_Wtime();
+   double fa_jac_mult_time = t6-t5;
+   double average_fa_jac_time;
+   MPI_Reduce(&fa_jac_mult_time, &average_fa_jac_time, 1,
+              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+   if (myid == 0)
+      std::cout << "FA Jac Mult time: " << average_fa_jac_time / num_procs << endl;
+
+   MPI_Barrier(MPI_COMM_WORLD);
+   mfem::Vector pso_pa_jac_r(U->Size());
+   double t7 = MPI_Wtime();
+   pso_pa_jac.Mult(*U, pso_pa_jac_r);
+   double t8 = MPI_Wtime();
+   double pa_jac_mult_time = t8-t7;
+   double average_pa_jac_time;
+   MPI_Reduce(&pa_jac_mult_time, &average_pa_jac_time, 1,
+              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+   if (myid == 0)
+      std::cout << "PA Jac Mult time: " << average_pa_jac_time / num_procs << endl;
+
+   double local_jac_speedup = (t6-t5) / (t8-t7);
+   double global_jac_speedup;
+   MPI_Reduce(&local_jac_speedup, &global_jac_speedup, 1,
+              MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+   if (myid == 0)
+      std::cout << "PA Jac mult speedup: " << global_jac_speedup / num_procs << endl;
+
+   // 13. Free the used memory.
+   delete U;
+   delete u;
+   delete k;
+   delete m;
+   delete fes;
+   delete pmesh;
+
+   MPI_Finalize();
+   return 0;
+}
+
+// Velocity coefficient
+void velocity_function(const Vector &x, Vector &v)
+{
+   int dim = x.Size();
+
+   // map to the reference [-1,1] domain
+   Vector X(dim);
+   for (int i = 0; i < dim; i++)
+   {
+      double center = (bb_min[i] + bb_max[i]) * 0.5;
+      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
+   }
+
+   switch (problem)
+   {
+      case 3:
+      {
+         // Translations in 1D, 2D, and 3D
+         switch (dim)
+         {
+            case 1: v(0) = 1.0; break;
+            case 2: v(0) = sqrt(2./3.); v(1) = sqrt(1./3.); break;
+            case 3: v(0) = sqrt(3./6.); v(1) = sqrt(2./6.); v(2) = sqrt(1./6.);
+               break;
+         }
+         break;
+      }
+      case 1:
+      case 2:
+      {
+         // Clockwise rotation in 2D around the origin
+         const double w = M_PI/2;
+         switch (dim)
+         {
+            case 1: v(0) = 1.0; break;
+            case 2: v(0) = w*X(1); v(1) = -w*X(0); break;
+            case 3: v(0) = w*X(1); v(1) = -w*X(0); v(2) = 0.0; break;
+         }
+         break;
+      }
+      case 0:
+      {
+         // Clockwise twisting rotation in 2D around the origin
+         const double w = M_PI/2;
+         double d = max((X(0)+1.)*(1.-X(0)),0.) * max((X(1)+1.)*(1.-X(1)),0.);
+         d = d*d;
+         switch (dim)
+         {
+            case 1: v(0) = 1.0; break;
+            case 2: v(0) = d*w*X(1); v(1) = -d*w*X(0); break;
+            case 3: v(0) = d*w*X(1); v(1) = -d*w*X(0); v(2) = 0.0; break;
+         }
+         break;
+      }
+   }
+}
+
+// Initial condition
+double u0_function(const Vector &x)
+{
+   int dim = x.Size();
+
+   // map to the reference [-1,1] domain
+   Vector X(dim);
+   for (int i = 0; i < dim; i++)
+   {
+      double center = (bb_min[i] + bb_max[i]) * 0.5;
+      X(i) = 2 * (x(i) - center) / (bb_max[i] - bb_min[i]);
+   }
+
+   switch (problem)
+   {
+      case 0:
+      case 1:
+      {
+         switch (dim)
+         {
+            case 1:
+               return exp(-40.*pow(X(0)-0.5,2));
+            case 2:
+            case 3:
+            {
+               double rx = 0.45, ry = 0.25, cx = 0., cy = -0.2, w = 10.;
+               if (dim == 3)
+               {
+                  const double s = (1. + 0.25*cos(2*M_PI*X(2)));
+                  rx *= s;
+                  ry *= s;
+               }
+               return ( erfc(w*(X(0)-cx-rx))*erfc(-w*(X(0)-cx+rx)) *
+                        erfc(w*(X(1)-cy-ry))*erfc(-w*(X(1)-cy+ry)) )/16;
+            }
+         }
+      }
+      case 2:
+      {
+         double x_ = X(0), y_ = X(1), rho, phi;
+         rho = hypot(x_, y_);
+         phi = atan2(y_, x_);
+         return pow(sin(M_PI*rho),2)*sin(3*phi);
+      }
+      case 3:
+      {
+         const double f = M_PI;
+         return sin(f*X(0))*sin(f*X(1));
+      }
+   }
+   return 0.0;
+}
+
+// Inflow boundary condition (zero for the problems considered in this example)
+double inflow_function(const Vector &x, const double t)
+{
+   switch (problem)
+   {
+      case 0:
+      case 1:
+      case 2:
+      case 3: return 0.0;
+   }
+   return 0.0;
+}
+
+class AdvectionDiffusionEvolution::SystemOperator : public mfem::Operator
+{
+public:
+   /// Nonlinear operator of the form that combines the mass, res, stiff,
+   /// and load elements for implicit/explicit ODE integration
+   /// \param[in] mass - bilinear form for mass matrix (not owned)
+   /// \param[in] res - nonlinear residual operator (not owned)
+   /// \param[in] stiff - bilinear form for stiffness matrix (not owned)
+   /// \param[in] load - load vector (not owned)
+   /// \param[in] a - used to move the spatial residual to the rhs
+   SystemOperator(BilinearForm &_mass, BilinearForm &_stiff,
+                  const mfem::Vector &b)
+      : Operator(_mass.Height()), mass(_mass), stiff(_stiff),
+        load(b), Jacobian(NULL), dt(0.0), x(NULL), work(height)
+      { }
+
+   /// Compute r = M@k + K@(x+dt*k) + l
+   /// (with `@` denoting matrix-vector multiplication)
+   /// \param[in] k - dx/dt 
+   /// \param[out] r - the residual
+   /// \note the signs on each operator must be accounted for elsewhere
+   void Mult(const mfem::Vector &k, mfem::Vector &r) const override
+   {
+      /// work = x+dt*k = x+dt*dx/dt = x+dx
+      add(1.0, *x, dt, k, work);
+      r = 0.0;
+      stiff.AddMult(work, r);
+      r += load;
+      mass.AddMult(k, r, -1.0);
+   }
+
+   /// Compute J = M + dt * K
+   /// \param[in] k - dx/dt 
+   mfem::Operator &GetGradient(const mfem::Vector &k) const override
+   {
+      delete Jacobian;
+      Jacobian = Add(-1.0, mass.SpMat(), dt, stiff.SpMat());
+      return *Jacobian;
+   }
+
+   /// Set current dt and x values - needed to compute action and Jacobian.
+   void setParameters(double _dt, const mfem::Vector *_x)
+   {
+      dt = _dt;
+      x = _x;
+   };
+
+   ~SystemOperator() {delete Jacobian;};
+
+private:
+   BilinearForm &mass;
+   BilinearForm &stiff;
+   const mfem::Vector &load;
+   mutable mfem::SparseMatrix *Jacobian;
+
+   double dt;
+   const mfem::Vector *x;
+
+   mutable mfem::Vector work, work2;
+
+};
+
+AdvectionDiffusionEvolution::AdvectionDiffusionEvolution(
+      BilinearForm &_M, BilinearForm &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b),
+     z(_M.Height())
+{
+   bool pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+   Array<int> ess_tdof_list;
+   if (pa)
+   {
+      M_prec.reset(new OperatorJacobiSmoother(M, ess_tdof_list));
+      M_solver.SetOperator(M);
+   }
+   else
+   {
+      M_prec.reset(new DSmoother(M.SpMat()));
+      M_solver.SetOperator(M.SpMat());
+   }
+
+   combined_oper.reset(new SystemOperator(_M, _K, _b));
+
+   M_solver.SetPreconditioner(*M_prec);
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(1e-9);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(100);
+   M_solver.SetPrintLevel(0);
+
+   linear_solver.iterative_mode = true;
+   linear_solver.SetRelTol(1e-12);
+   linear_solver.SetAbsTol(0.0);
+   linear_solver.SetMaxIter(100);
+   linear_solver.SetPrintLevel(0);
+   linear_solver.SetPreconditioner(prec);
+
+   newton.iterative_mode = false;
+   newton.SetRelTol(1e-9);
+   newton.SetAbsTol(0.0);
+   newton.SetMaxIter(100);
+   newton.SetPrintLevel(-1);
+   newton.SetSolver(linear_solver);
+   newton.SetOperator(*combined_oper);
+}
+
+void AdvectionDiffusionEvolution::Mult(const Vector &x, Vector &y) const
+{
+   // y = M^{-1} (K x + b)
+   K.Mult(x, z);
+   z += b;
+   M_solver.Mult(z, y);
+}
+
+void AdvectionDiffusionEvolution::ImplicitSolve(const double dt,
+                                                const Vector &x,
+                                                Vector &k)
+{
+   setOperParameters(dt, &x);
+   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
+   newton.Mult(zero, k);
+   MFEM_VERIFY(newton.GetConverged(), "Newton solver did not converge!");
+}
+
+void AdvectionDiffusionEvolution::setOperParameters(double dt,
+                                                    const mfem::Vector *x)
+{
+   combined_oper->setParameters(dt, x);
+}
+
+AdvectionDiffusionEvolution::~AdvectionDiffusionEvolution() {}
+
+
+PAJacobianOperator::PAJacobianOperator(ParBilinearForm &_mass, ParBilinearForm &_stiff)
+ : Operator(_mass.ParFESpace()->GetTrueVSize()), mass(_mass), stiff(_stiff),
+   dt(0.0) { }
+
+
+void PAJacobianOperator::Mult(const mfem::Vector &k, mfem::Vector &r) const
+{
+   r.UseDevice(true);
+   r = 0.0;
+   stiff.TrueAddMult(k, r, dt);
+   mass.TrueAddMult(k, r, -1.0);
+}
+   
+void PAJacobianOperator::setParameters(const double _dt)
+{
+   dt = _dt;
+};
+
+ParSystemOperator::ParSystemOperator(ParBilinearForm &_mass, ParBilinearForm &_stiff)
+   : Operator(_mass.ParFESpace()->GetTrueVSize()), mass(_mass), stiff(_stiff),
+   jacobian(NULL), stiff_jacobian(NULL), dt(0.0), x(NULL),
+   work(height)
+{
+   pa_jac.reset(new PAJacobianOperator(mass, stiff));
+}
+
+/// Compute r = M@k + K@(x+dt*k)
+/// (with `@` denoting matrix-vector multiplication)
+/// \param[in] k - dx/dt 
+/// \param[out] r - the residual
+/// \note the signs on each operator must be accounted for elsewhere
+void ParSystemOperator::Mult(const mfem::Vector &k, mfem::Vector &r) const
+{
+   r = 0.0;
+   work.UseDevice(true);
+   work = 0.0;
+   /// work = x+dt*k = x+dt*dx/dt = x+dx
+   if (x)
+   {
+      add(1.0, *x, dt, k, work);
+   }
+
+   stiff.TrueAddMult(work, r);
+   mass.TrueAddMult(k, r, -1.0);
+}
+
+/// Compute J = M + dt * K
+/// \param[in] k - dx/dt 
+mfem::Operator &ParSystemOperator::GetGradient(const mfem::Vector &k) const
+{
+   bool mass_pa = mass.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+   bool stiff_pa = stiff.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+   
+   if (mass_pa && stiff_pa)
+   {
+      return *pa_jac.get();
+   }
+   else
+   {
+      delete stiff_jacobian;
+      delete jacobian;
+      jacobian = mass.ParallelAssemble();
+      *jacobian *= -1.0; //alpha;
+      stiff_jacobian = stiff.ParallelAssemble();
+      jacobian->Add(dt, *stiff_jacobian);
+      return *jacobian;
+   }
+}
+
+/// Set current dt and x values - needed to compute action and Jacobian.
+void ParSystemOperator::setParameters(const double _dt, const mfem::Vector *_x)
+{
+   dt = _dt;
+   x = _x;
+   pa_jac->setParameters(_dt);
+};
+
+ParSystemOperator::~ParSystemOperator()
+{
+   delete jacobian;
+   delete stiff_jacobian;
+};
+
+ParAdvectionDiffusionEvolution::ParAdvectionDiffusionEvolution(
+      ParBilinearForm &_M, ParBilinearForm &_K)
+   : TimeDependentOperator(_M.ParFESpace()->GetTrueVSize()), M(_M), K(_K), z(_M.Height())
+{
+   bool mass_pa = M.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+   bool stiff_pa = K.GetAssemblyLevel() == AssemblyLevel::PARTIAL;
+
+   Array<int> ess_tdof_list;
+   M_solver = CGSolver(MPI_COMM_WORLD);
+   if (mass_pa)
+   {
+      M_prec.reset(new OperatorJacobiSmoother(M, ess_tdof_list));
+      M_solver.SetOperator(M);
+   }
+   else
+   {
+      M_.Reset(_M.ParallelAssemble(), true);
+
+      // M_prec.reset(new HypreSmoother());
+      // M_solver.SetOperator(M.As<HypreParMatrix>());
+      HypreParMatrix &M_mat = *M_.As<HypreParMatrix>();
+      // HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
+      M_prec.reset(new HypreSmoother(M_mat, HypreSmoother::Jacobi));
+   }
+
+   combined_oper.reset(new ParSystemOperator(_M, _K));
+
+   M_solver.SetPreconditioner(*M_prec);
+   M_solver.iterative_mode = false;
+   M_solver.SetRelTol(1e-9);
+   M_solver.SetAbsTol(0.0);
+   M_solver.SetMaxIter(100);
+   M_solver.SetPrintLevel(0);
+
+   if (mass_pa && stiff_pa)
+   {
+      diag.UseDevice(true);
+      diag.SetSize(M.ParFESpace()->GetTrueVSize());
+      diag = 0.0;
+      work.UseDevice(true);
+      work2.UseDevice(true);
+      work.SetSize(M.ParFESpace()->GetTrueVSize());
+      work2.SetSize(M.ParFESpace()->GetTrueVSize());
+      work = 0.0;
+      work2 = 0.0;
+      M.AssembleDiagonal(work);
+
+      ParBilinearForm k(M.ParFESpace());
+      ConstantCoefficient nu(-0.01);
+      k.AddDomainIntegrator(new mfem::DiffusionIntegrator(nu));
+      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
+      k.Assemble(0);
+      k.Finalize(0);
+      k.AssembleDiagonal(work2);
+
+      double dt = 0.1;
+      add(-1.0, work, dt, work2, diag);
+
+      prec = new OperatorChebyshevSmoother(combined_oper.get(), diag,
+                                           ess_tdof_list, 5,
+                                           M.ParFESpace()->GetComm());
+   }
+   else
+   {
+      prec = new HypreSmoother();
+   }
+   
+   linear_solver = GMRESSolver(MPI_COMM_WORLD);
+   linear_solver.iterative_mode = true;
+   linear_solver.SetRelTol(1e-12);
+   linear_solver.SetAbsTol(0.0);
+   linear_solver.SetMaxIter(2000);
+   linear_solver.SetPrintLevel(0);
+   linear_solver.SetPreconditioner(*prec);
+   linear_solver.SetKDim(2000);
+
+   newton.iterative_mode = true;
+   newton.SetRelTol(1e-9);
+   newton.SetAbsTol(0.0);
+   newton.SetMaxIter(10);
+   newton.SetPrintLevel(-1);
+   newton.SetSolver(linear_solver);
+   newton.SetOperator(*combined_oper);
+}
+
+void ParAdvectionDiffusionEvolution::Mult(const Vector &x, Vector &y) const
+{
+   // y = M^{-1} (K x + b)
+   K.Mult(x, z);
+   M_solver.Mult(z, y);
+}
+
+void ParAdvectionDiffusionEvolution::ImplicitSolve(const double dt,
+                                                   const Vector &x,
+                                                   Vector &k)
+{
+   setOperParameters(dt, &x);
+   Vector zero; // empty vector is interpreted as zero r.h.s. by NewtonSolver
+   newton.Mult(zero, k);
+   MFEM_VERIFY(newton.GetConverged(), "Newton solver did not converge!");
+}
+
+void ParAdvectionDiffusionEvolution::setOperParameters(const double dt,
+                                                       const mfem::Vector *x)
+{
+   combined_oper->setParameters(dt, x);
+}
+
+ParAdvectionDiffusionEvolution::~ParAdvectionDiffusionEvolution() {delete prec;}
@@ -34,15 +34,6 @@ if (MFEM_USE_MPI)
    )
 endif()

-if (MFEM_USE_SLEPC)
-  list(APPEND PETSC_EXAMPLES_SRCS
-    ex11p.cpp
-    )
-  list(APPEND PETSC_RC_FILES
-    rc_ex11p_lobpcg rc_ex11p_gd
-    )
-endif()
-
 # Include the source directory where mfem.hpp and mfem-performance.hpp are.
 include_directories(BEFORE ${PROJECT_BINARY_DIR})

@@ -87,22 +78,12 @@ set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
-if (MFEM_USE_SLEPC)
-  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
-  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
-  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
-endif()

 # Add the tests: one test per command-line-variable.
-set(TEST_OPTIONS_VARS
-    EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-    EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-    EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
-if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-endif()
-
-foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
+foreach(TEST_OPTIONS_VAR
+        EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+        EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+        EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
  string(REGEX REPLACE "^(.+)_ARGS" "\\1" TEST_NAME_UC ${TEST_OPTIONS_VAR})
  string(REGEX REPLACE "^([^_]+)" "\\1P" TEST_NAME_UC ${TEST_NAME_UC})
  string(TOLOWER ${TEST_NAME_UC} TEST_NAME_FULL)
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the PETSc suite.
@@ -1,435 +0,0 @@
-//                       MFEM Example 11 - Parallel Version
-//                              PETSc Modification
-//
-// Compile with: make ex11p
-//
-// Sample runs:  mpirun -np 4 ex11p -m ../../data/star.mesh
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_lobpcg
-//               mpirun -np 4 ex11p -m ../../data/star.mesh --slepcopts rc_ex11p_gd
-//
-// Description:  This example code demonstrates the use of MFEM to solve the
-//               eigenvalue problem -Delta u = lambda u with homogeneous
-//               Dirichlet boundary conditions.
-//
-//               We compute a number of the lowest eigenmodes by discretizing
-//               the Laplacian and Mass operators using a FE space of the
-//               specified order, or an isoparametric/isogeometric space if
-//               order < 1 (quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example demonstrates the use of the SLEPc eigensolver as an
-//               alternative to the LOBPCG eigenvalue solver. The shift and
-//               invert spectral transformation is used to help the convergence
-//               to the smaller eigenvalues. Alternative solver parameters can
-//               be passed in a file with "-slepcopts".
-//
-//               Reusing a single GLVis visualization window for multiple
-//               eigenfunctions is also illustrated.
-//
-//               We recommend viewing Example 1 before viewing this example.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#ifndef MFEM_USE_SLEPC
-#error This examples requires that MFEM is build with MFEM_USE_SLEPC=YES
-#endif
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int ser_ref_levels = 2;
-   int par_ref_levels = 1;
-   int order = 1;
-   int nev = 5;
-   int seed = 75;
-   bool slu_solver  = false;
-   bool sp_solver = false;
-   bool visualization = 1;
-   bool use_slepc = true;
-   const char *slepcrc_file = "";
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ser_ref_levels, "-rs", "--refine-serial",
-                  "Number of times to refine the mesh uniformly in serial.");
-   args.AddOption(&par_ref_levels, "-rp", "--refine-parallel",
-                  "Number of times to refine the mesh uniformly in parallel.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&nev, "-n", "--num-eigs",
-                  "Number of desired eigenmodes.");
-   args.AddOption(&seed, "-s", "--seed",
-                  "Random seed used to initialize LOBPCG.");
-#ifdef MFEM_USE_SUPERLU
-   args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu",
-                  "--no-superlu", "Use the SuperLU Solver.");
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   args.AddOption(&sp_solver, "-sp", "--strumpack", "-no-sp",
-                  "--no-strumpack", "Use the STRUMPACK Solver.");
-#endif
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&use_slepc, "-useslepc","--useslepc","-no-slepc",
-                  "--no-slepc","Use or not SLEPc to solve the eigenvalue problem");
-   args.AddOption(&slepcrc_file, "-slepcopts", "--slepcopts",
-                  "SlepcOptions file to use.");
-   args.Parse();
-   if (slu_solver && sp_solver)
-   {
-      if (myid == 0)
-         cout << "WARNING: Both SuperLU and STRUMPACK have been selected,"
-              << " please choose either one." << endl
-              << "         Defaulting to SuperLU." << endl;
-      sp_solver = false;
-   }
-   // The command line options are also passed to the STRUMPACK
-   // solver. So do not exit if some options are not recognized.
-   if (!sp_solver)
-   {
-      if (!args.Good())
-      {
-         if (myid == 0)
-         {
-            args.PrintUsage(cout);
-         }
-         MPI_Finalize();
-         return 1;
-      }
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 2b. We initialize SLEPc. This internally initializes PETSc as well.
-   MFEMInitializeSlepc(NULL,NULL,slepcrc_file,NULL);
-
-   // 3. Read the (serial) mesh from the given mesh file on all processors. We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh *mesh = new Mesh(mesh_file, 1, 1);
-   int dim = mesh->Dimension();
-
-   // 4. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
-   //    specified on the command line with -rs).
-   for (int lev = 0; lev < ser_ref_levels; lev++)
-   {
-      mesh->UniformRefinement();
-   }
-
-   // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution (1 time by
-   //    default, or specified on the command line with -rp). Once the parallel
-   //    mesh is defined, the serial mesh can be deleted.
-   ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh);
-   delete mesh;
-   for (int lev = 0; lev < par_ref_levels; lev++)
-   {
-      pmesh->UniformRefinement();
-   }
-
-   // 6. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-   }
-   else if (pmesh->GetNodes())
-   {
-      fec = pmesh->GetNodes()->OwnFEC();
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-   }
-   ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec);
-   HYPRE_Int size = fespace->GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of unknowns: " << size << endl;
-   }
-
-   // 7. Set up the parallel bilinear forms a(.,.) and m(.,.) on the finite
-   //    element space. The first corresponds to the Laplacian operator -Delta,
-   //    while the second is a simple mass matrix needed on the right hand side
-   //    of the generalized eigenvalue problem below. The boundary conditions
-   //    are implemented by elimination with special values on the diagonal to
-   //    shift the Dirichlet eigenvalues out of the computational range. After
-   //    serial and parallel assembly we extract the corresponding parallel
-   //    matrices A and M.
-   ConstantCoefficient one(1.0);
-   Array<int> ess_bdr;
-   if (pmesh->bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(pmesh->bdr_attributes.Max());
-      ess_bdr = 1;
-   }
-
-   ParBilinearForm *a = new ParBilinearForm(fespace);
-   a->AddDomainIntegrator(new DiffusionIntegrator(one));
-   if (pmesh->bdr_attributes.Size() == 0)
-   {
-      // Add a mass term if the mesh has no boundary, e.g. periodic mesh or
-      // closed surface.
-      a->AddDomainIntegrator(new MassIntegrator(one));
-   }
-   a->Assemble();
-   a->EliminateEssentialBCDiag(ess_bdr, 1.0);
-   a->Finalize();
-
-   ParBilinearForm *m = new ParBilinearForm(fespace);
-   m->AddDomainIntegrator(new MassIntegrator(one));
-   m->Assemble();
-   // shift the eigenvalue corresponding to eliminated dofs to a large value
-   m->EliminateEssentialBCDiag(ess_bdr, numeric_limits<double>::min());
-   m->Finalize();
-
-   PetscParMatrix *pA = NULL, *pM = NULL;
-   HypreParMatrix *A = NULL, *M = NULL;
-   Operator::Type tid =
-      !use_slepc ? Operator::Hypre_ParCSR : Operator::PETSC_MATAIJ;
-   OperatorHandle Ah(tid), Mh(tid);
-
-   a->ParallelAssemble(Ah);
-   if (!use_slepc) { Ah.Get(A); }
-   else { Ah.Get(pA); }
-   Ah.SetOperatorOwner(false);
-
-   m->ParallelAssemble(Mh);
-   if (!use_slepc) {Mh.Get(M); }
-   else {Mh.Get(pM); }
-   Mh.SetOperatorOwner(false);
-
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   Operator * Arow = NULL;
-#ifdef MFEM_USE_SUPERLU
-   if (slu_solver)
-   {
-      Arow = new SuperLURowLocMatrix(*A);
-   }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-   if (sp_solver)
-   {
-      Arow = new STRUMPACKRowLocMatrix(*A);
-   }
-#endif
-#endif
-
-   delete a;
-   delete m;
-
-   // 8. Define and configure the LOBPCG eigensolver and the BoomerAMG
-   //    preconditioner for A to be used within the solver. Set the matrices
-   //    which define the generalized eigenproblem A x = lambda M x.
-   Solver * precond = NULL;
-   if (!use_slepc)
-   {
-      if (!slu_solver && !sp_solver)
-      {
-         HypreBoomerAMG * amg = new HypreBoomerAMG(*A);
-         amg->SetPrintLevel(0);
-         precond = amg;
-      }
-      else
-      {
-#ifdef MFEM_USE_SUPERLU
-         if (slu_solver)
-         {
-            SuperLUSolver * superlu = new SuperLUSolver(MPI_COMM_WORLD);
-            superlu->SetPrintStatistics(false);
-            superlu->SetSymmetricPattern(true);
-            superlu->SetColumnPermutation(superlu::PARMETIS);
-            superlu->SetOperator(*Arow);
-            precond = superlu;
-         }
-#endif
-#ifdef MFEM_USE_STRUMPACK
-         if (sp_solver)
-         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
-            strumpack->SetPrintFactorStatistics(true);
-            strumpack->SetPrintSolveStatistics(false);
-            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
-            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
-            strumpack->SetOperator(*Arow);
-            strumpack->SetFromCommandLine();
-            precond = strumpack;
-         }
-#endif
-      }
-   }
-
-   HypreLOBPCG * lobpcg = NULL;
-   SlepcEigenSolver * slepc = NULL;
-   if (!use_slepc)
-   {
-
-      lobpcg = new HypreLOBPCG(MPI_COMM_WORLD);
-      lobpcg->SetNumModes(nev);
-      lobpcg->SetRandomSeed(seed);
-      lobpcg->SetPreconditioner(*precond);
-      lobpcg->SetMaxIter(200);
-      lobpcg->SetTol(1e-8);
-      lobpcg->SetPrecondUsageMode(1);
-      lobpcg->SetPrintLevel(1);
-      lobpcg->SetMassMatrix(*M);
-      lobpcg->SetOperator(*A);
-   }
-   else
-   {
-      slepc = new SlepcEigenSolver(MPI_COMM_WORLD);
-      slepc->SetNumModes(nev);
-      slepc->SetWhichEigenpairs(SlepcEigenSolver::TARGET_REAL);
-      slepc->SetTarget(0.0);
-      slepc->SetSpectralTransformation(SlepcEigenSolver::SHIFT_INVERT);
-      slepc->SetOperators(*pA,*pM);
-   }
-
-   // 9. Compute the eigenmodes and extract the array of eigenvalues. Define a
-   //    parallel grid function to represent each of the eigenmodes returned by
-   //    the solver.
-   Array<double> eigenvalues;
-   if (!use_slepc)
-   {
-      lobpcg->Solve();
-      lobpcg->GetEigenvalues(eigenvalues);
-   }
-   else
-   {
-      slepc->Solve();
-      eigenvalues.SetSize(nev);
-      for (int i=0; i<nev; i++)
-      {
-         slepc->GetEigenvalue(i,eigenvalues[i]);
-      }
-   }
-   Vector temp(fespace->GetTrueVSize());
-   ParGridFunction x(fespace);
-
-   // 10. Save the refined mesh and the modes in parallel. This output can be
-   //     viewed later using GLVis: "glvis -np <np> -m mesh -g mode".
-   {
-      ostringstream mesh_name, mode_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh->Print(mesh_ofs);
-
-      for (int i=0; i<nev; i++)
-      {
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-
-         }
-
-         mode_name << "mode_" << setfill('0') << setw(2) << i << "."
-                   << setfill('0') << setw(6) << myid;
-
-         ofstream mode_ofs(mode_name.str().c_str());
-         mode_ofs.precision(8);
-         x.Save(mode_ofs);
-         mode_name.str("");
-      }
-   }
-
-   // 11. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream mode_sock(vishost, visport);
-      mode_sock.precision(8);
-
-      for (int i=0; i<nev; i++)
-      {
-         if ( myid == 0 )
-         {
-            cout << "Eigenmode " << i+1 << '/' << nev
-                 << ", Lambda = " << eigenvalues[i] << endl;
-         }
-
-         // convert eigenvector from HypreParVector to ParGridFunction
-         if (!use_slepc)
-         {
-            x = lobpcg->GetEigenvector(i);
-         }
-         else
-         {
-            slepc->GetEigenvector(i,temp);
-            x.Distribute(temp);
-         }
-
-         mode_sock << "parallel " << num_procs << " " << myid << "\n"
-                   << "solution\n" << *pmesh << x << flush
-                   << "window_title 'Eigenmode " << i+1 << '/' << nev
-                   << ", Lambda = " << eigenvalues[i] << "'" << endl;
-
-         char c;
-         if (myid == 0)
-         {
-            cout << "press (q)uit or (c)ontinue --> " << flush;
-            cin >> c;
-         }
-         MPI_Bcast(&c, 1, MPI_CHAR, 0, MPI_COMM_WORLD);
-
-         if (c != 'c')
-         {
-            break;
-         }
-      }
-      mode_sock.close();
-   }
-
-   // 12. Free the used memory.
-   delete lobpcg;
-   delete slepc;
-   delete precond;
-   delete M;
-   delete A;
-   delete pA;
-   delete pM;
-#if defined(MFEM_USE_SUPERLU) || defined(MFEM_USE_STRUMPACK)
-   delete Arow;
-#endif
-   delete fespace;
-   if (order > 0)
-   {
-      delete fec;
-   }
-   delete pmesh;
-
-   // We finalize SLEPc
-   MFEMFinalizeSlepc();
-   MPI_Finalize();
-
-   return 0;
-}
@@ -23,9 +23,6 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES =
 PAR_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex9p ex10p
-ifeq ($(MFEM_USE_SLEPC),YES)
-   PAR_EXAMPLES += ex11p
-endif
 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -90,9 +87,6 @@ EX10_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p
 EX10_MF_ARGS     := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
 EX10_MFOP_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3
 EX10_JFNK_ARGS   := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
-EX11_ARGS_SINV   := -m ../../data/star.mesh --useslepc
-EX11_ARGS_LOBPCG := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
-EX11_ARGS_GD     := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd

 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
@@ -120,12 +114,6 @@ ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MF_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_MFOP_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_JFNK_ARGS))
-ifeq ($(MFEM_USE_SLEPC),YES)
-ex11p-test-par: ex11p
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
-endif

 # Testing: "test" target and mfem-test* variables are defined in config/test.mk

@@ -1,6 +0,0 @@
-# Options for the eigenvalue solver
-eps_view
-eps_converged_reason
-eps_type gd
-# Options for the spectral transform
-st_type precond
@@ -1,11 +0,0 @@
-# Options for the eigenvalue solver
-eps_monitor
-eps_converged_reason
-eps_view_values
-eps_type lobpcg
-eps_gen_hermitian
-eps_smallest_real
-eps_lobpcg_blocksize 5
-# Options for the spectral transform
-st_type precond
-st_pc_type gamg
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the Parallel Unstructured Mesh Infrastructure,
@@ -32,13 +32,6 @@
 //               is used for the Finite Element order and "-go" is used for the
 //               geometry order. Note that they can be used independently, i.e.
 //               "-o 8 -go 3" solves for 8th order FE on a third order geometry.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files

 #include "mfem.hpp"
 #include <fstream>
@@ -36,14 +36,6 @@
 //               option "-o" is used for the Finite Element order and "-go" for
 //               the geometry order. Note that they can be used independently:
 //               "-o 8 -go 3" solves for 8th order FE on third order geometry.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files
-

 #include "mfem.hpp"
 #include <fstream>
@@ -43,14 +43,6 @@
 //               also illustrated.
 //
 //               We recommend viewing Example 1 before viewing this example.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files
-

 #include "mfem.hpp"
 #include <fstream>
@@ -1,7 +1,7 @@
 //                       MFEM Example 6 - Parallel Version
 //                              PUMI Modification
 //
-// Compile with: make ex6p
+// Compile with: make ex1p
 //
 // Sample runs:  mpirun -np 8 ex6p
 //
@@ -18,13 +18,6 @@
 //               is added to modify the "adapt_ratio" which is the fraction of
 //               allowable error that scales the output size field of the error
 //               estimator.
-//
-// NOTE:         Model/Mesh files for this example are in the (large) data file
-//               repository of MFEM here https://github.com/mfem/data under the
-//               folder named "pumi", which consists of the following sub-folders:
-//               a) geom -->  model files
-//               b) parallel --> parallel pumi mesh files
-//               c) serial --> serial pumi mesh files

 #include "mfem.hpp"
 #include <fstream>
@@ -339,6 +332,7 @@ int main(int argc, char *argv[])

      apf::destroyField(Tmag_field);
      apf::destroyField(ipfield);
+      apf::destroyNumbering(pumi_mesh->findNumbering("LocalVertexNumbering"));

      // 18. Perform MesAdapt.
      ma::Input* erinput = ma::configure(pumi_mesh, sizefield);
@@ -5,7 +5,7 @@
                      | | | | | ||  _||  __/| | | | | |
                      |_| |_| |_||_|   \___||_| |_| |_|

-                               https://mfem.org
+                               http://mfem.org

 This directory contains modifications of the example codes that illustrate the
 use of MFEM features based on the SUNDIALS suite of time integration and
@@ -721,14 +721,12 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f,

   if (nls_type == KINSOL)
   {
-      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_LINESEARCH, true);
-      kinsolver->SetJFNK(true);
-      kinsolver->SetLSMaxIter(100);
+      KINSolver *kinsolver = new KINSolver(f.GetComm(), KIN_NONE, true);
      newton_solver = kinsolver;
      newton_solver->SetOperator(*reduced_oper);
      newton_solver->SetMaxIter(200);
      newton_solver->SetRelTol(rel_tol);
-      newton_solver->SetPrintLevel(1);
+      newton_solver->SetPrintLevel(0);
      kinsolver->SetMaxSetupCalls(4);
   }
   else
@@ -13,27 +13,17 @@
 //    ex9 -m ../../data/disc-nurbs.mesh       -p 1 -r 3 -s 7 -dt 0.005 -tf 9
 //    ex9 -m ../../data/periodic-cube.mesh    -p 0 -r 2 -s 8 -dt 0.02 -tf 8 -o 2
 //
-// Device sample runs:
-//    ex9 -pa
-//    ex9 -ea
-//    ex9 -fa
-//    ex9 -pa -m ../../data/periodic-cube.mesh
-//    ex9 -pa -m ../../data/periodic-cube.mesh -d cuda
-//    ex9 -ea -m ../../data/periodic-cube.mesh -d cuda
-//    ex9 -fa -m ../../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. The saving of time-dependent data files for external
-//               visualization with VisIt (visit.llnl.gov) and ParaView
-//               (paraview.org) is also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -63,54 +53,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   SparseMatrix &M, &K, A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(SparseMatrix &M_, SparseMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         A = K;
-         A *= -dt;
-         A += M;
-
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -120,21 +62,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   BilinearForm &M, &K;
+   SparseMatrix &M, &K;
   const Vector &b;
-   Solver *M_prec;
+   DSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b);
+   FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };


@@ -145,16 +85,11 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
   int ref_levels = 2;
   int order = 3;
-   bool pa = false;
-   bool ea = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   int ode_solver_type = 7;
+   int ode_solver_type = 4;
   double t_final = 10.0;
   double dt = 0.01;
-   bool visualization = false;
+   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
   bool binary = false;
   int vis_steps = 5;

@@ -173,14 +108,6 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver:\n\t"
                  "1 - Forward Euler,\n\t"
@@ -201,9 +128,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -223,9 +147,6 @@ int main(int argc, char *argv[])
   }
   args.PrintOptions(cout);

-   Device device(device_config);
-   device.Print();
-
   // 2. Read the mesh from the given mesh file. We can handle geometrically
   //    periodic meshes in this code.
   Mesh mesh(mesh_file, 1, 1);
@@ -247,7 +168,7 @@ int main(int argc, char *argv[])

   // 4. Define the discontinuous DG finite element space of the given
   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
   FiniteElementSpace fes(&mesh, &fec);

   cout << "Number of unknowns: " << fes.GetVSize() << endl;
@@ -260,23 +181,8 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   if (pa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k.SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-   else if (ea)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k.SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m.SetAssemblyLevel(AssemblyLevel::FULL);
-      k.SetAssemblyLevel(AssemblyLevel::FULL);
-   }
   m.AddDomainIntegrator(new MassIntegrator);
+   BilinearForm k(&fes);
   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k.AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -288,11 +194,11 @@ int main(int argc, char *argv[])
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

   m.Assemble();
+   m.Finalize();
   int skip_zeros = 0;
   k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
   k.Finalize(skip_zeros);
+   b.Assemble();

   // 6. Define the initial conditions, save the corresponding grid function to
   //    a file and (optionally) save data in the VisIt format and initialize
@@ -333,20 +239,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9", &mesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", &u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
   socketstream sout;
   if (visualization)
   {
@@ -373,7 +265,7 @@ int main(int argc, char *argv[])

   // 7. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(m, k, b);
+   FE_Evolution adv(m.SpMat(), k.SpMat(), b);

   double t = 0.0;
   adv.SetTime(t);
@@ -397,18 +289,12 @@ int main(int argc, char *argv[])
         cvode->UseSundialsLinearSolver();
         ode_solver = cvode; break;
      case 8:
-         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
-         arkode->Init(adv);
-         arkode->SetSStolerances(reltol, abstol);
-         arkode->SetMaxStep(dt);
-         arkode->SetOrder(4);
-         ode_solver = arkode; break;
      case 9:
         arkode = new ARKStepSolver(ARKStepSolver::EXPLICIT);
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(FEHLBERG_13_7_8);
+         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
         ode_solver = arkode; break;
   }

@@ -443,13 +329,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
      }
   }

@@ -463,7 +342,6 @@ int main(int argc, char *argv[])

   // 10. Free the used memory.
   delete ode_solver;
-   delete pd;
   delete dc;

   return 0;
@@ -471,23 +349,12 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &_M, BilinearForm &_K, const Vector &_b)
-   : TimeDependentOperator(_M.Height()), M(_M), K(_K), b(_b), z(_M.Height())
+FE_Evolution::FE_Evolution(SparseMatrix &_M, SparseMatrix &_K, const Vector &_b)
+   : TimeDependentOperator(_M.Size()), M(_M), K(_K), b(_b), z(_M.Size())
 {
-   Array<int> ess_tdof_list;
-   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACYFULL)
-   {
-      M_prec = new DSmoother(M.SpMat());
-      M_solver.SetOperator(M.SpMat());
-      dg_solver = new DG_Solver(M.SpMat(), K.SpMat(), *M.FESpace());
-   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-      M_solver.SetOperator(M);
-      dg_solver = NULL;
-   }
-   M_solver.SetPreconditioner(*M_prec);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);
+
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -503,21 +370,6 @@ void FE_Evolution::Mult(const Vector &x, Vector &y) const
   M_solver.Mult(z, y);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   MFEM_VERIFY(dg_solver != NULL,
-               "Implicit time integration is not supported with partial assembly");
-   K.Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -13,28 +13,17 @@
 //    mpirun -np 4 ex9p -m ../../data/disc-nurbs.mesh       -p 1 -rp 2 -s 7 -dt 0.0025 -tf 9
 //    mpirun -np 4 ex9p -m ../../data/periodic-cube.mesh    -p 0 -rp 1 -s 8 -dt 0.01 -tf 8 -o 2
 //
-// Device sample runs:
-//    mpirun -np 4 ex9p -pa
-//    mpirun -np 4 ex9p -ea
-//    mpirun -np 4 ex9p -fa
-//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh
-//    mpirun -np 4 ex9p -pa -m ../../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -ea -m ../../data/periodic-cube.mesh -d cuda
-//    mpirun -np 4 ex9p -fa -m ../../data/periodic-cube.mesh -d cuda
-//
 // Description:  This example code solves the time-dependent advection equation
 //               du/dt + v.grad(u) = 0, where v is a given fluid velocity, and
 //               u0(x)=u(0,x) is a given initial condition.
 //
 //               The example demonstrates the use of Discontinuous Galerkin (DG)
-//               bilinear forms in MFEM (face integrators), the use of implicit
-//               and explicit ODE time integrators, the definition of periodic
-//               boundary conditions through periodic meshes, as well as the use
-//               of GLVis for persistent visualization of a time-evolving
-//               solution. Saving of time-dependent data files for visualization
-//               with VisIt (visit.llnl.gov) and ParaView (paraview.org), as
-//               well as the optional saving with ADIOS2 (adios2.readthedocs.io)
-//               are also illustrated.
+//               bilinear forms in MFEM (face integrators), the use of explicit
+//               ODE time integrators, the definition of periodic boundary
+//               conditions through periodic meshes, as well as the use of GLVis
+//               for persistent visualization of a time-evolving solution. The
+//               saving of time-dependent data files for external visualization
+//               with VisIt (visit.llnl.gov) is also illustrated.

 #include "mfem.hpp"
 #include <fstream>
@@ -63,66 +52,6 @@ double inflow_function(const Vector &x);
 // Mesh bounding box
 Vector bb_min, bb_max;

-class DG_Solver : public Solver
-{
-private:
-   HypreParMatrix &M, &K;
-   SparseMatrix M_diag;
-   HypreParMatrix *A;
-   GMRESSolver linear_solver;
-   BlockILU prec;
-   double dt;
-public:
-   DG_Solver(HypreParMatrix &M_, HypreParMatrix &K_, const FiniteElementSpace &fes)
-      : M(M_),
-        K(K_),
-        A(NULL),
-        linear_solver(M.GetComm()),
-        prec(fes.GetFE(0)->GetDof(),
-             BlockILU::Reordering::MINIMUM_DISCARDED_FILL),
-        dt(-1.0)
-   {
-      linear_solver.iterative_mode = false;
-      linear_solver.SetRelTol(1e-9);
-      linear_solver.SetAbsTol(0.0);
-      linear_solver.SetMaxIter(100);
-      linear_solver.SetPrintLevel(0);
-      linear_solver.SetPreconditioner(prec);
-
-      M.GetDiag(M_diag);
-   }
-
-   void SetTimeStep(double dt_)
-   {
-      if (dt_ != dt)
-      {
-         dt = dt_;
-         // Form operator A = M - dt*K
-         delete A;
-         A = Add(-dt, K, 0.0, K);
-         SparseMatrix A_diag;
-         A->GetDiag(A_diag);
-         A_diag.Add(1.0, M_diag);
-         // this will also call SetOperator on the preconditioner
-         linear_solver.SetOperator(*A);
-      }
-   }
-
-   void SetOperator(const Operator &op)
-   {
-      linear_solver.SetOperator(op);
-   }
-
-   virtual void Mult(const Vector &x, Vector &y) const
-   {
-      linear_solver.Mult(x, y);
-   }
-
-   ~DG_Solver()
-   {
-      delete A;
-   }
-};

 /** A time-dependent operator for the right-hand side of the ODE. The DG weak
    form of du/dt = -v.grad(u) is M du/dt = K u + b, where M and K are the mass
@@ -132,21 +61,19 @@ public:
 class FE_Evolution : public TimeDependentOperator
 {
 private:
-   OperatorHandle M, K;
+   HypreParMatrix &M, &K;
   const Vector &b;
-   Solver *M_prec;
+   HypreSmoother M_prec;
   CGSolver M_solver;
-   DG_Solver *dg_solver;

   mutable Vector z;

 public:
-   FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K, const Vector &_b);
+   FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K, const Vector &_b);

   virtual void Mult(const Vector &x, Vector &y) const;
-   virtual void ImplicitSolve(const double dt, const Vector &x, Vector &k);

-   virtual ~FE_Evolution();
+   virtual ~FE_Evolution() { }
 };


@@ -164,17 +91,11 @@ int main(int argc, char *argv[])
   int ser_ref_levels = 2;
   int par_ref_levels = 0;
   int order = 3;
-   bool pa = false;
-   bool ea = false;
-   bool fa = false;
-   const char *device_config = "cpu";
-   int ode_solver_type = 7;
+   int ode_solver_type = 4;
   double t_final = 10.0;
   double dt = 0.01;
-   bool visualization = false;
+   bool visualization = true;
   bool visit = false;
-   bool paraview = false;
-   bool adios2 = false;
   bool binary = false;
   int vis_steps = 5;

@@ -195,14 +116,6 @@ int main(int argc, char *argv[])
                  "Number of times to refine the mesh uniformly in parallel.");
   args.AddOption(&order, "-o", "--order",
                  "Order (degree) of the finite elements.");
-   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
-                  "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&ea, "-ea", "--element-assembly", "-no-ea",
-                  "--no-element-assembly", "Enable Element Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
   args.AddOption(&ode_solver_type, "-s", "--ode-solver",
                  "ODE solver:\n\t"
                  "1 - Forward Euler,\n\t"
@@ -223,12 +136,6 @@ int main(int argc, char *argv[])
   args.AddOption(&visit, "-visit", "--visit-datafiles", "-no-visit",
                  "--no-visit-datafiles",
                  "Save data files for VisIt (visit.llnl.gov) visualization.");
-   args.AddOption(&paraview, "-paraview", "--paraview-datafiles", "-no-paraview",
-                  "--no-paraview-datafiles",
-                  "Save data files for ParaView (paraview.org) visualization.");
-   args.AddOption(&adios2, "-adios2", "--adios2-streams", "-no-adios2",
-                  "--no-adios2-streams",
-                  "Save data using adios2 streams.");
   args.AddOption(&binary, "-binary", "--binary-datafiles", "-ascii",
                  "--ascii-datafiles",
                  "Use binary (Sidre) or ascii format for VisIt data files.");
@@ -248,7 +155,6 @@ int main(int argc, char *argv[])
   {
      args.PrintOptions(cout);
   }
-
   // check for vaild ODE solver option
   if (ode_solver_type < 1 || ode_solver_type > 9)
   {
@@ -260,9 +166,6 @@ int main(int argc, char *argv[])
      return 3;
   }

-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
   // 3. Read the serial mesh from the given mesh file on all processors. We can
   //    handle geometrically periodic meshes in this code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
@@ -294,7 +197,7 @@ int main(int argc, char *argv[])

   // 6. Define the parallel discontinuous DG finite element space on the
   //    parallel refined mesh of the given polynomial order.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
+   DG_FECollection fec(order, dim);
   ParFiniteElementSpace *fes = new ParFiniteElementSpace(pmesh, &fec);

   HYPRE_Int global_vSize = fes->GlobalTrueVSize();
@@ -311,24 +214,8 @@ int main(int argc, char *argv[])
   FunctionCoefficient u0(u0_function);

   ParBilinearForm *m = new ParBilinearForm(fes);
-   ParBilinearForm *k = new ParBilinearForm(fes);
-   if (pa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-      k->SetAssemblyLevel(AssemblyLevel::PARTIAL);
-   }
-   else if (ea)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-      k->SetAssemblyLevel(AssemblyLevel::ELEMENT);
-   }
-   else if (fa)
-   {
-      m->SetAssemblyLevel(AssemblyLevel::FULL);
-      k->SetAssemblyLevel(AssemblyLevel::FULL);
-   }
-
   m->AddDomainIntegrator(new MassIntegrator);
+   ParBilinearForm *k = new ParBilinearForm(fes);
   k->AddDomainIntegrator(new ConvectionIntegrator(velocity, -1.0));
   k->AddInteriorFaceIntegrator(
      new TransposeIntegrator(new DGTraceIntegrator(velocity, 1.0, -0.5)));
@@ -339,13 +226,15 @@ int main(int argc, char *argv[])
   b->AddBdrFaceIntegrator(
      new BoundaryFlowIntegrator(inflow, velocity, -1.0, -0.5));

-   int skip_zeros = 0;
   m->Assemble();
-   k->Assemble(skip_zeros);
-   b->Assemble();
   m->Finalize();
+   int skip_zeros = 0;
+   k->Assemble(skip_zeros);
   k->Finalize(skip_zeros);
+   b->Assemble();

+   HypreParMatrix *M = m->ParallelAssemble();
+   HypreParMatrix *K = k->ParallelAssemble();
   HypreParVector *B = b->ParallelAssemble();

   // 8. Define the initial conditions, save the corresponding grid function to
@@ -384,8 +273,6 @@ int main(int argc, char *argv[])
      {
         dc = new VisItDataCollection("Example9-Parallel", pmesh);
         dc->SetPrecision(precision);
-         // To save the mesh using MFEM's parallel mesh format:
-         // dc->SetFormat(DataCollection::PARALLEL_FORMAT);
      }
      dc->RegisterField("solution", u);
      dc->SetCycle(0);
@@ -393,41 +280,6 @@ int main(int argc, char *argv[])
      dc->Save();
   }

-   ParaViewDataCollection *pd = NULL;
-   if (paraview)
-   {
-      pd = new ParaViewDataCollection("Example9P", pmesh);
-      pd->SetPrefixPath("ParaView");
-      pd->RegisterField("solution", u);
-      pd->SetLevelsOfDetail(order);
-      pd->SetDataFormat(VTKFormat::BINARY);
-      pd->SetHighOrderOutput(true);
-      pd->SetCycle(0);
-      pd->SetTime(0.0);
-      pd->Save();
-   }
-
-   // Optionally output a BP (binary pack) file using ADIOS2. This can be
-   // visualized with the ParaView VTX reader.
-#ifdef MFEM_USE_ADIOS2
-   ADIOS2DataCollection *adios2_dc = NULL;
-   if (adios2)
-   {
-      std::string postfix(mesh_file);
-      postfix.erase(0, std::string("../data/").size() );
-      postfix += "_o" + std::to_string(order);
-      const std::string collection_name = "ex9-p-" + postfix + ".bp";
-
-      adios2_dc = new ADIOS2DataCollection(MPI_COMM_WORLD, collection_name, pmesh);
-      // output data substreams are half the number of mpi processes
-      adios2_dc->SetParameter("SubStreams", std::to_string(num_procs/2) );
-      adios2_dc->RegisterField("solution", u);
-      adios2_dc->SetCycle(0);
-      adios2_dc->SetTime(0.0);
-      adios2_dc->Save();
-   }
-#endif
-
   socketstream sout;
   if (visualization)
   {
@@ -460,7 +312,7 @@ int main(int argc, char *argv[])

   // 9. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and define the ODE solver used for time integration.
-   FE_Evolution adv(*m, *k, *B);
+   FE_Evolution adv(*M, *K, *B);

   double t = 0.0;
   adv.SetTime(t);
@@ -532,23 +384,6 @@ int main(int argc, char *argv[])
            dc->SetTime(t);
            dc->Save();
         }
-
-         if (paraview)
-         {
-            pd->SetCycle(ti);
-            pd->SetTime(t);
-            pd->Save();
-         }
-
-#ifdef MFEM_USE_ADIOS2
-         // transient solutions can be visualized with ParaView
-         if (adios2)
-         {
-            adios2_dc->SetCycle(ti);
-            adios2_dc->SetTime(t);
-            adios2_dc->Save();
-         }
-#endif
      }
   }

@@ -568,18 +403,13 @@ int main(int argc, char *argv[])
   delete u;
   delete B;
   delete b;
+   delete K;
   delete k;
+   delete M;
   delete m;
   delete fes;
   delete pmesh;
   delete ode_solver;
-   delete pd;
-#ifdef MFEM_USE_ADIOS2
-   if (adios2)
-   {
-      delete adios2_dc;
-   }
-#endif
   delete dc;

   MPI_Finalize();
@@ -588,43 +418,15 @@ int main(int argc, char *argv[])


 // Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
+FE_Evolution::FE_Evolution(HypreParMatrix &_M, HypreParMatrix &_K,
                           const Vector &_b)
   : TimeDependentOperator(_M.Height()),
-     b(_b),
-     M_solver(_M.ParFESpace()->GetComm()),
-     z(_M.Height())
+     M(_M), K(_K), b(_b), M_solver(M.GetComm()), z(_M.Height())
 {
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      M.Reset(_M.ParallelAssemble(), true);
-      K.Reset(_K.ParallelAssemble(), true);
-   }
-   else
-   {
-      M.Reset(&_M, false);
-      K.Reset(&_K, false);
-   }
+   M_prec.SetType(HypreSmoother::Jacobi);
+   M_solver.SetPreconditioner(M_prec);
+   M_solver.SetOperator(M);

-   M_solver.SetOperator(*M);
-
-   Array<int> ess_tdof_list;
-   if (_M.GetAssemblyLevel()==AssemblyLevel::LEGACYFULL)
-   {
-      HypreParMatrix &M_mat = *M.As<HypreParMatrix>();
-      HypreParMatrix &K_mat = *K.As<HypreParMatrix>();
-      HypreSmoother *hypre_prec = new HypreSmoother(M_mat, HypreSmoother::Jacobi);
-      M_prec = hypre_prec;
-
-      dg_solver = new DG_Solver(M_mat, K_mat, *_M.FESpace());
-   }
-   else
-   {
-      M_prec = new OperatorJacobiSmoother(_M, ess_tdof_list);
-      dg_solver = NULL;
-   }
-
-   M_solver.SetPreconditioner(*M_prec);
   M_solver.iterative_mode = false;
   M_solver.SetRelTol(1e-9);
   M_solver.SetAbsTol(0.0);
@@ -632,28 +434,14 @@ FE_Evolution::FE_Evolution(ParBilinearForm &_M, ParBilinearForm &_K,
   M_solver.SetPrintLevel(0);
 }

-void FE_Evolution::ImplicitSolve(const double dt, const Vector &x, Vector &k)
-{
-   K->Mult(x, z);
-   z += b;
-   dg_solver->SetTimeStep(dt);
-   dg_solver->Mult(z, k);
-}
-
 void FE_Evolution::Mult(const Vector &x, Vector &y) const
 {
   // y = M^{-1} (K x + b)
-   K->Mult(x, z);
+   K.Mult(x, z);
   z += b;
   M_solver.Mult(z, y);
 }

-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-   delete dg_solver;
-}
-

 // Velocity coefficient
 void velocity_function(const Vector &x, Vector &v)
@@ -1,62 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(SUPERLU_EXAMPLES_SRCS)
-if (MFEM_USE_MPI)
-  list(APPEND SUPERLU_EXAMPLES_SRCS
-    ex1p.cpp
-    )
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_superlu" target, see below.
-add_custom_target(test_superlu
-  ${CMAKE_CTEST_COMMAND} -R superlu USES_TERMINAL)
-
-# Add one executable per cpp file, adding "superlu_" as prefix. Sets
-# "test_superlu" as a target that depends on the given examples.
-set(PFX superlu_)
-add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
-
-# Testing.
-# The SuperLU tests can be run separately using the target "test_superlu"
-# which builds the examples and runs:
-#   ctest -R superlu
-
-# Command line options for the tests.
-# Example 1: Test SuperLU on the simple Poisson problem
-set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
-set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${SUPERLU_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,321 +0,0 @@
-//                       MFEM Example 1 - Parallel Version
-//                             SuperLU Modification
-//
-// Compile with: make ex1p
-//
-// Sample runs:  mpirun -np 4 ex1p -m ../../data/square-disc.mesh
-//               mpirun -np 4 ex1p -m ../../data/star.mesh
-//               mpirun -np 4 ex1p -m ../../data/star-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/escher.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera.mesh
-//               mpirun -np 4 ex1p -m ../../data/fichera-mixed.mesh
-//               mpirun -np 4 ex1p -m ../../data/toroid-wedge.mesh
-//               mpirun -np 4 ex1p -m ../../data/periodic-annulus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/periodic-torus-sector.msh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-p2.vtk -o 2
-//               mpirun -np 4 ex1p -m ../../data/square-disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/star-mixed-p2.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/disc-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/pipe-nurbs.mesh -o -1
-//               mpirun -np 4 ex1p -m ../../data/ball-nurbs.mesh -o 2
-//               mpirun -np 4 ex1p -m ../../data/star-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/square-disc-surf.mesh
-//               mpirun -np 4 ex1p -m ../../data/inline-segment.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh
-//               mpirun -np 4 ex1p -m ../../data/amr-hex.mesh
-//               mpirun -np 4 ex1p -m ../../data/mobius-strip.mesh
-//
-// Description:  This example code demonstrates the use of MFEM to define a
-//               simple finite element discretization of the Laplace problem
-//               -Delta u = 1 with homogeneous Dirichlet boundary conditions.
-//               Specifically, we discretize using a FE space of the specified
-//               order, or if order < 1 using an isoparametric/isogeometric
-//               space (i.e. quadratic for quadratic curvilinear mesh, NURBS for
-//               NURBS mesh, etc.)
-//
-//               The example highlights the use of mesh refinement, finite
-//               element grid functions, as well as linear and bilinear forms
-//               corresponding to the left-hand side and right-hand side of the
-//               discrete linear system. We also cover the explicit elimination
-//               of essential boundary conditions, static condensation, and the
-//               optional connection to the GLVis tool for visualization.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Initialize MPI.
-   int num_procs, myid;
-   MPI_Init(&argc, &argv);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
-
-   // 2. Parse command-line options.
-   const char *mesh_file = "../../data/star.mesh";
-   int order = 1;
-   const char *device_config = "cpu";
-   bool visualization = true;
-   int slu_colperm = 4;
-   int slu_rowperm = 1;
-   int slu_iterref = 2;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&device_config, "-d", "--device",
-                  "Device configuration string, see Device::Configure().");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&slu_colperm, "-cp", "--colperm",
-                  "SuperLU Column Permutation Method:  0-NATURAL, 1-MMD-ATA "
-                  "2-MMD_AT_PLUS_A, 3-COLAMD, 4-METIS_AT_PLUS_A, 5-PARMETIS "
-                  "6-ZOLTAN");
-   args.AddOption(&slu_rowperm, "-rp", "--rowperm",
-                  "SuperLU Row Permutation Method:  0-NOROWPERM, 1-LargeDiag");
-   args.AddOption(&slu_iterref, "-rp", "--rowperm",
-                  "SuperLU Iterative Refinement:  0-NOREFINE, 1-Single, "
-                  "2-Double, 3-Extra");
-
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   // 3. Enable hardware devices such as GPUs, and programming models such as
-   //    CUDA, OCCA, RAJA and OpenMP based on command line options.
-   Device device(device_config);
-   if (myid == 0) { device.Print(); }
-
-   // 4. Read the (serial) mesh from the given mesh file on all processors.  We
-   //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
-   //    and volume meshes with the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the serial mesh on all processors to increase the resolution. In
-   //    this example we do 'ref_levels' of uniform refinement. We choose
-   //    'ref_levels' to be the largest number that gives a final mesh with no
-   //    more than 1,000 elements.
-   {
-      int ref_levels =
-         (int)floor(log(1000./mesh.GetNE())/log(2.)/dim);
-      for (int l = 0; l < ref_levels; l++)
-      {
-         mesh.UniformRefinement();
-      }
-   }
-
-   // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine
-   //    this mesh further in parallel to increase the resolution. Once the
-   //    parallel mesh is defined, the serial mesh can be deleted.
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-   {
-      int par_ref_levels = 2;
-      for (int l = 0; l < par_ref_levels; l++)
-      {
-         pmesh.UniformRefinement();
-      }
-   }
-
-   // 7. Define a parallel finite element space on the parallel mesh. Here we
-   //    use continuous Lagrange finite elements of the specified order. If
-   //    order < 1, we instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec;
-   bool delete_fec;
-   if (order > 0)
-   {
-      fec = new H1_FECollection(order, dim);
-      delete_fec = true;
-   }
-   else if (pmesh.GetNodes())
-   {
-      fec = pmesh.GetNodes()->OwnFEC();
-      delete_fec = false;
-      if (myid == 0)
-      {
-         cout << "Using isoparametric FEs: " << fec->Name() << endl;
-      }
-   }
-   else
-   {
-      fec = new H1_FECollection(order = 1, dim);
-      delete_fec = true;
-   }
-   ParFiniteElementSpace fespace(&pmesh, fec);
-   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (myid == 0)
-   {
-      cout << "Number of finite element unknowns: " << size << endl;
-   }
-
-   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes from the mesh as essential
-   //    (Dirichlet) and converting them to a list of true dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 9. Set up the parallel linear form b(.) which corresponds to the
-   //    right-hand side of the FEM linear system, which in this case is
-   //    (1,phi_i) where phi_i are the basis functions in fespace.
-   ParLinearForm b(&fespace);
-   ConstantCoefficient one(1.0);
-   b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   b.Assemble();
-
-   // 10. Define the solution vector x as a parallel finite element grid function
-   //     corresponding to fespace. Initialize x with initial guess of zero,
-   //     which satisfies the boundary conditions.
-   ParGridFunction x(&fespace);
-   x = 0.0;
-
-   // 11. Set up the parallel bilinear form a(.,.) on the finite element space
-   //     corresponding to the Laplacian operator -Delta, by adding the Diffusion
-   //     domain integrator.
-   ParBilinearForm a(&fespace);
-   a.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   // 12. Assemble the parallel bilinear form and the corresponding linear
-   //     system, applying any necessary transformations such as: parallel
-   //     assembly, eliminating boundary conditions, applying conforming
-   //     constraints for non-conforming AMR, static condensation, etc.
-   a.Assemble();
-
-   OperatorPtr A;
-   Vector B, X;
-   a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-   // 13. Solve the linear system A X = B utilizing SuperLU.
-   SuperLUSolver *superlu = new SuperLUSolver(MPI_COMM_WORLD);
-   Operator *SLU_A = new SuperLURowLocMatrix(*A.As<HypreParMatrix>());
-   superlu->SetPrintStatistics(true);
-   superlu->SetSymmetricPattern(false);
-
-   if (slu_colperm == 0)
-   {
-      superlu->SetColumnPermutation(superlu::NATURAL);
-   }
-   else if (slu_colperm == 1)
-   {
-      superlu->SetColumnPermutation(superlu::MMD_ATA);
-   }
-   else if (slu_colperm == 2)
-   {
-      superlu->SetColumnPermutation(superlu::MMD_AT_PLUS_A);
-   }
-   else if (slu_colperm == 3)
-   {
-      superlu->SetColumnPermutation(superlu::COLAMD);
-   }
-   else if (slu_colperm == 4)
-   {
-      superlu->SetColumnPermutation(superlu::METIS_AT_PLUS_A);
-   }
-   else if (slu_colperm == 5)
-   {
-      superlu->SetColumnPermutation(superlu::PARMETIS);
-   }
-   else if (slu_colperm == 6)
-   {
-      superlu->SetColumnPermutation(superlu::ZOLTAN);
-   }
-
-   if (slu_rowperm == 0)
-   {
-      superlu->SetRowPermutation(superlu::NOROWPERM);
-   }
-   else if (slu_rowperm == 1)
-   {
-#ifdef MFEM_USE_SUPERLU5
-      superlu->SetRowPermutation(superlu::LargeDiag);
-#else
-      superlu->SetRowPermutation(superlu::LargeDiag_MC64);
-#endif
-   }
-
-   if (slu_iterref == 0)
-   {
-      superlu->SetIterativeRefine(superlu::NOREFINE);
-   }
-   else if (slu_iterref == 1)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_SINGLE);
-   }
-   else if (slu_iterref == 2)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_DOUBLE);
-   }
-   else if (slu_iterref == 3)
-   {
-      superlu->SetIterativeRefine(superlu::SLU_EXTRA);
-   }
-
-   superlu->SetOperator(*SLU_A);
-   superlu->SetPrintStatistics(true);
-   superlu->Mult(B, X);
-
-   // 14. Recover the parallel grid function corresponding to X. This is the
-   //     local finite element solution on each processor.
-   a.RecoverFEMSolution(X, b, x);
-
-   // 15. Save the refined mesh and the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g sol".
-   {
-      ostringstream mesh_name, sol_name;
-      mesh_name << "mesh." << setfill('0') << setw(6) << myid;
-      sol_name << "sol." << setfill('0') << setw(6) << myid;
-
-      ofstream mesh_ofs(mesh_name.str().c_str());
-      mesh_ofs.precision(8);
-      pmesh.Print(mesh_ofs);
-
-      ofstream sol_ofs(sol_name.str().c_str());
-      sol_ofs.precision(8);
-      x.Save(sol_ofs);
-   }
-
-   // 16. Send the solution by socket to a GLVis server.
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock << "parallel " << num_procs << " " << myid << "\n";
-      sol_sock.precision(8);
-      sol_sock << "solution\n" << pmesh << x << flush;
-   }
-
-   // 17. Free the used memory.
-   if (delete_fec)
-   {
-      delete fec;
-   }
-   MPI_Finalize();
-
-   return 0;
-}
@@ -1,79 +0,0 @@
-# Copyright (c) 2010-2020, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/superlu/,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = 
-PAR_EXAMPLES = ex1p
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_SUPERLU),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with SuperLU)
-endif
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-PARALLEL_NAME := Parallel SuperLU example
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
-
-# Testing: Specific execution options:
-# Example 1: Test SuperLU on the simple poisson problem
-EX1_COMMON_ARGS := -m ../../data/star.mesh
-EX1P_ARGS := $(EX1_COMMON_ARGS)
-ex1p-test-par: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME),$(EX1P_ARGS))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build clean-exec
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
-	@rm -f mesh.* sol.*
@@ -13,29 +13,17 @@ set(SRCS
  bilinearform.cpp
  bilinearform_ext.cpp
  bilininteg.cpp
-  bilininteg_convection_pa.cpp
-  bilininteg_convection_ea.cpp
-  bilininteg_dgtrace_pa.cpp
-  bilininteg_dgtrace_ea.cpp
-  bilininteg_diffusion_mf.cpp
-  bilininteg_diffusion_pa.cpp
-  bilininteg_diffusion_ea.cpp
+  bilininteg_convection.cpp
+  bilininteg_dgtrace.cpp
+  bilininteg_diffusion.cpp
  bilininteg_divergence.cpp
  bilininteg_hcurl.cpp
-  bilininteg_hdiv.cpp
-  bilininteg_vectorfe.cpp
  bilininteg_gradient.cpp
-  bilininteg_mass_mf.cpp
-  bilininteg_mass_pa.cpp
-  bilininteg_mass_ea.cpp
-  bilininteg_transpose_ea.cpp
+  bilininteg_mass.cpp
  bilininteg_vecdiffusion.cpp
-  bilininteg_vecdiffusion_mf.cpp
  bilininteg_vecmass.cpp
-  bilininteg_vecmass_mf.cpp
  coefficient.cpp
  complex_fem.cpp
-  convergence.cpp
  datacollection.cpp
  eltrans.cpp
  estimators.cpp
@@ -46,9 +34,6 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  libceed/ceed.cpp
-  libceed/diffusion.cpp
-  libceed/mass.cpp
  linearform.cpp
  lininteg.cpp
  multigrid.cpp
@@ -73,7 +58,6 @@ set(HDRS
  bilininteg.hpp
  coefficient.hpp
  complex_fem.hpp
-  convergence.hpp
  datacollection.hpp
  eltrans.hpp
  estimators.hpp
@@ -85,9 +69,6 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  libceed/ceed.hpp
-  libceed/diffusion.hpp
-  libceed/mass.hpp
  linearform.hpp
  lininteg.hpp
  multigrid.hpp
@@ -147,6 +128,17 @@ if (MFEM_USE_MPI)
    prestriction.hpp)
 endif()

+if (MFEM_USE_CEED)
+   list(APPEND SRCS
+     libceed/ceed.cpp
+     libceed/diffusion.cpp
+     libceed/mass.cpp)
+   list(APPEND HDRS
+     libceed/ceed.hpp
+     libceed/diffusion.hpp
+     libceed/mass.hpp)
+endif()
+
 convert_filenames_to_full_paths(SRCS)
 convert_filenames_to_full_paths(HDRS)

@@ -15,8 +15,6 @@

 #include "adios2datacollection.hpp"

-#ifdef MFEM_USE_ADIOS2
-
 namespace mfem
 {

@@ -89,4 +87,4 @@ noexcept

 } //end namespace mfem

-#endif // MFEM_USE_ADIOS2
+
@@ -17,9 +17,6 @@
 #define MFEM_ADIOS2DATACOLLECTION

 #include "../config/config.hpp"
-
-#ifdef MFEM_USE_ADIOS2
-
 #include "../general/adios2stream.hpp"
 #include "datacollection.hpp"

@@ -88,6 +85,4 @@ private:

 }  // namespace mfem

-#endif // MFEM_USE_ADIOS2
-
 #endif /* MFEM_ADIOS2DATACOLLECTION */
@@ -76,7 +76,7 @@ BilinearForm::BilinearForm(FiniteElementSpace * f)
   precompute_sparsity = 0;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;
 }
@@ -94,7 +94,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   precompute_sparsity = ps;
   diag_policy = DIAG_KEEP;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   batch = 1;
   ext = NULL;

@@ -121,19 +121,20 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
-         ext = new FABilinearFormExtension(this);
+         // ext = new FABilinearFormExtension(this);
+         // Use the original BilinearForm implementation for now
         break;
      case AssemblyLevel::ELEMENT:
-         ext = new EABilinearFormExtension(this);
+         mfem_error("Element assembly not supported yet... stay tuned!");
+         // ext = new EABilinearFormExtension(this);
         break;
      case AssemblyLevel::PARTIAL:
         ext = new PABilinearFormExtension(this);
         break;
      case AssemblyLevel::NONE:
-         ext = new MFBilinearFormExtension(this);
+         mfem_error("Matrix-free action not supported yet... stay tuned!");
+         // ext = new MFBilinearFormExtension(this);
         break;
      default:
         mfem_error("Unknown assembly level");
@@ -143,7 +144,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 void BilinearForm::EnableStaticCondensation()
 {
   delete static_cond;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      static_cond = NULL;
      MFEM_WARNING("Static condensation not supported for this assembly level");
@@ -168,7 +169,7 @@ void BilinearForm::EnableHybridization(FiniteElementSpace *constr_space,
                                       const Array<int> &ess_tdof_list)
 {
   delete hybridization;
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      delete constr_integ;
      hybridization = NULL;
@@ -223,7 +224,7 @@ MatrixInverse * BilinearForm::Inverse() const

 void BilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      if (!static_cond) { mat->Finalize(skip_zeros); }
      if (mat_e) { mat_e->Finalize(skip_zeros); }
@@ -626,33 +627,6 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
      MFEM_ASSERT(diag.Size() == fes->GetTrueVSize(),
                  "Vector for holding diagonal has wrong size!");
      const Operator *P = fes->GetProlongationMatrix();
-      // For an AMR mesh, a convergent diagonal is assembled with |P^T| d_e,
-      // where |P^T| has the entry-wise absolute values of the conforming
-      // prolongation transpose operator.
-      if (P && !fes->Conforming())
-      {
-         Vector local_diag(P->Height());
-         ext->AssembleDiagonal(local_diag);
-         const SparseMatrix *SP = dynamic_cast<const SparseMatrix*>(P);
-#ifdef MFEM_USE_MPI
-         const HypreParMatrix *HP = dynamic_cast<const HypreParMatrix*>(P);
-#endif
-         if (SP)
-         {
-            SP->AbsMultTranspose(local_diag, diag);
-         }
-#ifdef MFEM_USE_MPI
-         else if (HP)
-         {
-            HP->AbsMultTranspose(1.0, local_diag, 0.0, diag);
-         }
-#endif
-         else
-         {
-            MFEM_ABORT("Prolongation matrix has unexpected type.");
-         }
-         return;
-      }
      if (!IsIdentityProlongation(P))
      {
         Vector local_diag(P->Height());
@@ -666,7 +640,8 @@ void BilinearForm::AssembleDiagonal(Vector &diag) const
   }
   else
   {
-      mat->GetDiag(diag);
+      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
+                 "matrix and use SparseMatrix::GetDiag?");
   }
 }

@@ -1109,7 +1084,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   mat = NULL;
   mat_e = NULL;
   extern_bfs = 0;
-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1134,7 +1109,7 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,
   bbfi_marker = mbf->bbfi_marker;
   btfbfi_marker = mbf->btfbfi_marker;

-   assembly = AssemblyLevel::LEGACYFULL;
+   assembly = AssemblyLevel::FULL;
   ext = NULL;
 }

@@ -1147,8 +1122,6 @@ void MixedBilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
   assembly = assembly_level;
   switch (assembly)
   {
-      case AssemblyLevel::LEGACYFULL:
-         break;
      case AssemblyLevel::FULL:
         // ext = new FAMixedBilinearFormExtension(this);
         // Use the original BilinearForm implementation for now
@@ -1219,7 +1192,7 @@ void MixedBilinearForm::AddMultTranspose(const Vector & x, Vector & y,

 MatrixInverse * MixedBilinearForm::Inverse() const
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("MixedBilinearForm::Inverse not possible with this assembly level!");
      return NULL;
@@ -1232,7 +1205,7 @@ MatrixInverse * MixedBilinearForm::Inverse() const

 void MixedBilinearForm::Finalize (int skip_zeros)
 {
-   if (assembly == AssemblyLevel::LEGACYFULL)
+   if (assembly == AssemblyLevel::FULL)
   {
      mat -> Finalize (skip_zeros);
   }
@@ -1459,57 +1432,9 @@ void MixedBilinearForm::Assemble (int skip_zeros)
   }
 }

-void MixedBilinearForm::AssembleDiagonal_ADAt(const Vector &D,
-                                              Vector &diag) const
-{
-   if (ext)
-   {
-      MFEM_ASSERT(diag.Size() == test_fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      MFEM_ASSERT(D.Size() == trial_fes->GetTrueVSize(),
-                  "Vector for holding diagonal has wrong size!");
-      const Operator *P_trial = trial_fes->GetProlongationMatrix();
-      const Operator *P_test = test_fes->GetProlongationMatrix();
-      if (!IsIdentityProlongation(P_trial))
-      {
-         Vector local_D(P_trial->Height());
-         P_trial->Mult(D, local_D);
-
-         if (!IsIdentityProlongation(P_test))
-         {
-            Vector local_diag(P_test->Height());
-            ext->AssembleDiagonal_ADAt(local_D, local_diag);
-            P_test->MultTranspose(local_diag, diag);
-         }
-         else
-         {
-            ext->AssembleDiagonal_ADAt(local_D, diag);
-         }
-      }
-      else
-      {
-         if (!IsIdentityProlongation(P_test))
-         {
-            Vector local_diag(P_test->Height());
-            ext->AssembleDiagonal_ADAt(D, local_diag);
-            P_test->MultTranspose(local_diag, diag);
-         }
-         else
-         {
-            ext->AssembleDiagonal_ADAt(D, diag);
-         }
-      }
-   }
-   else
-   {
-      MFEM_ABORT("Not implemented. Maybe assemble your bilinear form into a "
-                 "matrix and use SparseMatrix functions?");
-   }
-}
-
 void MixedBilinearForm::ConformingAssemble()
 {
-   if (assembly != AssemblyLevel::LEGACYFULL)
+   if (assembly != AssemblyLevel::FULL)
   {
      MFEM_WARNING("Conforming assemble not supported for this assembly level!");
      return;
@@ -25,15 +25,12 @@
 namespace mfem
 {

-/** @brief Enumeration defining the assembly level for bilinear and nonlinear
-    form classes derived from Operator. */
+/// Enumeration defining the assembly level for bilinear and nonlinear form
+/// classes derived from Operator.
 enum class AssemblyLevel
 {
-   /// Legacy fully assembled form, i.e. a global sparse matrix in MFEM, Hypre
-   /// or PETSC format. This assembly is ALWAYS performed on the host.
-   LEGACYFULL = 0,
-   /// Fully assembled form, i.e. a global sparse matrix in MFEM format. This
-   /// assembly is compatible with device execution.
+   /// Fully assembled form, i.e. a global sparse matrix in MFEM, Hypre or PETSC
+   /// format.
   FULL,
   /// Form assembled at element level, which computes and stores dense element
   /// matrices.
@@ -47,19 +44,15 @@ enum class AssemblyLevel
 };


-/** @brief A "square matrix" operator for the associated FE space and
-    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
-    M. This class also supports other assembly levels specified via the
-    SetAssemblyLevel() function. */
+/** Class for bilinear form - "Matrix" with associated FE space and
+    BLFIntegrators. */
 class BilinearForm : public Matrix
 {
 protected:
-   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
+   /// Sparse matrix to be associated with the form. Owned.
   SparseMatrix *mat;

-   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
-        from the b.c.  Owned.
-       \f$ M + M_e = M_{original} \f$ */
+   /// Matrix used to eliminate b.c. Owned.
   SparseMatrix *mat_e;

   /// FE space on which the form lives. Not owned.
@@ -69,12 +62,12 @@ protected:
   AssemblyLevel assembly;
   /// Element batch size used in the form action (1, 8, num_elems, etc.)
   int batch;
-   /** @brief Extension for supporting Full Assembly (FA), Element Assembly (EA),
+   /** Extension for supporting Full Assembly (FA), Element Assembly (EA),
       Partial Assembly (PA), or Matrix Free assembly (MF). */
   BilinearFormExtension *ext;

-   /** @brief Indicates the Mesh::sequence corresponding to the current state of
-       the BilinearForm. */
+   /// Indicates the Mesh::sequence corresponding to the current state of the
+   /// BilinearForm.
   long sequence;

   /** @brief Indicates the BilinearFormIntegrator%s stored in #dbfi, #bbfi,
@@ -122,7 +115,7 @@ protected:
      static_cond = NULL; hybridization = NULL;
      precompute_sparsity = 0;
      diag_policy = DIAG_KEEP;
-      assembly = AssemblyLevel::LEGACYFULL;
+      assembly = AssemblyLevel::FULL;
      batch = 1;
      ext = NULL;
   }
@@ -154,44 +147,35 @@ public:
   /// Get the size of the BilinearForm as a square matrix.
   int Size() const { return height; }

-   /// Set the desired assembly level.
-   /** Valid choices are:
-
-       - AssemblyLevel::LEGACYFULL (default)
-       - AssemblyLevel::FULL
-       - AssemblyLevel::PARTIAL
-       - AssemblyLevel::ELEMENT
-       - AssemblyLevel::NONE
-
-       This method must be called before assembly. */
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
+   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

   /// Returns the assembly level
   AssemblyLevel GetAssemblyLevel() const { return assembly; }

-   /** @brief Enable the use of static condensation. For details see the
-       description for class StaticCondensation in fem/staticcond.hpp This method
-       should be called before assembly. If the number of unknowns after static
+   /** Enable the use of static condensation. For details see the description
+       for class StaticCondensation in fem/staticcond.hpp This method should be
+       called before assembly. If the number of unknowns after static
       condensation is not reduced, it is not enabled. */
   void EnableStaticCondensation();

-   /** @brief Check if static condensation was actually enabled by a previous
-       call to EnableStaticCondensation(). */
+   /** Check if static condensation was actually enabled by a previous call to
+       EnableStaticCondensation(). */
   bool StaticCondensationIsEnabled() const { return static_cond; }

   /// Return the trace FE space associated with static condensation.
   FiniteElementSpace *SCFESpace() const
   { return static_cond ? static_cond->GetTraceFESpace() : NULL; }

-   /// Enable hybridization.
-   /** For details see the description for class
+   /** Enable hybridization; for details see the description for class
       Hybridization in fem/hybridization.hpp. This method should be called
       before assembly. */
   void EnableHybridization(FiniteElementSpace *constr_space,
                            BilinearFormIntegrator *constr_integ,
                            const Array<int> &ess_tdof_list);

-   /** @brief For scalar FE spaces, precompute the sparsity pattern of the matrix
+   /** For scalar FE spaces, precompute the sparsity pattern of the matrix
       (assuming dense element matrices) based on the types of integrators
       present in the bilinear form. */
   void UsePrecomputedSparsity(int ps = 1) { precompute_sparsity = ps; }
@@ -210,16 +194,15 @@ public:
   /// Use the sparsity of @a A to allocate the internal SparseMatrix.
   void UseSparsity(SparseMatrix &A);

-   /// Pre-allocate the internal SparseMatrix before assembly.
-   /**  If the flag 'precompute sparsity'
-       is set, the matrix is allocated in CSR format (i.e.
+   /** Pre-allocate the internal SparseMatrix before assembly. If the flag
+       'precompute sparsity' is set, the matrix is allocated in CSR format (i.e.
       finalized) and the entries are initialized with zeros. */
   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }

-   /// Access all the integrators added with AddDomainIntegrator().
+   /// Access all integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &dbfi; }

-   /// Access all the integrators added with AddBoundaryIntegrator().
+   /// Access all integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &bbfi; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
       If no marker was specified when the integrator was added, the
@@ -236,85 +219,64 @@ public:
       corresponding pointer (to Array<int>) will be NULL. */
   Array<Array<int>*> *GetBFBFI_Marker() { return &bfbfi_marker; }

-   /// Returns a reference to: \f$ M_{ij} \f$
   const double &operator()(int i, int j) { return (*mat)(i,j); }

-   /// Returns a reference to: \f$ M_{ij} \f$
+   /// Returns reference to a_{ij}.
   virtual double &Elem(int i, int j);

-   /// Returns constant reference to: \f$ M_{ij} \f$
+   /// Returns constant reference to a_{ij}.
   virtual const double &Elem(int i, int j) const;

-   /// Matrix vector multiplication:  \f$ y = M x \f$
+   /// Matrix vector multiplication.
   virtual void Mult(const Vector &x, Vector &y) const;

-   /** @brief Matrix vector multiplication with the original uneliminated
-       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
-       \f$ y = M x + M_e x \f$ */
   void FullMult(const Vector &x, Vector &y) const
   { mat->Mult(x, y); mat_e->AddMult(x, y); }

-   /// Add the matrix vector multiple to a vector:  \f$ y += a M x \f$
   virtual void AddMult(const Vector &x, Vector &y, const double a = 1.0) const
   { mat -> AddMult (x, y, a); }

-   /** @brief Add the original uneliminated matrix vector multiple to a vector.
-       The original matrix is \f$ M + Me \f$ so we have:
-       \f$ y += M x + M_e x \f$ */
   void FullAddMult(const Vector &x, Vector &y) const
   { mat->AddMult(x, y); mat_e->AddMult(x, y); }

-   /// Add the matrix transpose vector multiplication:  \f$ y += a M^T x \f$
   virtual void AddMultTranspose(const Vector & x, Vector & y,
                                 const double a = 1.0) const
   { mat->AddMultTranspose(x, y, a); }

-   /** @brief Add the original uneliminated matrix transpose vector
-       multiple to a vector. The original matrix is \f$ M + M_e \f$
-       so we have: \f$ y += M^T x + {M_e}^T x \f$ */
   void FullAddMultTranspose(const Vector & x, Vector & y) const
   { mat->AddMultTranspose(x, y); mat_e->AddMultTranspose(x, y); }

-   /// Matrix transpose vector multiplication:  \f$ y = M^T x \f$
   virtual void MultTranspose(const Vector & x, Vector & y) const
   { y = 0.0; AddMultTranspose (x, y); }

-   /// Compute \f$ y^T M x \f$
   double InnerProduct(const Vector &x, const Vector &y) const
   { return mat->InnerProduct (x, y); }

-   /// Returns a pointer to (approximation) of the matrix inverse:  \f$ M^{-1} \f$
+   /// Returns a pointer to (approximation) of the matrix inverse.
   virtual MatrixInverse *Inverse() const;

   /// Finalizes the matrix initialization.
   virtual void Finalize(int skip_zeros = 1);

-   /// Returns a const reference to the sparse matrix.
+   /// Returns a reference to the sparse matrix
   const SparseMatrix &SpMat() const
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }
-
-   /// Returns a reference to the sparse matrix:  \f$ M \f$
   SparseMatrix &SpMat()
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }
-
-   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

-   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   /// Returns a reference to the sparse matrix of eliminated b.c.
   const SparseMatrix &SpMatElim() const
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
      return *mat_e;
   }
-
-   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
   SparseMatrix &SpMatElim()
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
@@ -349,7 +311,6 @@ public:
   void AddBdrFaceIntegrator(BilinearFormIntegrator *bfi,
                             Array<int> &bdr_marker);

-   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
   void operator=(const double a)
   {
      if (mat != NULL) { *mat = a; }
@@ -367,10 +328,10 @@ public:
       for an AMR mesh. */
   void AssembleDiagonal(Vector &diag) const;

-   /// Get the finite element space prolongation operator.
+   /// Get the finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return fes->GetConformingProlongation(); }
-   /// Get the finite element space restriction operator
+   /// Get the finite element space restriction matrix
   virtual const Operator *GetRestriction() const
   { return fes->GetConformingRestriction(); }
   /// Get the output finite element space prolongation matrix
@@ -416,7 +377,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
                         OpType &A, Vector &X, Vector &B,
@@ -438,7 +401,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
   {
@@ -526,12 +491,10 @@ public:
                                 double value);

   /// Eliminate the given @a vdofs. NOTE: here, @a vdofs is a list of DOFs.
-   /** In this case the eliminations are applied to the internal \f$ M \f$
-       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
                       DiagonalPolicy dpolicy = DIAG_ONE);

-   /// Eliminate the given @a vdofs, storing the eliminated part internally in \f$ M_e \f$.
+   /// Eliminate the given @a vdofs, storing the eliminated part internally.
   /** This method works in conjunction with EliminateVDofsInRHS() and allows
       elimination of boundary conditions in multiple right-hand sides. In this
       method, @a vdofs is a list of DOFs. */
@@ -560,11 +523,9 @@ public:
   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
                            Vector &b);

-   /// Compute inner product for full uneliminated matrix \f$ y^T M x + y^T M_e x \f$
   double FullInnerProduct(const Vector &x, const Vector &y) const
   { return mat->InnerProduct(x, y) + mat_e->InnerProduct(x, y); }

-   /// Update the @a FiniteElementSpace and delete all data associated with the old one.
   virtual void Update(FiniteElementSpace *nfes = NULL);

   /// (DEPRECATED) Return the FE space associated with the BilinearForm.
@@ -576,13 +537,7 @@ public:
   /// Read-only access to the associated FiniteElementSpace.
   const FiniteElementSpace *FESpace() const { return fes; }

-   /// Sets diagonal policy used upon construction of the linear system.
-   /** Policies include:
-
-       - DIAG_ZERO (Set the diagonal values to zero)
-       - DIAG_ONE  (Set the diagonal values to one)
-       - DIAG_KEEP (Keep the diagonal values)
-   */
+   /// Sets diagonal policy used upon construction of the linear system
   void SetDiagonalPolicy(DiagonalPolicy policy);

   /// Indicate that integrators are not owned by the BilinearForm
@@ -595,16 +550,16 @@ public:

 /**
   Class for assembling of bilinear forms `a(u,v)` defined on different
-   trial and test spaces. The assembled matrix `M` is such that
+   trial and test spaces. The assembled matrix `A` is such that

-       a(u,v) = V^t M U
+       a(u,v) = V^t A U

   where `U` and `V` are the vectors representing the functions `u` and `v`,
   respectively.  The first argument, `u`, of `a(,)` is in the trial space
   and the second argument, `v`, is in the test space. Thus,

-       # of rows of M = dimension of the test space and
-       # of cols of M = dimension of the trial space.
+       # of rows of A = dimension of the test space and
+       # of cols of A = dimension of the trial space.

   Both trial and test spaces should be defined on the same mesh.
 */
@@ -673,15 +628,11 @@ public:
                     FiniteElementSpace *te_fes,
                     MixedBilinearForm *mbf);

-   /// Returns a reference to: \f$ M_{ij} \f$
   virtual double &Elem(int i, int j);

-   /// Returns a reference to: \f$ M_{ij} \f$
   virtual const double &Elem(int i, int j) const;

-   /// Matrix multiplication: \f$ y = M x \f$
   virtual void Mult(const Vector & x, Vector & y) const;
-
   virtual void AddMult(const Vector & x, Vector & y,
                        const double a = 1.0) const;

@@ -691,7 +642,6 @@ public:

   virtual MatrixInverse *Inverse() const;

-   /// Finalizes the matrix initialization.
   virtual void Finalize(int skip_zeros = 1);

   /** Extract the associated matrix as SparseMatrix blocks. The number of
@@ -699,14 +649,8 @@ public:
       test and trial spaces, respectively. */
   void GetBlocks(Array2D<SparseMatrix *> &blocks) const;

-   /// Returns a const reference to the sparse matrix:  \f$ M \f$
   const SparseMatrix &SpMat() const { return *mat; }
-
-   /// Returns a reference to the sparse matrix:  \f$ M \f$
   SparseMatrix &SpMat() { return *mat; }
-
-   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /// Adds a domain integrator. Assumes ownership of @a bfi.
@@ -753,19 +697,14 @@ public:
       corresponding pointer (to Array<int>) will be NULL. */
   Array<Array<int>*> *GetBTFBFI_Marker() { return &btfbfi_marker; }

-   /// Sets all sparse values of \f$ M \f$ to @a a.
   void operator=(const double a) { *mat = a; }

-   /// Set the desired assembly level. The default is AssemblyLevel::LEGACYFULL.
+   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

   void Assemble(int skip_zeros = 1);

-   /** @brief Assemble the diagonal of ADA^T into diag, where A is this mixed
-       bilinear form and D is a diagonal. */
-   void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const;
-
   /// Get the input finite element space prolongation matrix
   virtual const Operator *GetProlongation() const
   { return trial_fes->GetProlongationMatrix(); }
@@ -858,7 +797,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormRectangularSystemMatrix(const Array<int> &trial_tdof_list,
                                    const Array<int> &test_tdof_list, OpType &A)
@@ -888,7 +829,9 @@ public:
       returned in the variable @a A, of type OpType, holding a *reference* to
       the system matrix (created with the method OpType::MakeRef()). The
       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
+       destructor is called.
+
+       Currently, this method can be used only with AssemblyLevel::FULL. */
   template <typename OpType>
   void FormRectangularLinearSystem(const Array<int> &trial_tdof_list,
                                    const Array<int> &test_tdof_list,
@@ -15,7 +15,6 @@
 #include "../general/forall.hpp"
 #include "bilinearform.hpp"
 #include "libceed/ceed.hpp"
-#include "pgridfunc.hpp"

 namespace mfem
 {
@@ -36,206 +35,6 @@ const Operator *BilinearFormExtension::GetRestriction() const
   return a->GetRestriction();
 }

-// Data and methods for partially-assembled bilinear forms
-MFBilinearFormExtension::MFBilinearFormExtension(BilinearForm *form)
-   : BilinearFormExtension(form),
-     trialFes(a->FESpace()),
-     testFes(a->FESpace())
-{
-   elem_restrict = NULL;
-   int_face_restrict_lex = NULL;
-   bdr_face_restrict_lex = NULL;
-}
-
-void MFBilinearFormExtension::Assemble()
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssembleMF(*a->FESpace());
-   }
-}
-
-void MFBilinearFormExtension::AssembleDiagonal(Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
-   {
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalMF(localY);
-      }
-      const ElementRestriction* H1elem_restrict =
-         dynamic_cast<const ElementRestriction*>(elem_restrict);
-      if (H1elem_restrict)
-      {
-         H1elem_restrict->MultTransposeUnsigned(localY, y);
-      }
-      else
-      {
-         elem_restrict->MultTranspose(localY, y);
-      }
-   }
-   else
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AssembleDiagonalMF(y);
-      }
-   }
-}
-
-void MFBilinearFormExtension::Update()
-{
-   FiniteElementSpace *fes = a->FESpace();
-   height = width = fes->GetVSize();
-   trialFes = fes;
-   testFes = fes;
-
-   elem_restrict = nullptr;
-   int_face_restrict_lex = nullptr;
-   bdr_face_restrict_lex = nullptr;
-}
-
-void MFBilinearFormExtension::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                               OperatorHandle &A)
-{
-   Operator *oper;
-   Operator::FormSystemOperator(ess_tdof_list, oper);
-   A.Reset(oper); // A will own oper
-}
-
-void MFBilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                               Vector &x, Vector &b,
-                                               OperatorHandle &A,
-                                               Vector &X, Vector &B,
-                                               int copy_interior)
-{
-   Operator *oper;
-   Operator::FormLinearSystem(ess_tdof_list, x, b, oper, X, B, copy_interior);
-   A.Reset(oper); // A will own oper
-}
-
-void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-   if (DeviceCanUseCeed() || !elem_restrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultMF(x, y);
-      }
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultMF(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultMF(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultMF(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int iSz = integrators.Size();
-   if (elem_restrict)
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposeMF(localX, localY);
-      }
-      elem_restrict->MultTranspose(localY, y);
-   }
-   else
-   {
-      y.UseDevice(true);
-      y = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         integrators[i]->AddMultTransposeMF(x, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         for (int i = 0; i < iFISz; ++i)
-         {
-            intFaceIntegrators[i]->AddMultTransposeMF(faceIntX, faceIntY);
-         }
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
-   {
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         for (int i = 0; i < bFISz; ++i)
-         {
-            bdrFaceIntegrators[i]->AddMultTransposeMF(faceBdrX, faceBdrY);
-         }
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}

 // Data and methods for partially-assembled bilinear forms
 PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
@@ -248,7 +47,7 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
   bdr_face_restrict_lex = NULL;
 }

-void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
+void PABilinearFormExtension::SetupRestrictionOperators()
 {
   ElementDofOrdering ordering = UsesTensorBasis(*a->FESpace())?
                                 ElementDofOrdering::LEXICOGRAPHIC:
@@ -266,8 +65,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
   if (int_face_restrict_lex == NULL && a->GetFBFI()->Size() > 0)
   {
      int_face_restrict_lex = trialFes->GetFaceRestriction(
-                                 ElementDofOrdering::LEXICOGRAPHIC,
-                                 FaceType::Interior);
+                                 ElementDofOrdering::LEXICOGRAPHIC, FaceType::Interior);
      faceIntX.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      faceIntY.SetSize(int_face_restrict_lex->Height(), Device::GetMemoryType());
      faceIntY.UseDevice(true); // ensure 'faceIntY = 0.0' is done on device
@@ -276,9 +74,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
   if (bdr_face_restrict_lex == NULL && a->GetBFBFI()->Size() > 0)
   {
      bdr_face_restrict_lex = trialFes->GetFaceRestriction(
-                                 ElementDofOrdering::LEXICOGRAPHIC,
-                                 FaceType::Boundary,
-                                 m);
+                                 ElementDofOrdering::LEXICOGRAPHIC, FaceType::Boundary);
      faceBdrX.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      faceBdrY.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      faceBdrY.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device
@@ -287,7 +83,7 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)

 void PABilinearFormExtension::Assemble()
 {
-   SetupRestrictionOperators(L2FaceValues::DoubleValued);
+   SetupRestrictionOperators();

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
   const int integratorCount = integrators.Size();
@@ -296,9 +92,6 @@ void PABilinearFormExtension::Assemble()
      integrators[i]->AssemblePA(*a->FESpace());
   }

-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
-
   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
   const int intFaceIntegratorCount = intFaceIntegrators.Size();
   for (int i = 0; i < intFaceIntegratorCount; ++i)
@@ -319,7 +112,7 @@ void PABilinearFormExtension::AssembleDiagonal(Vector &y) const
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

   const int iSz = integrators.Size();
-   if (elem_restrict && !DeviceCanUseCeed())
+   if (elem_restrict)
   {
      localY = 0.0;
      for (int i = 0; i < iSz; ++i)
@@ -494,463 +287,6 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   }
 }

-// Data and methods for element-assembled bilinear forms
-EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
-   : PABilinearFormExtension(form),
-     factorize_face_terms(form->FESpace()->IsDGSpace())
-{
-}
-
-void EABilinearFormExtension::Assemble()
-{
-   SetupRestrictionOperators(L2FaceValues::SingleValued);
-
-   ne = trialFes->GetMesh()->GetNE();
-   elemDofs = trialFes->GetFE(0)->GetDof();
-
-   ea_data.SetSize(ne*elemDofs*elemDofs, Device::GetMemoryType());
-   ea_data.UseDevice(true);
-
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-   const int integratorCount = integrators.Size();
-   for (int i = 0; i < integratorCount; ++i)
-   {
-      integrators[i]->AssembleEA(*a->FESpace(), ea_data, i);
-   }
-
-   faceDofs = trialFes ->
-              GetTraceElement(0, trialFes->GetMesh()->GetFaceBaseGeometry(0)) ->
-              GetDof();
-
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Element assembly does not support AddBoundaryIntegrator yet.");
-
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int intFaceIntegratorCount = intFaceIntegrators.Size();
-   if (intFaceIntegratorCount>0)
-   {
-      nf_int = trialFes->GetNFbyType(FaceType::Interior);
-      ea_data_int.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_ext.SetSize(2*nf_int*faceDofs*faceDofs, Device::GetMemoryType());
-   }
-   for (int i = 0; i < intFaceIntegratorCount; ++i)
-   {
-      intFaceIntegrators[i]->AssembleEAInteriorFaces(*a->FESpace(),
-                                                     ea_data_int,
-                                                     ea_data_ext,
-                                                     i);
-   }
-
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int boundFaceIntegratorCount = bdrFaceIntegrators.Size();
-   if (boundFaceIntegratorCount>0)
-   {
-      nf_bdr = trialFes->GetNFbyType(FaceType::Boundary);
-      ea_data_bdr.SetSize(nf_bdr*faceDofs*faceDofs, Device::GetMemoryType());
-      ea_data_bdr = 0.0;
-   }
-   for (int i = 0; i < boundFaceIntegratorCount; ++i)
-   {
-      bdrFaceIntegrators[i]->AssembleEABoundaryFaces(*a->FESpace(),ea_data_bdr,i);
-   }
-
-   if (factorize_face_terms && int_face_restrict_lex)
-   {
-      auto restFint = dynamic_cast<const L2FaceRestriction&>(*int_face_restrict_lex);
-      restFint.AddFaceMatricesToElementMatrices(ea_data_int, ea_data);
-   }
-   if (factorize_face_terms && bdr_face_restrict_lex)
-   {
-      auto restFbdr = dynamic_cast<const L2FaceRestriction&>(*bdr_face_restrict_lex);
-      restFbdr.AddFaceMatricesToElementMatrices(ea_data_bdr, ea_data);
-   }
-}
-
-void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   // Apply the Element Restriction
-   const bool useRestrict = !DeviceCanUseCeed() && elem_restrict;
-   if (!useRestrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-   }
-   // Apply the Element Matrices
-   const int NDOFS = elemDofs;
-   auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
-   auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
-   auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-   MFEM_FORALL(glob_j, ne*NDOFS,
-   {
-      const int e = glob_j/NDOFS;
-      const int j = glob_j%NDOFS;
-      double res = 0.0;
-      for (int i = 0; i < NDOFS; i++)
-      {
-         res += A(i, j, e)*X(i, e);
-      }
-      Y(j, e) += res;
-   });
-   // Apply the Element Restriction transposed
-   if (useRestrict)
-   {
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   // Treatment of interior faces
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      // Apply the Interior Face Restriction
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         // Apply the interior face matrices
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
-         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
-         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
-            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(i, j, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
-         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(i, j, 0, f)*X(i, 0, f);
-            }
-            Y(j, 1, f) += res;
-            res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(i, j, 1, f)*X(i, 1, f);
-            }
-            Y(j, 0, f) += res;
-         });
-         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   // Treatment of boundary faces
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
-   {
-      // Apply the Boundary Face Restriction
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         // Apply the boundary face matrices
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceBdrX.Read(), NDOFS, nf_bdr);
-         auto Y = Reshape(faceBdrY.ReadWrite(), NDOFS, nf_bdr);
-         auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A(i, j, f)*X(i, f);
-            }
-            Y(j, f) += res;
-         });
-         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   // Apply the Element Restriction
-   const bool useRestrict = DeviceCanUseCeed() || !elem_restrict;
-   if (!useRestrict)
-   {
-      y.UseDevice(true); // typically this is a large vector, so store on device
-      y = 0.0;
-   }
-   else
-   {
-      elem_restrict->Mult(x, localX);
-      localY = 0.0;
-   }
-   // Apply the Element Matrices transposed
-   const int NDOFS = elemDofs;
-   auto X = Reshape(useRestrict?localX.Read():x.Read(), NDOFS, ne);
-   auto Y = Reshape(useRestrict?localY.ReadWrite():y.ReadWrite(), NDOFS, ne);
-   auto A = Reshape(ea_data.Read(), NDOFS, NDOFS, ne);
-   MFEM_FORALL(glob_j, ne*NDOFS,
-   {
-      const int e = glob_j/NDOFS;
-      const int j = glob_j%NDOFS;
-      double res = 0.0;
-      for (int i = 0; i < NDOFS; i++)
-      {
-         res += A(j, i, e)*X(i, e);
-      }
-      Y(j, e) += res;
-   });
-   // Apply the Element Restriction transposed
-   if (useRestrict)
-   {
-      elem_restrict->MultTranspose(localY, y);
-   }
-
-   // Treatment of interior faces
-   Array<BilinearFormIntegrator*> &intFaceIntegrators = *a->GetFBFI();
-   const int iFISz = intFaceIntegrators.Size();
-   if (int_face_restrict_lex && iFISz>0)
-   {
-      // Apply the Interior Face Restriction
-      int_face_restrict_lex->Mult(x, faceIntX);
-      if (faceIntX.Size()>0)
-      {
-         faceIntY = 0.0;
-         // Apply the interior face matrices transposed
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceIntX.Read(), NDOFS, 2, nf_int);
-         auto Y = Reshape(faceIntY.ReadWrite(), NDOFS, 2, nf_int);
-         if (!factorize_face_terms)
-         {
-            auto A_int = Reshape(ea_data_int.Read(), NDOFS, NDOFS, 2, nf_int);
-            MFEM_FORALL(glob_j, nf_int*NDOFS,
-            {
-               const int f = glob_j/NDOFS;
-               const int j = glob_j%NDOFS;
-               double res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 0, f)*X(i, 0, f);
-               }
-               Y(j, 0, f) += res;
-               res = 0.0;
-               for (int i = 0; i < NDOFS; i++)
-               {
-                  res += A_int(j, i, 1, f)*X(i, 1, f);
-               }
-               Y(j, 1, f) += res;
-            });
-         }
-         auto A_ext = Reshape(ea_data_ext.Read(), NDOFS, NDOFS, 2, nf_int);
-         MFEM_FORALL(glob_j, nf_int*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(j, i, 0, f)*X(i, 0, f);
-            }
-            Y(j, 1, f) += res;
-            res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A_ext(j, i, 1, f)*X(i, 1, f);
-            }
-            Y(j, 0, f) += res;
-         });
-         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->MultTranspose(faceIntY, y);
-      }
-   }
-
-   // Treatment of boundary faces
-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (!factorize_face_terms && bdr_face_restrict_lex && bFISz>0)
-   {
-      // Apply the Boundary Face Restriction
-      bdr_face_restrict_lex->Mult(x, faceBdrX);
-      if (faceBdrX.Size()>0)
-      {
-         faceBdrY = 0.0;
-         // Apply the boundary face matrices transposed
-         const int NDOFS = faceDofs;
-         auto X = Reshape(faceBdrX.Read(), NDOFS, nf_bdr);
-         auto Y = Reshape(faceBdrY.ReadWrite(), NDOFS, nf_bdr);
-         auto A = Reshape(ea_data_bdr.Read(), NDOFS, NDOFS, nf_bdr);
-         MFEM_FORALL(glob_j, nf_bdr*NDOFS,
-         {
-            const int f = glob_j/NDOFS;
-            const int j = glob_j%NDOFS;
-            double res = 0.0;
-            for (int i = 0; i < NDOFS; i++)
-            {
-               res += A(j, i, f)*X(i, f);
-            }
-            Y(j, f) += res;
-         });
-         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->MultTranspose(faceBdrY, y);
-      }
-   }
-}
-
-// Data and methods for fully-assembled bilinear forms
-FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
-   : EABilinearFormExtension(form),
-     mat(form->FESpace()->GetVSize(),form->FESpace()->GetVSize(),0),
-     face_mat(form->FESpace()->GetVSize(),0,0),
-     use_face_mat(false)
-{
-#ifdef MFEM_USE_MPI
-   if ( ParFiniteElementSpace* pfes =
-           dynamic_cast<ParFiniteElementSpace*>(form->FESpace()) )
-   {
-      if (pfes->IsDGSpace())
-      {
-         use_face_mat = true;
-         pfes->ExchangeFaceNbrData();
-         face_mat.SetWidth(pfes->GetFaceNbrVSize());
-      }
-   }
-#endif
-}
-
-void FABilinearFormExtension::Assemble()
-{
-   EABilinearFormExtension::Assemble();
-   FiniteElementSpace &fes = *a->FESpace();
-   if (fes.IsDGSpace())
-   {
-      const L2ElementRestriction *restE =
-         static_cast<const L2ElementRestriction*>(elem_restrict);
-      const L2FaceRestriction *restF =
-         static_cast<const L2FaceRestriction*>(int_face_restrict_lex);
-      // 1. Fill I
-      //  1.1 Increment with restE
-      restE->FillI(mat);
-      //  1.2 Increment with restF
-      if (restF) { restF->FillI(mat, face_mat); }
-      //  1.3 Sum the non-zeros in I
-      auto h_I = mat.HostReadWriteI();
-      int cpt = 0;
-      const int vd = fes.GetVDim();
-      const int ndofs = ne*elemDofs*vd;
-      for (int i = 0; i < ndofs; i++)
-      {
-         const int nnz = h_I[i];
-         h_I[i] = cpt;
-         cpt += nnz;
-      }
-      const int nnz = cpt;
-      h_I[ndofs] = nnz;
-      mat.GetMemoryJ().New(nnz, mat.GetMemoryJ().GetMemoryType());
-      mat.GetMemoryData().New(nnz, mat.GetMemoryData().GetMemoryType());
-      if (use_face_mat && restF)
-      {
-         auto h_I_face = face_mat.HostReadWriteI();
-         int cpt = 0;
-         for (int i = 0; i < ndofs; i++)
-         {
-            const int nnz = h_I_face[i];
-            h_I_face[i] = cpt;
-            cpt += nnz;
-         }
-         const int nnz_face = cpt;
-         h_I_face[ndofs] = nnz_face;
-         face_mat.GetMemoryJ().New(nnz_face,
-                                   face_mat.GetMemoryJ().GetMemoryType());
-         face_mat.GetMemoryData().New(nnz_face,
-                                      face_mat.GetMemoryData().GetMemoryType());
-      }
-      // 2. Fill J and Data
-      // 2.1 Fill J and Data with Elem ea_data
-      restE->FillJAndData(ea_data, mat);
-      // 2.2 Fill J and Data with Face ea_data_ext
-      if (restF) { restF->FillJAndData(ea_data_ext, mat, face_mat); }
-      // 2.3 Shift indirections in I back to original
-      auto I = mat.HostReadWriteI();
-      for (int i = ndofs; i > 0; i--)
-      {
-         I[i] = I[i-1];
-      }
-      I[0] = 0;
-      if (use_face_mat && restF)
-      {
-         auto I_face = face_mat.HostReadWriteI();
-         for (int i = ndofs; i > 0; i--)
-         {
-            I_face[i] = I_face[i-1];
-         }
-         I_face[0] = 0;
-      }
-   }
-   else // continuous Galerkin case
-   {
-      const ElementRestriction &rest =
-         static_cast<const ElementRestriction&>(*elem_restrict);
-      rest.FillSparseMatrix(ea_data, mat);
-   }
-}
-
-void FABilinearFormExtension::Mult(const Vector &x, Vector &y) const
-{
-   mat.Mult(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMult(shared_x, y); }
-   }
-#endif
-}
-
-void FABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
-{
-   mat.MultTranspose(x, y);
-#ifdef MFEM_USE_MPI
-   if (const ParFiniteElementSpace *pfes =
-          dynamic_cast<const ParFiniteElementSpace*>(testFes))
-   {
-      ParGridFunction x_gf;
-      x_gf.MakeRef(const_cast<ParFiniteElementSpace*>(pfes),
-                   const_cast<Vector&>(x),0);
-      x_gf.ExchangeFaceNbrData();
-      Vector &shared_x = x_gf.FaceNbrData();
-      if (shared_x.Size()) { face_mat.AddMultTranspose(shared_x, y); }
-   }
-#endif
-}
-
-
 MixedBilinearFormExtension::MixedBilinearFormExtension(MixedBilinearForm *form)
   : Operator(form->Height(), form->Width()), a(form)
 {
@@ -998,12 +334,6 @@ void PAMixedBilinearFormExtension::Assemble()
   {
      integrators[i]->AssemblePA(*trialFes, *testFes);
   }
-   MFEM_VERIFY(a->GetBBFI()->Size() == 0,
-               "Partial assembly does not support AddBoundaryIntegrator yet.");
-   MFEM_VERIFY(a->GetTFBFI()->Size() == 0,
-               "Partial assembly does not support AddTraceFaceIntegrator yet.");
-   MFEM_VERIFY(a->GetBTFBFI()->Size() == 0,
-               "Partial assembly does not support AddBdrTraceFaceIntegrator yet.");
 }

 void PAMixedBilinearFormExtension::Update()
@@ -1157,68 +487,4 @@ void PAMixedBilinearFormExtension::AddMultTranspose(const Vector &x, Vector &y,
   }
 }

-void PAMixedBilinearFormExtension::AssembleDiagonal_ADAt(const Vector &D,
-                                                         Vector &diag) const
-{
-   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
-
-   const int iSz = integrators.Size();
-
-   if (elem_restrict_trial)
-   {
-      const ElementRestriction* H1elem_restrict_trial =
-         dynamic_cast<const ElementRestriction*>(elem_restrict_trial);
-      if (H1elem_restrict_trial)
-      {
-         H1elem_restrict_trial->MultUnsigned(D, localTrial);
-      }
-      else
-      {
-         elem_restrict_trial->Mult(D, localTrial);
-      }
-   }
-
-   if (elem_restrict_test)
-   {
-      localTest = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         if (elem_restrict_trial)
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(localTrial, localTest);
-         }
-         else
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(D, localTest);
-         }
-      }
-      const ElementRestriction* H1elem_restrict_test =
-         dynamic_cast<const ElementRestriction*>(elem_restrict_test);
-      if (H1elem_restrict_test)
-      {
-         H1elem_restrict_test->MultTransposeUnsigned(localTest, diag);
-      }
-      else
-      {
-         elem_restrict_test->MultTranspose(localTest, diag);
-      }
-   }
-   else
-   {
-      diag.UseDevice(true); // typically this is a large vector, so store on device
-      diag = 0.0;
-      for (int i = 0; i < iSz; ++i)
-      {
-         if (elem_restrict_trial)
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(localTrial, diag);
-         }
-         else
-         {
-            integrators[i]->AssembleDiagonalPA_ADAt(D, diag);
-         }
-      }
-   }
-}
-
 } // namespace mfem
@@ -22,12 +22,9 @@ namespace mfem
 class BilinearForm;
 class MixedBilinearForm;

-/// Class extending the BilinearForm class to support different AssemblyLevels.
-/**  FA - Full Assembly
-     PA - Partial Assembly
-     EA - Element Assembly
-     MF - Matrix Free
-*/
+
+/** @brief Class extending the BilinearForm class to support the different
+    AssemblyLevel%s. */
 class BilinearFormExtension : public Operator
 {
 protected:
@@ -45,7 +42,6 @@ public:
   /// Get the finite element space restriction matrix
   virtual const Operator *GetRestriction() const;

-   /// Assemble at the level given for the BilinearFormExtension subclass
   virtual void Assemble() = 0;

   virtual void AssembleDiagonal(Vector &diag) const
@@ -62,6 +58,46 @@ public:
   virtual void Update() = 0;
 };

+/// Data and methods for fully-assembled bilinear forms
+class FABilinearFormExtension : public BilinearFormExtension
+{
+public:
+   FABilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~FABilinearFormExtension() {}
+};
+
+/// Data and methods for element-assembled bilinear forms
+class EABilinearFormExtension : public BilinearFormExtension
+{
+public:
+   EABilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }
+
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~EABilinearFormExtension() {}
+};
+
 /// Data and methods for partially-assembled bilinear forms
 class PABilinearFormExtension : public BilinearFormExtension
 {
@@ -77,6 +113,7 @@ protected:
 public:
   PABilinearFormExtension(BilinearForm*);

+   void SetupRestrictionOperators();
   void Assemble();
   void AssembleDiagonal(Vector &diag) const;
   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
@@ -84,85 +121,35 @@ public:
                         Vector &x, Vector &b,
                         OperatorHandle &A, Vector &X, Vector &B,
                         int copy_interior = 0);
+
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;
   void Update();
-
-protected:
-   void SetupRestrictionOperators(const L2FaceValues m);
 };

-/// Data and methods for element-assembled bilinear forms
-class EABilinearFormExtension : public PABilinearFormExtension
-{
-protected:
-   int ne;
-   int elemDofs;
-   // The element matrices are stored row major
-   Vector ea_data;
-   int nf_int, nf_bdr;
-   int faceDofs;
-   Vector ea_data_int, ea_data_ext, ea_data_bdr;
-   bool factorize_face_terms;
-
-public:
-   EABilinearFormExtension(BilinearForm *form);
-
-   void Assemble();
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};
-
-/// Data and methods for fully-assembled bilinear forms
-class FABilinearFormExtension : public EABilinearFormExtension
-{
-private:
-   SparseMatrix mat;
-   /// face_mat handles parallelism for DG face terms.
-   SparseMatrix face_mat;
-   bool use_face_mat;
-
-public:
-   FABilinearFormExtension(BilinearForm *form);
-
-   void Assemble();
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-};

 /// Data and methods for matrix-free bilinear forms
 class MFBilinearFormExtension : public BilinearFormExtension
 {
-protected:
-   const FiniteElementSpace *trialFes, *testFes; // Not owned
-   mutable Vector localX, localY;
-   mutable Vector faceIntX, faceIntY;
-   mutable Vector faceBdrX, faceBdrY;
-   const Operator *elem_restrict; // Not owned
-   const Operator *int_face_restrict_lex; // Not owned
-   const Operator *bdr_face_restrict_lex; // Not owned
-
 public:
-   MFBilinearFormExtension(BilinearForm *form);
+   MFBilinearFormExtension(BilinearForm *form)
+      : BilinearFormExtension(form) { }

-   void Assemble();
-   void AssembleDiagonal(Vector &diag) const;
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   /// TODO
+   void Assemble() {}
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A) {}
   void FormLinearSystem(const Array<int> &ess_tdof_list,
                         Vector &x, Vector &b,
                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
-   void Mult(const Vector &x, Vector &y) const;
-   void MultTranspose(const Vector &x, Vector &y) const;
-   void Update();
+                         int copy_interior = 0) {}
+   void Mult(const Vector &x, Vector &y) const {}
+   void MultTranspose(const Vector &x, Vector &y) const {}
+   void Update() {}
+   ~MFBilinearFormExtension() {}
 };

-/// Class extending the MixedBilinearForm class to support different AssemblyLevels.
-/**  FA - Full Assembly
-     PA - Partial Assembly
-     EA - Element Assembly
-     MF - Matrix Free
-*/
+/** @brief Class extending the MixedBilinearForm class to support the different
+    AssemblyLevel%s. */
 class MixedBilinearFormExtension : public Operator
 {
 protected:
@@ -199,8 +186,6 @@ public:
   virtual void AddMultTranspose(const Vector &x, Vector &y,
                                 const double c=1.0) const = 0;

-   virtual void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const = 0;
-
   virtual void Update() = 0;
 };

@@ -251,9 +236,6 @@ public:
   void MultTranspose(const Vector &x, Vector &y) const;
   /// y += c*A^T*x
   void AddMultTranspose(const Vector &x, Vector &y, const double c=1.0) const;
-   /// Assemble the diagonal of ADA^T for a diagonal vector D.
-   void AssembleDiagonal_ADAt(const Vector &D, Vector &diag) const;
-
   /// Update internals for when a new MixedBilinearForm is given to this class
   void Update();
 };
@@ -47,40 +47,7 @@ void BilinearFormIntegrator::AssemblePABoundaryFaces(const FiniteElementSpace&)

 void BilinearFormIntegrator::AssembleDiagonalPA(Vector &)
 {
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleEA(const FiniteElementSpace &fes,
-                                        Vector &emat,
-                                        const bool add)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleEA(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleEAInteriorFaces(const FiniteElementSpace
-                                                     &fes,
-                                                     Vector &ea_data_int,
-                                                     Vector &ea_data_ext,
-                                                     const bool add)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleEAInteriorFaces(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleEABoundaryFaces(const FiniteElementSpace
-                                                     &fes,
-                                                     Vector &ea_data_bdr,
-                                                     const bool add)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleEABoundaryFaces(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
-{
-   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA_ADAt(...)\n"
+   MFEM_ABORT("BilinearFormIntegrator::AssembleDiagonalPA(...)\n"
              "   is not implemented for this class.");
 }

@@ -96,30 +63,6 @@ void BilinearFormIntegrator::AddMultTransposePA(const Vector &, Vector &) const
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AssembleMF(const FiniteElementSpace &fes)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultMF(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::AddMultMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultTransposeMF(const Vector &, Vector &) const
-{
-   mfem_error ("BilinearFormIntegrator::AddMultTransposeMF(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleDiagonalMF(...)\n"
-               "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AssembleElementMatrix (
   const FiniteElement &el, ElementTransformation &Trans,
   DenseMatrix &elmat )
@@ -157,22 +100,8 @@ void BilinearFormIntegrator::AssembleElementVector(
   const FiniteElement &el, ElementTransformation &Tr, const Vector &elfun,
   Vector &elvect)
 {
-   // Note: This default implementation is general but not efficient
-   DenseMatrix elmat;
-   AssembleElementMatrix(el, Tr, elmat);
-   elvect.SetSize(elmat.Height());
-   elmat.Mult(elfun, elvect);
-}
-
-void BilinearFormIntegrator::AssembleFaceVector(
-   const FiniteElement &el1, const FiniteElement &el2,
-   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
-{
-   // Note: This default implementation is general but not efficient
-   DenseMatrix elmat;
-   AssembleFaceMatrix(el1, el2, Tr, elmat);
-   elvect.SetSize(elmat.Height());
-   elmat.Mult(elfun, elvect);
+   mfem_error("BilinearFormIntegrator::AssembleElementVector\n"
+              "   is not implemented for this class.");
 }


@@ -554,12 +483,10 @@ void DiffusionIntegrator::AssembleElementMatrix

 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), dshapedxt(nd,spaceDim), invdfdx(dim,spaceDim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd,dim);
   dshapedxt.SetSize(nd,spaceDim);
   invdfdx.SetSize(dim,spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   elmat.SetSize(nd);

@@ -577,20 +504,7 @@ void DiffusionIntegrator::AssembleElementMatrix
      // AdjugateJacobian = / adj(J),         if J is square
      //                    \ adj(J^t.J).J^t, otherwise
      Mult(dshape, Trans.AdjugateJacobian(), dshapedxt);
-      if (MQ)
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(dshapedxt, invdfdx, dshape);
-         AddMultABt(dshape, dshapedxt, elmat);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(dshapedxt, D, elmat);
-      }
-      else
+      if (!MQ)
      {
         if (Q)
         {
@@ -598,6 +512,13 @@ void DiffusionIntegrator::AssembleElementMatrix
         }
         AddMult_a_AAt(w, dshapedxt, elmat);
      }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(dshapedxt, invdfdx, dshape);
+         AddMultABt(dshape, dshapedxt, elmat);
+      }
   }
 }

@@ -616,14 +537,12 @@ void DiffusionIntegrator::AssembleElementMatrix2(
   DenseMatrix dshape(tr_nd, dim), dshapedxt(tr_nd, spaceDim);
   DenseMatrix te_dshape(te_nd, dim), te_dshapedxt(te_nd, spaceDim);
   DenseMatrix invdfdx(dim, spaceDim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(tr_nd, dim);
   dshapedxt.SetSize(tr_nd, spaceDim);
   te_dshape.SetSize(te_nd, dim);
   te_dshapedxt.SetSize(te_nd, spaceDim);
   invdfdx.SetSize(dim, spaceDim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   elmat.SetSize(te_nd, tr_nd);

@@ -643,20 +562,7 @@ void DiffusionIntegrator::AssembleElementMatrix2(
      Mult(dshape, invdfdx, dshapedxt);
      Mult(te_dshape, invdfdx, te_dshapedxt);
      // invdfdx, dshape, and te_dshape no longer needed
-      if (MQ)
-      {
-         MQ->Eval(invdfdx, Trans, ip);
-         invdfdx *= w;
-         Mult(te_dshapedxt, invdfdx, te_dshape);
-         AddMultABt(te_dshape, dshapedxt, elmat);
-      }
-      else if (VQ)
-      {
-         VQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(dshapedxt, D, elmat);
-      }
-      else
+      if (!MQ)
      {
         if (Q)
         {
@@ -665,6 +571,13 @@ void DiffusionIntegrator::AssembleElementMatrix2(
         dshapedxt *= w;
         AddMultABt(te_dshapedxt, dshapedxt, elmat);
      }
+      else
+      {
+         MQ->Eval(invdfdx, Trans, ip);
+         invdfdx *= w;
+         Mult(te_dshapedxt, invdfdx, te_dshape);
+         AddMultABt(te_dshape, dshapedxt, elmat);
+      }
   }
 }

@@ -676,19 +589,12 @@ void DiffusionIntegrator::AssembleElementVector(
   int dim = el.GetDim();
   double w;

-   if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "Unexpected dimension for VectorCoefficient");
-   }
-
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), invdfdx(dim), mq(dim);
-   Vector D(VQ ? VQ->GetVDim() : 0);
 #else
   dshape.SetSize(nd,dim);
   invdfdx.SetSize(dim);
   mq.SetSize(dim);
-   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif
   vec.SetSize(dim);
   pointflux.SetSize(dim);
@@ -707,7 +613,7 @@ void DiffusionIntegrator::AssembleElementVector(
      CalcAdjugate(Tr.Jacobian(), invdfdx); // invdfdx = adj(J)
      w = ip.weight / Tr.Weight();

-      if (!MQ && !VQ)
+      if (!MQ)
      {
         dshape.MultTranspose(elfun, vec);
         invdfdx.MultTranspose(vec, pointflux);
@@ -718,21 +624,11 @@ void DiffusionIntegrator::AssembleElementVector(
      }
      else
      {
+
         dshape.MultTranspose(elfun, pointflux);
         invdfdx.MultTranspose(pointflux, vec);
-         if (MQ)
-         {
-            MQ->Eval(mq, Tr, ip);
-            mq.Mult(vec, pointflux);
-         }
-         else
-         {
-            VQ->Eval(D, Tr, ip);
-            for (int j=0; j<dim; ++j)
-            {
-               pointflux[j] *= D[j];
-            }
-         }
+         MQ->Eval(mq, Tr, ip);
+         mq.Mult(vec, pointflux);
      }
      pointflux *= w;
      invdfdx.Mult(pointflux, vec);
@@ -993,25 +889,22 @@ void BoundaryMassIntegrator::AssembleFaceMatrix(
   {
      int order = 2 * el1.GetOrder();

-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   elmat = 0.0;
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      // Set the integration point in the face and the neighboring element
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring element's integration point
-      const IntegrationPoint &eip = Trans.GetElement1IntPoint();
+      IntegrationPoint eip;
+      Trans.Loc1.Transform(ip, eip);
      el1.CalcShape(eip, shape);

-      w = Trans.Weight() * ip.weight;
+      Trans.Face->SetIntPoint(&ip);
+      w = Trans.Face->Weight() * ip.weight;
      if (Q)
      {
-         w *= Q -> Eval(Trans, ip);
+         w *= Q -> Eval(*Trans.Face, ip);
      }

      AddMult_a_VVt(w, shape, elmat);
@@ -1596,7 +1489,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
   double w;

 #ifdef MFEM_THREAD_SAFE
-   Vector D;
   DenseMatrix curlshape(nd,dimc), curlshape_dFt(nd,dimc), M;
 #else
   curlshape.SetSize(nd,dimc);
@@ -1604,7 +1496,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
 #endif
   elmat.SetSize(nd);
   if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }

   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
@@ -1648,12 +1539,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
         Mult(curlshape_dFt, M, curlshape);
         AddMultABt(curlshape, curlshape_dFt, elmat);
      }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADAt(curlshape_dFt, D, elmat);
-      }
      else if (Q)
      {
         w *= Q->Eval(Trans, ip);
@@ -2089,7 +1974,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
      D.SetSize(VQ ? VQ->GetVDim() : 0);
      K.SetSize(MQ ? MQ->GetVDim() : 0, MQ ? MQ->GetVDim() : 0);
 #endif
-      DenseMatrix tmp(test_vshape.Height(), K.Width());
+      DenseMatrix tmp(trial_vshape.Height(), K.Width());

      elmat.SetSize (test_dof, trial_dof);

@@ -2265,7 +2150,16 @@ void VectorDiffusionIntegrator::AssembleElementMatrix(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      ir = &DiffusionIntegrator::GetRule(el,el);
+      // integrand is rational function if det(J) is not constant
+      int order = 2 * Trans.OrderGrad(&el); // order of the numerator
+      if (el.Space() == FunctionSpace::rQk)
+      {
+         ir = &RefinedIntRules.Get(el.GetGeomType(), order);
+      }
+      else
+      {
+         ir = &IntRules.Get(el.GetGeomType(), order);
+      }
   }

   elmat = 0.0;
@@ -2316,7 +2210,11 @@ void VectorDiffusionIntegrator::AssembleElementVector(
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
-      ir = &DiffusionIntegrator::GetRule(el,el);
+      // integrand is rational function if det(J) is not constant
+      int order = 2 * Tr.OrderGrad(&el); // order of the numerator
+      ir = (el.Space() == FunctionSpace::rQk) ?
+           &RefinedIntRules.Get(el.GetGeomType(), order) :
+           &IntRules.Get(el.GetGeomType(), order);
   }

   elvect = 0.0;
@@ -2637,23 +2535,23 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
      {
         order++;
      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
+      Trans.Loc1.Transform(ip, eip1);
+      if (ndof2)
+      {
+         Trans.Loc2.Transform(ip, eip2);
+      }
      el1.CalcShape(eip1, shape1);

+      Trans.Face->SetIntPoint(&ip);
+      Trans.Elem1->SetIntPoint(&eip1);
+
      u->Eval(vu, *Trans.Elem1, eip1);

      if (dim == 1)
@@ -2662,7 +2560,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
      }
      else
      {
-         CalcOrtho(Trans.Jacobian(), nor);
+         CalcOrtho(Trans.Face->Jacobian(), nor);
      }

      un = vu * nor;
@@ -2677,6 +2575,7 @@ void DGTraceIntegrator::AssembleFaceMatrix(const FiniteElement &el1,
         double rho_p;
         if (un >= 0.0 && ndof2)
         {
+            Trans.Elem2->SetIntPoint(&eip2);
            rho_p = rho->Eval(*Trans.Elem2, eip2);
         }
         else
@@ -2792,7 +2691,7 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
      {
         order = 2*el1.GetOrder();
      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   // assemble: < {(Q \nabla u).n},[v] >      --> elmat
@@ -2800,26 +2699,22 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(
   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
+      IntegrationPoint eip1, eip2;

-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.Face->SetIntPoint(&ip);
      if (dim == 1)
      {
         nor(0) = 2*eip1.x - 1.0;
      }
      else
      {
-         CalcOrtho(Trans.Jacobian(), nor);
+         CalcOrtho(Trans.Face->Jacobian(), nor);
      }

      el1.CalcShape(eip1, shape1);
      el1.CalcDShape(eip1, dshape1);
+      Trans.Elem1->SetIntPoint(&eip1);
      w = ip.weight/Trans.Elem1->Weight();
      if (ndof2)
      {
@@ -2865,8 +2760,10 @@ void DGDiffusionIntegrator::AssembleFaceMatrix(

      if (ndof2)
      {
+         Trans.Loc2.Transform(ip, eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
+         Trans.Elem2->SetIntPoint(&eip2);
         w = ip.weight/2/Trans.Elem2->Weight();
         if (!MQ)
         {
@@ -3076,20 +2973,16 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
   {
      // a simple choice for the integration order; is this OK?
      const int order = 2 * max(el1.GetOrder(), ndofs2 ? el2.GetOrder() : 0);
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int pind = 0; pind < ir->GetNPoints(); ++pind)
   {
      const IntegrationPoint &ip = ir->IntPoint(pind);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
+      IntegrationPoint eip1, eip2; // integration point in the reference space
+      Trans.Loc1.Transform(ip, eip1);
+      Trans.Face->SetIntPoint(&ip);
+      Trans.Elem1->SetIntPoint(&eip1);

      el1.CalcShape(eip1, shape1);
      el1.CalcDShape(eip1, dshape1);
@@ -3103,12 +2996,14 @@ void DGElasticityIntegrator::AssembleFaceMatrix(
      }
      else
      {
-         CalcOrtho(Trans.Jacobian(), nor);
+         CalcOrtho(Trans.Face->Jacobian(), nor);
      }

      double w, wLM;
      if (ndofs2)
      {
+         Trans.Loc2.Transform(ip, eip2);
+         Trans.Elem2->SetIntPoint(&eip2);
         el2.CalcShape(eip2, shape2);
         el2.CalcDShape(eip2, dshape2);
         CalcAdjugate(Trans.Elem2->Jacobian(), adjJ);
@@ -3238,36 +3133,33 @@ void TraceJumpIntegrator::AssembleFaceMatrix(
      order += trial_face_fe.GetOrder();
      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
      {
-         order += Trans.OrderW();
+         order += Trans.Face->OrderW();
      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int p = 0; p < ir->GetNPoints(); p++)
   {
      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-
-      // Access the neighboring elements' integration points
-      // Note: eip2 will only contain valid data if Elem2 exists
-      const IntegrationPoint &eip1 = Trans.GetElement1IntPoint();
-      const IntegrationPoint &eip2 = Trans.GetElement2IntPoint();
-
+      IntegrationPoint eip1, eip2;
      // Trace finite element shape function
+      Trans.Face->SetIntPoint(&ip);
      trial_face_fe.CalcShape(ip, face_shape);
      // Side 1 finite element shape function
+      Trans.Loc1.Transform(ip, eip1);
      test_fe1.CalcShape(eip1, shape1);
+      Trans.Elem1->SetIntPoint(&eip1);
      if (ndof2)
      {
         // Side 2 finite element shape function
+         Trans.Loc2.Transform(ip, eip2);
         test_fe2.CalcShape(eip2, shape2);
+         Trans.Elem2->SetIntPoint(&eip2);
      }
      w = ip.weight;
      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
      {
-         w *= Trans.Weight();
+         w *= Trans.Face->Weight();
      }
      face_shape *= w;
      for (i = 0; i < ndof1; i++)
@@ -3332,7 +3224,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
         order = test_fe1.GetOrder() - 1;
      }
      order += trial_face_fe.GetOrder();
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+      ir = &IntRules.Get(Trans.FaceGeom, order);
   }

   for (int p = 0; p < ir->GetNPoints(); p++)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tucker Babcock	4adfa0fc86	updting io benchmark	2020-05-05 15:44:44 -04:00
Tucker Babcock	857a24f6c4	Merge branch 'PCFinalProject' of github.com:mfem/mfem into PCFinalProject	2020-05-04 11:14:22 -07:00
Tucker Babcock	1ae22c7c69	adding io benchmark	2020-05-04 11:13:42 -07:00
Tucker Babcock	161ebff2a1	Merge branch 'PCFinalProject' of https://github.com/mfem/mfem into PCFinalProject	2020-05-04 12:42:18 -04:00
Tucker Babcock	3548f2cb83	adding num ranks printing	2020-05-04 12:42:13 -04:00
Tucker Babcock	41a7730048	adding barriers ahead of timings and averaging timing over all ranks	2020-05-04 09:40:57 -07:00
Tucker Babcock	b638fb8960	adding all of the operator testing to one file	2020-05-03 22:02:26 -07:00
Tucker Babcock	dc80f42710	Merge branch 'PCFinalProject' of https://github.com/mfem/mfem into PCFinalProject	2020-05-04 00:51:51 -04:00
Tucker Babcock	4414a3fc01	adding test to mfem examples	2020-05-04 00:50:16 -04:00
Tucker Babcock	2f683f80fa	Merge branch 'mpiio-gf-dev' into PCFinalProject	2020-04-30 14:09:33 -07:00
Tucker Babcock	6ea2f7bf55	Merge branch 'mpiio-gf-dev' of github.com:mfem/mfem into mpiio-gf-dev	2020-04-30 14:06:16 -07:00
Tucker Babcock	581cafa7a7	updating documentation	2020-04-30 14:06:10 -07:00
Tucker Babcock	c5bab73f9a	Merge branch 'mpiio-gf-dev' into PCFinalProject	2020-04-30 15:45:12 -04:00
Tucker Babcock	b02eb71967	adding number of files printing control to example 1	2020-04-30 15:39:19 -04:00
Tucker Babcock	d236571e4a	cleaned up code in pgridfunc and added printing to example two.	2020-04-28 21:04:11 -07:00
Tucker Babcock	8d444d7f92	ordering by nodes appears to work now as well	2020-04-28 16:28:07 -07:00
Tucker Babcock	71937096f8	ordering by vdim works with high order	2020-04-28 16:26:30 -07:00
Tucker Babcock	4e0978cf3b	can save and load files correctly for p = 1, errors otherwise.	2020-04-28 15:10:12 -07:00
Tucker Babcock	e52fdd205a	initial commit adding MPI-IO writing of GridFunction supporting writing to arbitrary number of files. Reading support to come	2020-04-27 22:41:28 -07:00