Merge master in lor-gpu

Merge pull request #3199 from mfem/mfem-4.5-dev
Final changes for mfem-4.5
2022-10-23 09:54:23 -07:00 · 2022-10-22 15:33:35 -07:00 · 2022-10-22 15:10:18 -07:00 · 2022-10-21 16:58:01 -07:00 · 2022-10-21 16:57:53 -07:00 · 2022-10-21 16:50:22 -07:00
297 changed files with 14884 additions and 21611 deletions
@@ -23,9 +23,8 @@ install:
 - set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
 - set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include

-# Install METIS, use a mirror because the original source server is not always
-# up. Original url:
-#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
+# Install METIS, use MFEM's mirror because the original source server is often
+# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
@@ -1,4 +1,4 @@
-name: Build Deploy Container
+name: "Docker"

 on:

@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.

 # In this CI section, we build different variants of mfem and run test on them.
-name: builds-and-tests
+name: "Tests"

 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -47,17 +47,17 @@ jobs:
  builds-and-tests:
    strategy:
      matrix:
-        os: [ubuntu-20.04, macos-10.15, windows-2022]
+        os: [ubuntu-latest, macos-latest, windows-latest]
        target: [dbg, opt]
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
        exclude:
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
            build-system: cmake
-          - os: macos-10.15
+          - os: macos-latest
            build-system: cmake
-          - os: windows-2022
+          - os: windows-latest
            build-system: make
        # 'include' allows us to:
        # - Add a variable to all jobs without creating a new matrix dimension.
@@ -72,15 +72,15 @@ jobs:
            codecov: NO
          - target: opt
            codecov: YES
-          - os: windows-2022
+          - os: windows-latest
            codecov: NO
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
@@ -112,35 +112,35 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install lcov

    - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
      uses: mpi4py/setup-mpi@v1.0.3

    # Get Hypre through cache, or build it.
@@ -154,7 +154,7 @@ jobs:
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
@@ -163,7 +163,7 @@ jobs:
        build-system: make

    - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
@@ -175,14 +175,14 @@ jobs:
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
@@ -196,20 +196,16 @@ jobs:
        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1

    - name: prepare binary cache location
-      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
      run: |
        mkdir -p vcpkg_cache

    - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
      env:
        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
      run: |
-        $PortFile = 'C:\vcpkg\ports\metis\portfile.cmake'
-        $OriginalURL = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-${METIS_VERSION}.tar.gz'
-        $NewURL = 'https://github.com/mfem/tpls/raw/gh-pages/metis-5.1.0.tar.gz'
-        (Get-Content $PortFile).replace($OriginalURL, $NewURL) | Set-Content $PortFile
-        vcpkg install metis --triplet=x64-windows-static
+        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports

    # MFEM build and test
    - name: build
@@ -252,7 +248,7 @@ jobs:
      shell: bash

    - name: cmake unit tests (Ubuntu 20.04)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
@@ -260,7 +256,7 @@ jobs:
      shell: bash

    - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "Static Analysis"
+
+on:
+  push:
+    branches: [ "master", "next"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: build-analysis
+name: "Build Analysis"

 permissions:
  actions: write
@@ -31,7 +31,7 @@ env:

 jobs:
  gitignore:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest

    steps:
    - name: Cancel Previous Runs
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: repo-check
+name: "Checks"

 permissions:
  actions: write
@@ -28,7 +28,7 @@ on:

 jobs:
  file-headers-check:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -78,7 +78,7 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -88,14 +88,14 @@ jobs:

    - name: get astyle
      run: |
-        sudo apt-get install astyle=3.1-1ubuntu2
+        sudo apt-get install astyle

    - name: style check
      run: |
        ./config/githooks/pre-push --style

  documentation:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -106,6 +106,8 @@ jobs:
    - name: get doxygen and graphviz
      run: |
        sudo apt-get install doxygen graphviz
+        cd doc
+        doxygen -u CodeDocumentation.conf.in

    - name: build documentation
      run: |
@@ -118,7 +120,7 @@ jobs:
      github.ref != 'refs/heads/master' &&
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -18,6 +18,11 @@ CMakeFiles/
 # Backup files
 *~

+*.sqlite
+*.nsys-rep
+*.qdstrm
+*.csv
+
 # Default install location
 /mfem/

@@ -275,6 +280,7 @@ miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
+miniapps/tools/check-tmop-metric

 miniapps/toys/automata
 miniapps/toys/life
@@ -307,6 +313,7 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh

+miniapps/multidomain/multidomain
 miniapps/hooke/hooke

 # Unit test binary and outputs
@@ -327,6 +334,7 @@ tests/benchmarks/bench_ceed
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals
+tests/benchmarks/bench_lor

 # Test script output
 tests/scripts/*.err
@@ -8,31 +8,48 @@
                               https://mfem.org


-Version 4.4.1 (development)
-===========================
+Version 4.5, released on October 22, 2022
+=========================================

 Meshing improvements
 --------------------
+- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
+  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
+  and work with all existing MFEM interfaces like finite element spaces etc.
+
+- Added a method, ParMesh::GetSerialMesh(), that reconstructs a partitioned
+  parallel mesh on a given single rank. Also, added ParMesh::PrintAsSerial(),
+  which saves the reconstructed serial mesh to a C++ stream on rank 0.
+
+- Added more 3D TMOP metrics, as well as specialized metrics for mesh
+  untangling and worst-case quality improvement.
+
+- Added a new method, Mesh::NodesUpdated, which should be called after the mesh
+  node coordinates have changed, e.g. after the mesh has moved. This is
+  necessary, for example, with device assembly of linear and bilinear forms.
+
 - Added support for mixed meshes and pyramids in GSLIB-FindPoints.

 Discretization improvements
 ---------------------------
- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration.
-
- Added support for partial assembly and fully matrix-free operators on mixed
-  meshes (different element types and p-adaptivity) through libCEED, including
-  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
-  currently limited by MFEM capabilities, i.e. 2D serial meshes. All mixed
-  element topologies are supported in serial and parallel: segment, triangle,
-  square, tetrahedron, cube, prism, and pyramid.
-
 - Added full assembly and device support for several LinearForm integrators:
  * DomainLF: (f, v)
  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
  * DomainLFGrad: (f, grad(v))
  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
+  The device assembly of linear forms has to be explicitly enabled by calling
+  LinearForm::UseFastAssembly(true), otherwise the legacy linear form assembly
+  is used by default.
+
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration with arbitrary user-supplied coefficients.
+
+- Added a new class FaceQuadratureSpace that allows for the construction of
+  QuadratureFunctions on the interior or boundary faces of a mesh.
+
+- Added a class CoefficientVector for efficient access of variable coefficient
+  values at quadrature points (in particular for GPU/device kernels).

 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
  spatial Gaussian white noise.
@@ -40,8 +57,25 @@ Discretization improvements
 - Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
  See fem/estimators.hpp.

+- Various fixes and improvements in LinearFormExtension.
+
 Linear and nonlinear solvers
 ----------------------------
+- Added a new class DGMassInverse that performs a local element-wise CG
+  iteration to solve systems involving the discontinuous Galerkin mass matrix,
+  including support for device/GPU acceleration.
+
+- Added more flexibility to the constrained solver classes:
+  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
+    (necessary for penalty contact)
+  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
+  * All constraint solver classes can take a user-defined preconditioner
+
+ - Added functions to toggle additional options for the SuperLU_Dist and Hypre
+   preconditioners (ParaSails, Euclid, ILU).
+
+- Added boundary elimination with device support for `SparseMatrix` and
+  `HypreParMatrix`.

 New and updated examples and miniapps
 -------------------------------------
@@ -51,12 +85,28 @@ New and updated examples and miniapps
  automatic differentiation tools like a native dual number implementation or a
  third party library such as Enzyme. See miniapps/elasticity for more details.

+- Added example for body-fitted volumetric and shape integration using the
+  Algoim library in miniapps/shifted.
+
 - Add a new example code, Example 33/33p, to demonstrate the solution of
  spectral fractional PDEs with MFEM.

 Integrations, testing and documentation
 ---------------------------------------
 - Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
+  More sophisticated developer containers are available in the new repo
+  https://github.com/mfem/containers.
+
+- Added support for the LLVM-based automatic differentiation tool Enzyme, see
+  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
+  header are provided. The functionality and interaction are demonstrated in a
+  new miniapp in miniapps/elasticity.
+
+- Added support for partial assembly and fully matrix-free operators on mixed
+  meshes (different element types and p-adaptivity) through libCEED, including
+  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
+  currently limited to 2D serial meshes. All mixed element topologies are
+  supported in both serial and parallel.

 - Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
  which provides parallel non-conforming, non-matching, variational, volumetric
@@ -64,29 +114,40 @@ Integrations, testing and documentation
  between arbitrarily distributed and unrelated finite element meshes in a
  variationally consistent way.

- Added support for the LLVM-based automatic differentiation tool Enzyme, see
-  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
-  header are provided. The functionality and interaction are demonstrated in a
-  new miniapp in miniapps/elasticity.
+- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
+  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
+  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.

- Added example for body-fitted volumetric and shape integration using the
-  Algoim library.
+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.

 - Added Windows 2022 CI testing with GitHub actions.

 Miscellaneous
 -------------
- Various other simplifications, extensions, and bugfixes in the code.
+- The method SparseMatrix::EnsureMultTranspose() is now automatically called
+  by the methods AddMultTranspose(), MultTranspose(), and AbsMultTranspose().
+  Added a method with the same name to class HypreParMatrix which is also called
+  automatically by the HypreParMatrix::MultTranspose() methods.

+- Updated various MemoryUsage methods to return 'std::size_t' instead of 'long'
+  since the latter is 32-bit in Win64 builds.

- Added boundary elimination with device support for `SparseMatrix` and
-  `HypreParMatrix`.
-  
 - When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).

- Added TMOP metrics for mesh untangling and worst-case quality improvement.
+- In various places in the library, replace the use of 'long' with 'long long'
+  to better support Win64 builds where 'long' is 32-bit and 'long long' is
+  64-bit. On Linux and MacOS, both types are typically 64-bit.
+
+- The behavior of GridFunction::GetTrueVector() has been changed to not return
+  an empty true vector.
+
+- Added support for ordering search points byVDIM in FindPointsGSLIB.
+
+- Various other simplifications, extensions, and bugfixes in the code.
+

 Version 4.4, released on March 21, 2022
 =======================================
@@ -119,11 +180,6 @@ Meshing improvements
 - Added a simpler interface to access mesh face information, see FaceInformation
  and GetFaceInformation in the Mesh class.

- Added the method ParMesh::GetSerialMesh() that reconstructs a partitioned
-  parallel mesh on a given single rank. Also, added the method
-  ParMesh::PrintAsSerial() that saves the reconstructed serial mesh to a C++
-  stream on rank 0.
-
 - Gmsh meshes where all elements have zero physical tag (the default Gmsh output
  format if no physical groups are defined) are now successfully loaded, and
  elements are reassigned attribute number 1.
@@ -213,9 +269,6 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
  formatting. See the "make style" target.

- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
-
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -226,13 +279,6 @@ Miscellaneous

 - Fixed several MinGW build issues on Windows.

- In various places in the library, replace the use of 'long' with 'long long'
-  to better support Win64 builds where 'long' is 32-bit and 'long long' is
-  64-bit. On Linux and MacOS, both types are typically 64-bit.
-
- Update various "MemoryUsage" methods to return 'std::size_t' instead of 'long'
-  since the latter is 32-bit in Win64 builds.
-
 - Added 'double' atomicAdd implementation for previous versions of CUDA.

 - HypreParVector and Vector now support C++ move semantics, and the copy
@@ -51,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.4.1)
+set(${PROJECT_NAME}_VERSION 4.5.0)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -81,6 +81,10 @@ if (MFEM_USE_STRUMPACK)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
+# SUNDIALS >= 6.4.0 requires C++14:
+if (MFEM_USE_SUNDIALS AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
+  set(CMAKE_CXX_STANDARD 14)
+endif()
 if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
  set(CMAKE_CXX_STANDARD 14)
 endif()
@@ -137,7 +141,7 @@ if (MFEM_USE_CUDA)
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
   set(CUBLAS_FOUND TRUE)
-   set(CUSBLAS_LIBRARIES "cublas")
+   set(CUBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -200,10 +204,10 @@ if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
  set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
  if (MFEM_MPIEXEC)
-    set(MPIEXEC ${MFEM_MPIEXEC})
+    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
  endif()
  if (MFEM_MPIEXEC_NP)
-    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
+    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
  endif()
  # Parallel MFEM depends on hypre
  find_package(HYPRE REQUIRED)
@@ -477,12 +481,21 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
  endif()
 endif()

+# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
+# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
+#   Target "mfem" links to target "Threads::Threads" but the target was not
+#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
+#   an ALIAS target is missing?
+# Call Stack (most recent call first):
+#   CMakeLists.txt:474 (mfem_add_library)
+find_package(Threads REQUIRED)
+
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist METIS SuiteSparse SUNDIALS
    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
@@ -102,7 +102,9 @@ The MFEM source code has the following structure:
  .
  ├── config
  │   ├── cmake
-  │   └── githooks
+  │   ├── docker
+  │   ├── githooks
+  │   └── vcpkg
  ├── data
  ├── doc
  ├── examples
@@ -111,6 +113,7 @@ The MFEM source code has the following structure:
  │   ├── ginkgo
  │   ├── hiop
  │   ├── jupyter
+  │   ├── moonolith
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
@@ -118,13 +121,15 @@ The MFEM source code has the following structure:
  ├── fem
  │   ├── ceed
  │   ├── fe
-  │   ├── qinterp
+  │   ├── lor
  │   ├── moonolith
+  │   ├── qinterp
  │   └── tmop
  ├── general
  ├── linalg
  │   └── simd
  ├── mesh
+  │   └── submesh
  ├── miniapps
  │   ├── adjoint
  │   ├── autodiff
@@ -134,6 +139,7 @@ The MFEM source code has the following structure:
  │   ├── hooke
  │   ├── meshing
  │   ├── mtop
+  │   ├── multidomain
  │   ├── navier
  │   ├── nurbs
  │   ├── parelag
@@ -7,6 +7,10 @@

                               https://mfem.org

+This file provides a detailed description of how to build and install the MFEM
+library. For a simple build, see the step-by-step instructions on the website
+at https://mfem.org/building.
+
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
 of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
@@ -16,7 +20,11 @@ requires an MPI C++ compiler, as well as the following external libraries:
  https://github.com/hypre-space/hypre

 - METIS (a family of multilevel partitioning algorithms)
-  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
+  https://github.com/mfem/tpls
+
+  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
+  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
+  and we don't support yet the new repo https://github.com/KarypisLab/METIS.

 The hypre dependency can be downloaded as a tarball from GitHub or from the
 project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
@@ -472,10 +480,10 @@ MFEM_USE_CODIPACK = YES/NO
  www.scicomp.uni-kl.de/codi/

 MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical 
-  methods and C++ algorithms for working with implicitly-defined geometry and 
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
-  provides interface to Algoim v1. Thus, to check out the specific state use: 
+  Enable the usage of Algoim - a collection of high-order accurate numerical
+  methods and C++ algorithms for working with implicitly-defined geometry and
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM
+  provides interface to Algoim v1. Thus, to check out the specific state use:
  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
  https://algoim.github.io

@@ -550,7 +558,7 @@ MFEM_USE_FMS = YES/NO
   Enables support for the FMS library which consists of the DataCollection
   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   conversion routines between FMS's FmsDataCollection structure and MFEM's
   DataCollection class, see the header file fem/fmsconvert.hpp.

 MFEM_USE_PARELAG = YES/NO
@@ -597,7 +605,7 @@ The specific libraries and their options are:

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
+  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.

@@ -754,12 +762,12 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.7.

- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
+- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
  headers so it just needs to be downloaded at the same level as MFEM. Download
  the specific version we use as:
  "git clone https://github.com/algoim/algoim.git;
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
+  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
  "ln -s blitz-1.0.2 blitz".
  Build Blitz using CMake as:
@@ -19,7 +19,7 @@ if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
  # Set ENZYME_FOUND
  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)

-  # Set CXX flags to accomodate the Enzyme Clang plugin
+  # Set CXX flags to accommodate the Enzyme Clang plugin
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
  set(MFEM_USE_ENZYME YES)
 else()
@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - HDF5_FOUND - If HDF5 was found
+#   - HDF5_LIBRARIES - The HDF5 libraries
+#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
+
+# First Check for HDF5_DIR
+if(NOT HDF5_DIR)
+    MESSAGE(FATAL_ERROR "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
+endif()
+
+# Find includes
+find_path( HDF5_INCLUDE_DIRS hdf5.h
+           PATHS  ${HDF5_DIR}/include/
+           NO_DEFAULT_PATH
+           NO_CMAKE_ENVIRONMENT_PATH
+           NO_CMAKE_PATH
+           NO_SYSTEM_ENVIRONMENT_PATH
+           NO_CMAKE_SYSTEM_PATH)
+
+find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
+              PATHS ${HDF5_DIR}/lib
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
+              PATHS ${HDF5_DIR}/lib
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+
+# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
+# listed variables are TRUE
+find_package_handle_standard_args(HDF5  DEFAULT_MSG
+                                  HDF5_INCLUDE_DIRS
+                                  __HDF5_LIBRARY
+                                  __HDF5_HL_LIBRARY
+                                  HDF5_LIBRARIES )
@@ -14,6 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-include(MfemCmakeUtilities)
-mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
-  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")
+find_package(umpire REQUIRED CONFIG)
+set(UMPIRE_FOUND ${umpire_FOUND})
+set(UMPIRE_LIBRARIES "umpire")
@@ -43,22 +43,14 @@ function(convert_filenames_to_full_paths NAMES)
  set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()

-# Wrapper for add_executable that calls the HIP wrapper if applicable
+# Wrapper for add_executable
 macro(mfem_add_executable NAME)
-  if (MFEM_USE_HIP)
-    add_executable(${NAME} ${ARGN})
-  else()
-    add_executable(${NAME} ${ARGN})
-  endif()
+  add_executable(${NAME} ${ARGN})
 endmacro()

-# Wrapper for add_library that calls the HIP wrapper if applicable
+# Wrapper for add_library
 macro(mfem_add_library NAME)
-  if (MFEM_USE_HIP)
-    add_library(${NAME} ${ARGN})
-  else()
-    add_library(${NAME} ${ARGN})
-  endif()
+  add_library(${NAME} ${ARGN})
 endmacro()

 # Simple shortcut to add_custom_target() with option to add the target to the
@@ -31,9 +31,11 @@

 // Windows specific options
 #ifdef _WIN32
+#ifndef _USE_MATH_DEFINES
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
+#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().
@@ -179,7 +179,7 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
 endif

 # ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
 # the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
 ifeq ($(MFEM_USE_HIP),YES)
   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
@@ -251,12 +251,16 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
-SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
-SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
+SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
+# SUNDIALS >= 6.4.0 requires C++14:
+ifeq ($(MFEM_USE_SUNDIALS),YES)
+   BASE_FLAGS = -std=c++14
+endif
+SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
+ $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
+ -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
-
 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
@@ -309,7 +313,7 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortan library:
+# Additional Fortran library:
 # MPI_FORTRAN_LIB += -lgfortran

 # MUMPS library configuration
@@ -554,15 +554,14 @@ function go()
   local cmd_line="${1##+( )}"
   cmd_line="${cmd_line%%+( )}"
   shopt -u extglob
-   eval local cmd=(${cmd_line})
   local res=""
   echo $sep
   echo "<${group}>" "${cmd_line}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
-      timed_run "${cmd[@]}"
+      timed_run eval "${cmd_line}"
   else
-      "${cmd[@]}"
+      eval "${cmd_line}"
   fi
   if [ "$?" -eq 0 ]; then
      res="${green}  OK  ${none}"
@@ -0,0 +1,8 @@
+--- a/CMakeLists.txt	Wed Dec 21 18:24:22 2016
+++ b/CMakeLists.txt	Wed Dec 21 18:24:26 2016
+@@ -20,4 +20,4 @@
+ # Recursively look for CMakeLists.txt in subdirs.
+ add_subdirectory("include")
+ add_subdirectory("libmetis")
+-add_subdirectory("programs")
+# add_subdirectory("programs")
@@ -0,0 +1,15 @@
+--- a/CMakeLists.txt	Sat Mar 30 17:24:45 2013
+++ b/CMakeLists.txt	Wed Dec 21 18:23:43 2016
+@@ -4,11 +4,7 @@
+ set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+ 
+-if(MSVC)
+-  set(METIS_INSTALL FALSE)
+-else()
+-  set(METIS_INSTALL TRUE)
+-endif()
+set(METIS_INSTALL TRUE)
+ 
+ # Configure libmetis library.
+ if(SHARED)
@@ -0,0 +1,34 @@
+diff --git a/include/metis.h b/include/metis.h
+index dc5406a..7732437 100644
+--- a/include/metis.h
+++ b/include/metis.h
+@@ -72,10 +72,14 @@ typedef __int64 int64_t;
+ #define PRId64       "I64d"
+ #define SCNd32       "ld"
+ #define SCNd64       "I64d"
+#ifdef _WIN32
+#include <stdint.h>
+#else
+ #define INT32_MIN    ((int32_t)_I32_MIN)
+ #define INT32_MAX    _I32_MAX
+ #define INT64_MIN    ((int64_t)_I64_MIN)
+ #define INT64_MAX    _I64_MAX
+#endif
+ #else
+ #include <inttypes.h>
+ #endif
+diff --git a/GKlib/gk_arch.h b/GKlib/gk_arch.h
+index 78b1431..7258763 100644
+--- a/GKlib/gk_arch.h
+++ b/GKlib/gk_arch.h
+@@ -32,8 +32,8 @@
+ 
+ 
+ #ifdef __MSC__ 
+-  #include "ms_stdint.h"
+-  #include "ms_inttypes.h"
+  #include <stdint.h>
+  #include <inttypes.h>
+   #include "ms_stat.h"
+ #else
+ #ifndef SUNOS
@@ -0,0 +1,11 @@
+--- a/GKlib/gk_arch.h	Wed Dec 21 18:34:18 2016
+++ b/GKlib/gk_arch.h	Wed Dec 21 18:30:49 2016
+@@ -58,7 +58,7 @@
+ #define PTRDIFF_MAX  INT64_MAX
+ #endif
+ 
+-#ifdef __MSC__
+#if defined(__MSC__) && (_MSC_VER < 1900)
+ /* MSC does not have rint() function */
+ #define rint(x) ((int)((x)+0.5))  
+ 
@@ -0,0 +1,14 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index e94f050..b9613a7 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -1,7 +1,8 @@
+ cmake_minimum_required(VERSION 2.8)
+ project(METIS)
+ 
+-set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
+set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
+
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+ 
+ set(METIS_INSTALL TRUE)
@@ -0,0 +1,11 @@
+--- a/libmetis/metislib.h	Sat Mar 30 17:24:45 2013
+++ b/libmetis/metislib.h	Wed Dec 21 18:30:59 2016
+@@ -31,7 +31,7 @@
+ #include <proto.h>
+ 
+ 
+-#if defined(COMPILER_MSC)
+#if defined(COMPILER_MSC) && (_MSC_VER < 1900)
+ #if defined(rint)
+   #undef rint
+ #endif
@@ -0,0 +1,10 @@
+--- a/libmetis/CMakeLists.txt	Sat Mar 30 17:24:45 2013
+++ b/libmetis/CMakeLists.txt	Wed Dec 21 17:41:37 2016
+@@ -11,6 +11,6 @@
+ if(METIS_INSTALL)
+   install(TARGETS metis
+     LIBRARY DESTINATION lib
+-    RUNTIME DESTINATION lib
+    RUNTIME DESTINATION bin
+     ARCHIVE DESTINATION lib)
+ endif()
@@ -0,0 +1,44 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index b9613a7..e43ffee 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -22,3 +22,23 @@ include_directories(include)
+ add_subdirectory("include")
+ add_subdirectory("libmetis")
+ # add_subdirectory("programs")
+
+if(METIS_INSTALL)
+  set(PRJ_NAME metis)
+  set(PRJ_VER 5.1.0)
+  install(EXPORT metisTargets 
+    FILE ${PRJ_NAME}Targets.cmake
+    DESTINATION lib/cmake/${PRJ_NAME})
+  include(CMakePackageConfigHelpers)
+  write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
+    VERSION ${PRJ_VER}
+    COMPATIBILITY SameMajorVersion)
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
+    "include(\${CMAKE_CURRENT_LIST_DIR}/${PRJ_NAME}Targets.cmake)")
+  install(FILES 
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
+    DESTINATION lib/cmake/${PRJ_NAME})
+endif()
+
+diff --git a/libmetis/CMakeLists.txt b/libmetis/CMakeLists.txt
+index 7a5fc74..5a68cf0 100644
+--- a/libmetis/CMakeLists.txt
+++ b/libmetis/CMakeLists.txt
+@@ -9,8 +9,9 @@ if(UNIX)
+ endif()
+ 
+ if(METIS_INSTALL)
+-  install(TARGETS metis
+  install(TARGETS metis EXPORT metisTargets
+     LIBRARY DESTINATION lib
+     RUNTIME DESTINATION bin
+-    ARCHIVE DESTINATION lib)
+    ARCHIVE DESTINATION lib
+    INCLUDES DESTINATION include)
+ endif()
@@ -0,0 +1,41 @@
+vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
+set(OPTIONS -DSHARED=OFF)
+
+set(METIS_VERSION 5.1.0)
+
+vcpkg_download_distfile(ARCHIVE
+    URLS "https://github.com/mfem/tpls/raw/gh-pages/metis-${METIS_VERSION}.tar.gz"
+    FILENAME "metis-${METIS_VERSION}.tar.gz"
+    SHA512 deea47749d13bd06fbeaf98a53c6c0b61603ddc17a43dae81d72c8015576f6495fd83c11b0ef68d024879ed5415c14ebdbd87ce49c181bdac680573bea8bdb25
+)
+
+vcpkg_extract_source_archive_ex(
+    OUT_SOURCE_PATH SOURCE_PATH
+    ARCHIVE ${ARCHIVE}
+    REF ${METIS_VERSION}
+    PATCHES
+        enable-install.patch
+        disable-programs.patch
+        fix-runtime-install-destination.patch
+        fix-metis-vs14-math.patch
+        fix-gklib-vs14-math.patch
+        fix-linux-build-error.patch
+        install-metisConfig.patch
+        fix-INT_MIN_define.patch
+)
+
+vcpkg_configure_cmake(
+    SOURCE_PATH ${SOURCE_PATH}
+    PREFER_NINJA
+    OPTIONS ${OPTIONS}
+)
+
+vcpkg_install_cmake()
+vcpkg_copy_pdbs()
+vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/metis)
+
+file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
+
+# Handle copyright
+file(COPY ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/metis)
+file(INSTALL ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
@@ -0,0 +1,7 @@
+{
+  "name": "metis-mfem",
+  "version-string": "5.1.0",
+  "port-version": 0,
+  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
+  "homepage": "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
+}
@@ -1,7 +1,7 @@
 MFEM mesh v1.0

 #
-# MFEM Geomety Types (see mesh/geom.hpp):
+# MFEM Geometry Types (see mesh/geom.hpp):
 #
 # POINT       = 0
 # SEGMENT     = 1
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.4.1
+PROJECT_NUMBER         = v4.5.0

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -2349,7 +2349,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.

-DOT_GRAPH_MAX_NODES    = 50
+DOT_GRAPH_MAX_NODES    = 100

 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the
@@ -1,310 +0,0 @@
-// Example run: ./FOSLS2D_maxwell -ref 4  -o 3 -sol 1 -k 3.0
-
-//  ∇ × E - ω H = 0
-// -ω E + ∇ × H = J
-
-// --------------------------------------------------------------------------
-// |   |            E             |             H              |     RHS    | 
-// --------------------------------------------------------------------------
-// | F | (∇ × E,∇ × F)+ ω^2 (E,F) | - ω (∇ × H,F) - ω (H,curF) | - ω (J,F)  |
-// |   |                          |                            |            |
-// | G |-ω (E,∇ × G)-ω (∇ × E,G)  | (∇ × H,∇ × G)+ ω^2(H,G)    | (J,∇ × G)  |
-
-// for E in H1 (scalar) we have ∇ × E = [0 1;-1 0] ∇ E
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Define exact solution
-double E_exact(const Vector &x);
-void H_exact(const Vector &x, Vector &H);
-double frhs(const Vector &x);
-void fvrhs(const Vector &x, Vector &f);
-void get_maxwell_solution(const Vector &x, double & E, Vector & curlE, double & curl2E);
-
-int dim;
-double omega;
-int isol = 0;
-
-int main(int argc, char *argv[])
-{
-   StopWatch chrono;
-
-   // 1. Parse command-line options.
-   // geometry file
-   const char *mesh_file = "../data/star.mesh";
-   // finite element order of approximation
-   int order = 1;
-   // visualization flag
-   bool visualization = 1;
-   int ref = 1;
-   // number of wavelengths
-   double k = 0.6;
-   // optional command line inputs
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&ref, "-ref", "--init-refinements",
-                  "Number of initial mesh refinements");
-   args.AddOption(&k, "-k", "--wavelengths",
-                  "Number of wavelengths.");
-   args.AddOption(&isol, "-sol", "--solution",
-                  "Exact Solution: 0) Polynomial, 1) Sinusoidal.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * k;
-
-   // Mesh mesh(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0, false);
-   Mesh mesh(mesh_file, 1, 1);
-
-   dim = mesh.Dimension();
-   if (dim == 3) {MFEM_ABORT("This is 2D Maxwell")};
-
-   for (int i = 0; i < ref; i++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   H1_FECollection H1fec(order,dim);
-   FiniteElementSpace H1fes(&mesh, &H1fec);
-
-   ND_FECollection NDfec(order, dim);
-   FiniteElementSpace NDfes(&mesh, &NDfec);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      // Essential BC on E. Nothing on H
-      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-   Array<int> block_offsets(3);
-   block_offsets[0] = 0;
-   block_offsets[1] = H1fes.GetVSize();
-   block_offsets[2] = NDfes.GetVSize();
-   block_offsets.PartialSum();
-
-   BlockVector x(block_offsets), b(block_offsets);
-   x = 0.0;
-   b = 0.0;
-
-   FunctionCoefficient Eex(E_exact);
-   VectorFunctionCoefficient Hex(dim, H_exact);
-   GridFunction E_gf;
-   GridFunction H_gf;
-   E_gf.MakeRef(&H1fes, x.GetBlock(0));
-   E_gf.ProjectBdrCoefficient(Eex,ess_bdr);
-   H_gf.MakeRef(&NDfes, x.GetBlock(1));
-
-   FunctionCoefficient f(frhs);
-   ProductCoefficient f_E(-omega, f);
-   VectorFunctionCoefficient f_H(1,fvrhs);
-   LinearForm b_E;
-   b_E.Update(&H1fes, b.GetBlock(0), 0);
-   b_E.AddDomainIntegrator(new DomainLFIntegrator(f_E));
-   b_E.Assemble();
-
-   LinearForm b_H;
-   b_H.Update(&NDfes, b.GetBlock(1), 0);
-   b_H.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
-   b_H.Assemble();
-
-   // 7. Bilinear form a(.,.) on the finite element space
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient omeg2(pow(omega, 2));
-   ConstantCoefficient negomega(-(omega));
-   DenseMatrix mat(2);
-   mat(0,0) = 0.; mat(0,1) = 1.;
-   mat(1,0) = -1.; mat(1,1) = 0.;
-   MatrixConstantCoefficient rot(mat);
-
-   BilinearForm a_EE(&H1fes);
-   a_EE.AddDomainIntegrator(new DiffusionIntegrator(one));
-   a_EE.AddDomainIntegrator(new MassIntegrator(omeg2));
-   a_EE.Assemble();
-   a_EE.EliminateEssentialBC(ess_bdr, x.GetBlock(0), b.GetBlock(0));
-   a_EE.Finalize();
-   SparseMatrix &A_EE = a_EE.SpMat();
-
-   ScalarMatrixProductCoefficient c1(-omega, rot);
-   MixedBilinearForm a_EH(&H1fes,&NDfes);
-   // - omega (rot grad E, G) - (omega E, curl G)
-   a_EH.AddDomainIntegrator(new MixedVectorGradientIntegrator(c1));
-   a_EH.AddDomainIntegrator(new MixedScalarWeakCurlIntegrator(negomega));
-   a_EH.Assemble();
-   a_EH.EliminateTrialDofs(ess_bdr, x.GetBlock(0), b.GetBlock(1));
-   a_EH.Finalize();
-   SparseMatrix &A_EH = a_EH.SpMat();
-   SparseMatrix * A_HE = Transpose(A_EH);
-
-   BilinearForm a_HH(&NDfes);
-   a_HH.AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
-   a_HH.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2)); // one is the coeff
-   a_HH.Assemble();
-   a_HH.Finalize();
-   SparseMatrix &A_HH = a_HH.SpMat();
-
-   BlockMatrix LS_Maxwellop(block_offsets);
-   LS_Maxwellop.SetBlock(0, 0, &A_EE);
-   LS_Maxwellop.SetBlock(0, 1, A_HE);
-   LS_Maxwellop.SetBlock(1, 0, &A_EH);
-   LS_Maxwellop.SetBlock(1, 1, &A_HH);
-
-   UMFPackSolver invE;
-   invE.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   invE.SetOperator(LS_Maxwellop.GetBlock(0,0));
-
-   UMFPackSolver invH;
-   invH.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
-   invH.SetOperator(LS_Maxwellop.GetBlock(1,1));
-
-   BlockDiagonalPreconditioner prec(block_offsets);
-   prec.SetDiagonalBlock(0, &invE);
-   prec.SetDiagonalBlock(1, &invH);
-
-   int maxit(5000);
-   double rtol(1.e-16);
-   double atol(0.0);
-
-   CGSolver pcg;
-   pcg.SetAbsTol(atol);
-   pcg.SetRelTol(rtol);
-   pcg.SetMaxIter(maxit);
-   pcg.SetOperator(LS_Maxwellop);
-   pcg.SetPreconditioner(prec);
-   pcg.SetPrintLevel(3);
-   pcg.Mult(b, x);
-
-   int order_quad = max(2, 2 * order + 1);
-   const IntegrationRule *irs[Geometry::NumGeom];
-   for (int i = 0; i < Geometry::NumGeom; ++i)
-   {
-      irs[i] = &(IntRules.Get(i, order_quad));
-   }
-
-   double Error_E = E_gf.ComputeL2Error(Eex, irs);
-   double Error_H = H_gf.ComputeL2Error(Hex, irs);
-   
-   cout << "|| E_h - E || = " << Error_E << "\n";
-   cout << "|| H_h - H || = " << Error_H << "\n";
-   cout << "Total error = " << sqrt(Error_H*Error_H+Error_E*Error_E) << "\n";
-
-   GridFunction E_exgf(&H1fes);
-   E_exgf.ProjectCoefficient(Eex);
-
-   GridFunction H_exgf(&NDfes);
-   H_exgf.ProjectCoefficient(Hex);
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int visport = 19916;
-      socketstream sol_sock(vishost, visport);
-      sol_sock.precision(8);
-      socketstream ex_sock(vishost, visport);
-      ex_sock.precision(8);
-      socketstream sol_sockH(vishost, visport);
-      sol_sockH.precision(8);
-      socketstream ex_sockH(vishost, visport);
-      ex_sockH.precision(8);
-      sol_sock << "solution\n"
-               << mesh << E_gf << "window_title 'Numerical E'" << "keys rRljc\n"
-               << flush;
-      ex_sock << "solution\n"
-               << mesh << E_exgf << "window_title 'Exact E'" << "keys rRljc\n"
-               << flush;
-      sol_sockH << "solution\n"
-               << mesh << H_gf << "window_title 'Numerical H'" << "keys rRljc\n"
-               << flush;
-      ex_sockH << "solution\n"
-               << mesh << H_exgf << "window_title 'Exact H'" << "keys rRljc\n"
-               << flush;        
-   }
-   delete A_HE;
-   return 0;
-}
-
-double E_exact(const Vector &x)
-{
-   double E, curl2E;
-   Vector curlE(2);
-   get_maxwell_solution(x, E, curlE, curl2E);
-   return E;  //Scalar
-}
-
-//define exact solution
-void H_exact(const Vector &x, Vector &H)
-{
-   double E, curl2E;
-   Vector curlE(2);
-   get_maxwell_solution(x, E, curlE, curl2E);
-   H[0] = curlE[0]/omega;
-   H[1] = curlE[1]/omega;
-}
-
-double frhs(const Vector &x)
-{
-   double E, curl2E;
-   Vector curlE(2);
-   get_maxwell_solution(x, E, curlE, curl2E);
-
-   // - omega E + curl H = f
-   // - omega E + curl (curl E) / omega = f
-   double f = - omega * E + curl2E / omega;
-   return f;
-}
-
-void fvrhs(const Vector &x, Vector &f)
-{
-   double E, curl2E;
-   Vector curlE(2);
-   get_maxwell_solution(x, E, curlE, curl2E);
-   f[0] = - omega * E + curl2E / omega;
-}
-
-
-void get_maxwell_solution(const Vector &X, double & E, Vector & curlE, double & curl2E)
-{
-   double x = X[0];
-   double y = X[1];
-   double Ex, Ey, Exx, Eyy;
-   if (isol == 0) // polynomial
-   {
-      E = x * (1.0 - x) * y * (1.0 - y);
-      Ex = (1.0 - 2.0 * x) * y * (1.0 - y);
-      Ey = x * (1.0 - x) * (1.0 - 2.0 * y);
-
-      Exx = -2.0 * y * (1.0 - y);
-      Eyy = -2.0 * x * (1.0 - x);
-   }
-   else 
-   {
-      double s = omega * (y+x);
-      E = cos(s);
-      Ex = -omega * sin(s);  
-      Ey = Ex;
-      Exx = - omega * omega * E;  
-      Eyy = Exx;  
-   }
-   curlE[0] = Ey;
-   curlE[1] = -Ex;
-   curl2E = -Exx - Eyy;
-}
@@ -1,61 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/EM-diffusion,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = primal_dpg
-	 				
-PAR_EXAMPLES = 
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-	rm -rf ParaView
-
-clean-exec:
@@ -1,201 +0,0 @@
-//                     MFEM primal_dpg example
-//
-// Compile with: make primal_dpg
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-void E_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-int dim;
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-   int ref = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
-                  " solution.");   
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");
-   args.AddOption(&ref, "-ref", "--refinements",
-                  "Number of refinements.");                         
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   kappa = freq * M_PI;
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-
-   for (int i = 0; i<ref; i++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   dim = mesh.Dimension();
-   int sdim = mesh.SpaceDimension();
-   // 3. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   ND_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace NDfes(&mesh, &fec);
-
-   FiniteElementCollection * trace_fec = nullptr;
-   if (dim == 3)
-   {
-      trace_fec = new ND_Trace_FECollection(order,mesh.Dimension());
-   }
-   else
-   {
-      trace_fec = new H1_Trace_FECollection(order,mesh.Dimension());
-   }
-   FiniteElementSpace trace_fes(&mesh, trace_fec);
-
-   int test_order = order+1;
-
-   ND_FECollection test_fec(test_order,mesh.Dimension());
-
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fecs; 
-
-   trial_fes.Append(&NDfes);
-   trial_fes.Append(&trace_fes);
-   test_fecs.Append(&test_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
-
-
-   ConstantCoefficient one(1.0);
-   a->AddTrialIntegrator(new CurlCurlIntegrator(one),0,0);
-   a->AddTrialIntegrator(new VectorFEMassIntegrator(one),0,0);
-   a->AddTrialIntegrator(new TangentTraceIntegrator,1,0);
-   
-   a->AddTestIntegrator(new CurlCurlIntegrator(one),0,0);
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-
-   VectorFunctionCoefficient f(sdim, f_exact);
-   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
-
-
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      NDfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-
-   Vector X,B;
-   OperatorPtr Ah;
-
-   VectorFunctionCoefficient E(sdim, E_exact);
-
-   Array<int> offsets(3);
-   offsets[0] = 0;
-   offsets[1] = NDfes.GetVSize();
-   offsets[2] = trace_fes.GetVSize();
-   offsets.PartialSum();
-
-   BlockVector x(offsets);
-   x = 0.;
-
-
-   GridFunction E_gf(&NDfes);
-   E_gf.MakeRef(&NDfes,x.GetBlock(0));
-   E_gf.ProjectBdrCoefficientTangent(E,ess_bdr);
-   E_gf.ProjectCoefficient(E);
-
-
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-
-   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
-
-
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   GMRESSolver cg;
-   cg.SetRelTol(1e-8);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete M;
-
-   a->RecoverFEMSolution(X,x);
-
-   E_gf.MakeRef(&NDfes,x.GetData());
-
-   double L2Error = E_gf.ComputeL2Error(E);
-   mfem::out << "L2_error = " << L2Error << endl;
-
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream solu_sock(vishost, visport);
-   solu_sock.precision(8);
-   solu_sock << "solution\n" << mesh << E_gf <<
-             "window_title 'Numerical u' "
-             << flush;
-   delete trace_fec;
-   return 0;
-}
-
-
-void E_exact(const Vector &x, Vector &E)
-{
-   if (dim == 3)
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(2));
-      E(2) = sin(kappa * x(0));
-   }
-   else
-   {
-      E(0) = sin(kappa * x(1));
-      E(1) = sin(kappa * x(0));
-      if (x.Size() == 3) { E(2) = 0.0; }
-   }
-}
-
-void f_exact(const Vector &x, Vector &f)
-{
-   if (dim == 3)
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
-      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
-   }
-   else
-   {
-      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
-      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
-      if (x.Size() == 3) { f(2) = 0.0; }
-   }
-}
@@ -1,677 +0,0 @@
-//                   MFEM Ultraweak DPG acoustics example
-//
-// Compile with: make uw_dpg
-//
-
-//     - Δ p - ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//  ∇ p + i ω u = 0, in Ω
-//  ∇⋅u + i ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/(i ω) 
-
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) + i ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-// -(u , ∇ q) + i ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂  = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | i ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | i ω (p,q) |-(u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-void acoustics_solution(const Vector & X, complex<double> & p, 
-                        vector<complex<double>> &dp, complex<double> & d2p);
-
-void acoustics_solution_r(const Vector & X, double & p, 
-                          Vector &dp, double & d2p);      
-
-void acoustics_solution_i(const Vector & X, double & p, 
-                          Vector &dp, double & d2p);                                                
-
-double p_exact_r(const Vector &x);
-double p_exact_i(const Vector &x);
-void u_exact_r(const Vector &x, Vector & u);
-void u_exact_i(const Vector &x, Vector & u);
-double rhs_func_r(const Vector &x);
-double rhs_func_i(const Vector &x);
-void gradp_exact_r(const Vector &x, Vector &gradu);
-void gradp_exact_i(const Vector &x, Vector &gradu);
-double divu_exact_r(const Vector &x);
-double divu_exact_i(const Vector &x);
-double d2_exact_r(const Vector &x);
-double d2_exact_i(const Vector &x);
-double hatp_exact_r(const Vector & X);
-double hatp_exact_i(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-void hatu_exact_r(const Vector & X, Vector & hatu);
-void hatu_exact_i(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-enum prob_type
-{
-   plane_wave,
-   gaussian_beam  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   bool static_cond = false;
-   int iprob = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");    
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: plane wave, 1: Gaussian beam");                     
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");       
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                                            
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 1) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-   omega = 2.*M_PI*rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   mfem::out << "p_fes space true dofs = " << p_fes->GetTrueVSize() << endl;
-   mfem::out << "u_fes space true dofs = " << u_fes->GetTrueVSize() << endl;
-   mfem::out << "hatp_fes space true dofs = " << hatp_fes->GetTrueVSize() << endl;
-   mfem::out << "hatu_fes space true dofs = " << hatu_fes->GetTrueVSize() << endl;
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   ComplexNormalEquations * a = new ComplexNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices();
-
-   // i ω (p,q)
-   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omeg),0,0);
-
-// -(u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(negone)),nullptr,1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),nullptr,0,1);
-
-//  i ω (u,v)
-   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(omeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,nullptr,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,nullptr,3,0);
-
-   // for impedence condition (only on the boundary)
-   // TODO
-   // a->AddTrialIntegrator(new TraceIntegrator,nullptr,2,0);
-
-
-// test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),nullptr,1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -i ω (∇q,δv)
-      a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negomeg),0,1);
-      // i ω (v,∇ δq)
-      a->AddTestIntegrator(nullptr,new MixedVectorWeakDivergenceIntegrator(negomeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),nullptr,1,1);
-
-      // - i ω (∇⋅v,δq)   
-      a->AddTestIntegrator(nullptr,new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // i ω (q,∇⋅v)   
-      a->AddTestIntegrator(nullptr,new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),nullptr,0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f_rhs_r(rhs_func_r);
-   FunctionCoefficient f_rhs_i(rhs_func_i);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),new DomainLFIntegrator(f_rhs_i),0);
-   
-   
-   FunctionCoefficient hatpex_r(hatp_exact_r);
-   FunctionCoefficient hatpex_i(hatp_exact_i);
-   VectorFunctionCoefficient hatuex_r(dim,hatu_exact_r);
-   VectorFunctionCoefficient hatuex_i(dim,hatu_exact_i);
-   Array<int> elements_to_refine;
-
-   socketstream p_out_r;
-   socketstream p_out_i;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out_r.open(vishost, visport);
-      p_out_i.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         // ess_bdr[1] = 0;
-         // ess_bdr[2] = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-         // hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int j = 0; j < ess_tdof_list.Size(); j++)
-      {
-         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-                        //   + hatp_fes->GetTrueVSize(); 
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-
-      Vector x(2*offsets.Last());
-      x = 0.;
-      double * xdata = x.GetData();
-
-      ComplexGridFunction hatp_gf(hatp_fes);
-      hatp_gf.real().MakeRef(hatp_fes,&xdata[offsets[2]]);
-      hatp_gf.imag().MakeRef(hatp_fes,&xdata[offsets.Last()+ offsets[2]]);
-      hatp_gf.ProjectBdrCoefficient(hatpex_r,hatpex_i, ess_bdr);
-
-      // ComplexGridFunction hatu_gf(hatu_fes);
-      // hatu_gf.real().MakeRef(hatu_fes,&xdata[offsets[3]]);
-      // hatu_gf.imag().MakeRef(hatu_fes,&xdata[offsets.Last()+ offsets[3]]);
-      // hatu_gf.ProjectBdrCoefficientNormal(hatuex_r,hatuex_i, ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
-
-      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
-
-      SparseMatrix * Ar = dynamic_cast<BlockMatrix *>(&Ahc->real())->CreateMonolithic();
-      SparseMatrix * Ai = dynamic_cast<BlockMatrix *>(&Ahc->imag())->CreateMonolithic();
-
-      ComplexSparseMatrix Ac(Ar,Ai,true,true);
-      SparseMatrix * A = Ac.GetSystemMatrix();
-
-      mfem::out << "Size of the linear system: " << A->Height() << std::endl;
-
-
-      UMFPackSolver umf(*A);
-      umf.Mult(B,X);
-
-      delete A;
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-      double residual = residuals.Norml2();
-
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-
-      ComplexGridFunction p(p_fes);
-      p.real().MakeRef(p_fes,x.GetData());
-      p.imag().MakeRef(p_fes,&x.GetData()[offsets.Last()]);
-
-      ComplexGridFunction pgf_ex(p_fes);
-      FunctionCoefficient p_ex_r(p_exact_r);
-      FunctionCoefficient p_ex_i(p_exact_i);
-      pgf_ex.ProjectCoefficient(p_ex_r, p_ex_i);
-
-      int dofs = X.Size()/2;
-
-      double p_err_r = p.real().ComputeL2Error(p_ex_r);
-      double p_err_i = p.imag().ComputeL2Error(p_ex_i);
-
-      double L2Error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = residual;
-      dof0 = dofs;
-
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  0.0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::setw(10) << std::scientific 
-                << std::endl;
-
-      if (visualization)
-      {
-         p_out_r.precision(8);
-         p_out_r << "solution\n" << mesh << p.real() <<
-                  "window_title 'Real Numerical presure' "
-                  << flush;
-
-         p_out_i.precision(8);
-         p_out_i << "solution\n" << mesh << p.imag() <<
-                  "window_title 'Imag Numerical presure' "
-                  << flush;         
-      }
-
-      if (i == ref)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double p_exact_r(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_r(x,p,dp,d2p);
-   return p;
-}
-
-double p_exact_i(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_i(x,p,dp,d2p);
-   return p;
-}
-
-double hatp_exact_r(const Vector & X)
-{
-   return p_exact_r(X);
-}
-
-double hatp_exact_i(const Vector & X)
-{
-   return p_exact_i(X);
-}
-
-void gradp_exact_r(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   double p,d2p;
-   acoustics_solution_r(x,p,grad,d2p);
-}
-
-void gradp_exact_i(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   double p,d2p;
-   acoustics_solution_i(x,p,grad,d2p);
-}
-
-double d2_exact_r(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_r(x,p,dp,d2p);
-   return d2p;
-}
-
-double d2_exact_i(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_i(x,p,dp,d2p);
-   return d2p;
-}
-
-//  u = - ∇ p / (i ω )
-//    = i (∇ p_r + i * ∇ p_i)  / ω 
-//    = - ∇ p_i / ω + i ∇ p_r / ω 
-void u_exact_r(const Vector &x, Vector & u)
-{
-   gradp_exact_i(x,u);
-   u *= -1./omega;
-}
-
-void u_exact_i(const Vector &x, Vector & u)
-{
-   gradp_exact_r(x,u);
-   u *= 1./omega;
-}
-
-void hatu_exact_r(const Vector & X, Vector & hatu)
-{
-   u_exact_r(X,hatu);
-}
-void hatu_exact_i(const Vector & X, Vector & hatu)
-{
-   u_exact_i(X,hatu);
-}
-
-//  ∇⋅u = i Δ p / ω
-//      = i (Δ p_r + i * Δ p_i)  / ω 
-//      = - Δ p_i / ω + i Δ p_r / ω 
-
-double divu_exact_r(const Vector &x)
-{
-   return -d2_exact_i(x)/omega;
-}
-
-double divu_exact_i(const Vector &x)
-{
-   return d2_exact_r(x)/omega;
-}
-
-// f = ∇⋅u + i ω p 
-// f_r = ∇⋅u_r - ω p_i  
-double rhs_func_r(const Vector &x)
-{
-   double p = p_exact_i(x);
-   double divu = divu_exact_r(x);
-   return divu - omega * p;
-}
-
-// f_i = ∇⋅u_i + ω p_r
-double rhs_func_i(const Vector &x)
-{
-   double p = p_exact_r(x);
-   double divu = divu_exact_i(x);
-   return divu + omega * p;
-}
-
-
-void acoustics_solution_r(const Vector & X, double & p, 
-                          Vector &dp, double & d2p)
-{
-   complex<double> zp, d2zp;
-   vector<complex<double>> dzp;
-   acoustics_solution(X,zp,dzp,d2zp);
-   p = zp.real();
-   d2p = d2zp.real();
-   dp.SetSize(X.Size());
-   for (int i = 0; i<X.Size(); i++)
-   {
-      dp[i] = dzp[i].real();
-   }
-}                           
-
-void acoustics_solution_i(const Vector & X, double & p, 
-                          Vector &dp, double & d2p)
-{
-   complex<double> zp, d2zp;
-   vector<complex<double>> dzp;
-   acoustics_solution(X,zp,dzp,d2zp);
-   p = zp.imag();
-   d2p = d2zp.imag();
-   dp.SetSize(X.Size());
-   for (int i = 0; i<X.Size(); i++)
-   {
-      dp[i] = dzp[i].imag();
-   }
-}
-
-
-void acoustics_solution(const Vector & X, complex<double> & p, vector<complex<double>> & dp, 
-         complex<double> & d2p)
-{
-   dp.resize(X.Size());
-   complex<double> zi = complex<double>(0., 1.);
-   switch (prob)
-   {
-   case plane_wave:
-   {
-      double beta = omega/std::sqrt((double)X.Size());
-      complex<double> alpha = beta * zi * X.Sum();
-      p = exp(-alpha);
-      d2p = - dim * beta * beta * p;
-      for (int i = 0; i<X.Size(); i++)
-      {
-         dp[i] = - zi * beta * p;
-      }
-   }
-      break;
-   default:
-   {
-      double rk = omega;
-      double alpha = 45 * M_PI/180.;
-      double sina = sin(alpha); 
-      double cosa = cos(alpha);
-      // shift the origin
-      double xprim=X(0) + 0.1; 
-      double yprim=X(1) + 0.1;
-
-      double  x = xprim*sina - yprim*cosa;
-      double  y = xprim*cosa + yprim*sina;
-      double  dxdxprim = sina, dxdyprim = -cosa;
-      double  dydxprim = cosa, dydyprim =  sina;
-      //wavelength
-      double rl = 2.*M_PI/rk;
-
-      // beam waist radius
-      double w0 = 0.05;
-
-      // function w
-      double fact = rl/M_PI/(w0*w0);
-      double aux = 1. + (fact*y)*(fact*y);
-
-      double w = w0*sqrt(aux);
-      double dwdy = w0*fact*fact*y/sqrt(aux);
-      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
-
-      double phi0 = atan(fact*y);
-      double dphi0dy = cos(phi0)*cos(phi0)*fact;
-      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
-
-      double r = y + 1./y/(fact*fact);
-      double drdy = 1. - 1./(y*y)/(fact*fact);
-      double d2rdydy = 2./(y*y*y)/(fact*fact);
-
-      // pressure
-      complex<double> zi = complex<double>(0., 1.);
-      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
-
-      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
-      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
-      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
-      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
-      complex<double> zd2edydx = zd2edxdy;
-      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
-                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
-
-      double pf = pow(2.0/M_PI/(w*w),0.25);
-      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
-      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
-                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
-
-
-      complex<double> zp = pf*exp(ze);
-      complex<double> zdpdx = zp*zdedx;
-      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
-      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
-      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
-      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
-      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
-
-      p = zp;
-      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
-      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
-
-      d2p = (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
-          + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
-   }
-   break;
-   }
-
-}
@@ -1,294 +0,0 @@
-//                 MFEM FOSLS acoustics Example
-//
-// Compile with: make fosls
-//
-// Definite/Indefinite Helmholtz
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// FOSLS:
-//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
-
-// (p,u) ∈ H^1(Ω) × H(div,Ω)
-// -------------------------------------------------------------------
-// |   |            p           |            u             |   RHS    | 
-// -------------------------------------------------------------------
-// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
-// |   |                        |                          |          |
-// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
-
-// where (q,v) ∈ H^1(Ω) × H(div,Ω)
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int sr = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&sr, "-sr", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-   for (int i = 0; i < sr; i++ )
-   {
-      mesh.UniformRefinement();
-   }
-
-   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
-   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
-   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
-   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
-
-   Array<FiniteElementSpace *> fespaces(2);
-   fespaces[0] = H1fes;
-   fespaces[1] = RTfes;
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   BlockBilinearForm a(fespaces);
-   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
-   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
-   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient negomeg(-omega);
-   ConstantCoefficient omeg2(omega*omega);
-
-
-   Array2D<BilinearFormIntegrator * > blfi(2,2);
-
-   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
-   SumIntegrator * integ00 = new SumIntegrator();
-   integ00->AddIntegrator(new DiffusionIntegrator(one));
-   integ00->AddIntegrator(new MassIntegrator(omeg2));
-   blfi(0,0) = integ00;
-
-   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
-   SumIntegrator * integ01 = new SumIntegrator();
-#ifdef DEFINITE
-   // -ω (∇⋅u,q)
-   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
-#else   
-   // ω (∇⋅u,q)
-   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
-#endif
-   // - ω (u, ∇ q)
-   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
-   blfi(0,1) = integ01;
-
-   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
-   SumIntegrator * integ10 = new SumIntegrator();
-#ifdef DEFINITE
-   // - ω (p,∇⋅v)
-   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
-#else
-   // ω (p,∇⋅v)
-   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
-#endif
-   // - ω (∇ p,v)
-   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
-   blfi(1,0) = integ10;
-
-   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
-   SumIntegrator * integ11 = new SumIntegrator();
-   integ11->AddIntegrator(new DivDivIntegrator(one));
-   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
-   blfi(1,1) = integ11;
-
-
-   BlockLinearForm b(fespaces);
-   Array<LinearFormIntegrator * > lfi(2);
-   // ± ω (f,q)
-   FunctionCoefficient f_rhs(rhs_func);
-#ifdef DEFINITE  
-   ProductCoefficient w_f(omeg,f_rhs);
-#else   
-   ProductCoefficient w_f(negomeg,f_rhs);
-#endif   
-   // lfi[0] = new DomainLFIntegrator(w_f);
-   lfi[0] = new DomainLFIntegrator(w_f);
-
-
-   //  -(f,∇⋅v)
-   ProductCoefficient neg_f(negone,f_rhs);
-   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
-   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
-
-   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
-   integ->SetIntegrators(blfi);
-   a.AddDomainIntegrator(integ);
-   a.Assemble();
-
-   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
-   lininteg->SetIntegrators(lfi);
-   b.AddDomainIntegrator(lininteg);
-   b.Assemble();
-
-   int size = 0;
-   for (int i = 0; i<fespaces.Size(); i++)
-   {
-      size += fespaces[i]->GetVSize();
-   }
-
-   Vector x(size);
-   x = 0.0;
-   FunctionCoefficient p_ex(p_exact);
-   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
-   VectorFunctionCoefficient u_ex(dim,u_exact);
-   FunctionCoefficient divu_ex(divu_exact);
-   GridFunction p_gf, u_gf;   
-   GridFunction pex_gf(H1fes);   
-
-   p_gf.MakeRef(H1fes,x,0);
-   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
-   p_gf.ProjectCoefficient(p_ex);
-   pex_gf.ProjectCoefficient(p_ex);
-
-   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
-   u_gf = 0.;
-
-
-   OperatorPtr A;
-   Vector X,B;
-   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
-
-   GSSmoother M((SparseMatrix&)(*A));
-   CGSolver cg;
-   cg.SetRelTol(1e-10);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   a.RecoverFEMSolution(X,b,x);
-
-   p_gf.MakeRef(H1fes,x,0);
-   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
-
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << p_gf <<
-                "window_title 'Numerical p' "
-                << flush;
-      // socketstream sols_sock(vishost, visport);
-      // sols_sock.precision(8);
-      // sols_sock << "solution\n" << mesh << u_gf <<
-               //  "window_title 'Numerical sigma' "
-               //  << flush;
-
-      socketstream solex_sock(vishost, visport);
-      solex_sock.precision(8);
-      solex_sock << "solution\n" << mesh << pex_gf <<
-                    "window_title 'Exact p' "
-                    << flush;          
-   }
-
-   return 0;
-}
-
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = fosls uw_dpg  strong_dpg complex_uw_dpg
-PAR_EXAMPLES = uw_dpgp
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
@@ -1,837 +0,0 @@
-//                   MFEM Ultraweak DPG acoustics example
-//
-// Compile with: make pcomplex_uw_dpg
-//
-// sample runs
-// ./pcomplex_uw_dpg -o 3 -m ../../../data/inline-quad.mesh -sref 2 -pref 3 -rnum 4.1 -prob 0 -sc -graph-norm
-
-//     - Δ p - ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//  ∇ p + i ω u = 0, in Ω
-//  ∇⋅u + i ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/(i ω) 
-
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) + i ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-// -(u , ∇ q) + i ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂  = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | i ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | i ω (p,q) |-(u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-void acoustics_solution(const Vector & X, complex<double> & p, 
-                        vector<complex<double>> &dp, complex<double> & d2p);
-
-void acoustics_solution_r(const Vector & X, double & p, 
-                          Vector &dp, double & d2p);      
-
-void acoustics_solution_i(const Vector & X, double & p, 
-                          Vector &dp, double & d2p);                                                
-
-double p_exact_r(const Vector &x);
-double p_exact_i(const Vector &x);
-void u_exact_r(const Vector &x, Vector & u);
-void u_exact_i(const Vector &x, Vector & u);
-double rhs_func_r(const Vector &x);
-double rhs_func_i(const Vector &x);
-void gradp_exact_r(const Vector &x, Vector &gradu);
-void gradp_exact_i(const Vector &x, Vector &gradu);
-double divu_exact_r(const Vector &x);
-double divu_exact_i(const Vector &x);
-double d2_exact_r(const Vector &x);
-double d2_exact_i(const Vector &x);
-double hatp_exact_r(const Vector & X);
-double hatp_exact_i(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-void hatu_exact_r(const Vector & X, Vector & hatu);
-void hatu_exact_i(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-enum prob_type
-{
-   plane_wave,
-   gaussian_beam  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   bool static_cond = false;
-   int iprob = 0;
-   int sr = 0;
-   int pr = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");     
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: plane wave, 1: Gaussian beam");                    
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&sr, "-sref", "--serial_ref",
-                  "Number of parallel refinements.");                                              
-   args.AddOption(&pr, "-pref", "--parallel_ref",
-                  "Number of parallel refinements.");  
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                                            
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-
-   if (iprob > 1) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-   omega = 2.*M_PI*rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   for (int i = 0; i<sr; i++)
-   {
-      mesh.UniformRefinement();
-   }
-   dim = mesh.Dimension();
-   mesh.EnsureNCMesh();
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   // if (myid == 0)
-   // {
-   //    mfem::out << "p_fes space true dofs = " << p_fes->GetTrueVSize() << endl;
-   //    mfem::out << "u_fes space true dofs = " << u_fes->GetTrueVSize() << endl;
-   //    mfem::out << "hatp_fes space true dofs = " << hatp_fes->GetTrueVSize() << endl;
-   //    mfem::out << "hatu_fes space true dofs = " << hatu_fes->GetTrueVSize() << endl;
-   // }
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<ParFiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   ComplexParNormalEquations * a = new ComplexParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices();
-   // i ω (p,q)
-   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omeg),0,0);
-
-// -(u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(negone)),nullptr,1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),nullptr,0,1);
-
-//  i ω (u,v)
-   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(omeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,nullptr,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,nullptr,3,0);
-
-// test integrators 
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),nullptr,1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -i ω (∇q,δv)
-      a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negomeg),0,1);
-      // i ω (v,∇ δq)
-      a->AddTestIntegrator(nullptr,new MixedVectorWeakDivergenceIntegrator(negomeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),nullptr,1,1);
-
-      // - i ω (∇⋅v,δq)   
-      a->AddTestIntegrator(nullptr,new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // i ω (q,∇⋅v)   
-      a->AddTestIntegrator(nullptr,new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),nullptr,0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f_rhs_r(rhs_func_r);
-   FunctionCoefficient f_rhs_i(rhs_func_i);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),new DomainLFIntegrator(f_rhs_i),0);
-   
-
-   FunctionCoefficient hatpex_r(hatp_exact_r);
-   FunctionCoefficient hatpex_i(hatp_exact_i);
-
-   VectorFunctionCoefficient hatuex_r(dim,hatu_exact_r);
-   VectorFunctionCoefficient hatuex_i(dim,hatu_exact_i);
-   Array<int> elements_to_refine;
-
-   socketstream p_out_r;
-   socketstream p_out_i;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out_r.open(vishost, visport);
-      p_out_i.open(vishost, visport);
-   }
-
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   if (myid == 0)
-   {
-      mfem::out << "\n  Ref |" 
-                << "       Mesh       |"
-                << "    Dofs    |" 
-                << "   ω   |" 
-                << "  L2 Error  |" 
-                << " Relative % |" 
-                << "  Rate  |" 
-                << "  Residual  |" 
-                << "  Rate  |" 
-                << " PCG it |"
-                << " PCG time |"  << endl;
-      mfem::out << " --------------------"      
-                <<  "---------------------"    
-                <<  "---------------------"    
-                <<  "---------------------"    
-                <<  "---------------------"    
-                <<  "-------------------" << endl;      
-   }
-
-
-   for (int it = 0; it<pr; it++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         // ess_bdr[1] = 0;
-         // ess_bdr[2] = 0;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-         // hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int j = 0; j < ess_tdof_list.Size(); j++)
-      {
-         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-                           //   + hatp_fes->GetTrueVSize(); 
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-
-      Vector x(2*offsets.Last());
-      x = 0.;
-      double * xdata = x.GetData();
-
-      ParComplexGridFunction hatp_gf(hatp_fes);
-      hatp_gf.real().MakeRef(hatp_fes,&xdata[offsets[2]]);
-      hatp_gf.imag().MakeRef(hatp_fes,&xdata[offsets.Last()+ offsets[2]]);
-      hatp_gf.ProjectBdrCoefficient(hatpex_r,hatpex_i, ess_bdr);
-      // ParComplexGridFunction hatu_gf(hatu_fes);
-      // hatu_gf.real().MakeRef(hatu_fes,&xdata[offsets[3]]);
-      // hatu_gf.imag().MakeRef(hatu_fes,&xdata[offsets.Last()+ offsets[3]]);
-      // hatu_gf.ProjectCoefficientNormal(hatuex_r,hatuex_i, ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
-
-      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
-      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
-      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
-
-      int num_blocks = BlockA_r->NumRowBlocks();
-      Array<int> tdof_offsets(2*num_blocks+1);
-
-      tdof_offsets[0] = 0;
-      int skip = (static_cond) ? 0 : 2;
-      int k = (static_cond) ? 2 : 0;
-      for (int i=0; i<num_blocks;i++)
-      {
-         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize(); 
-         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize(); 
-      }
-      tdof_offsets.PartialSum();
-
-      BlockOperator blockA(tdof_offsets);
-      for (int i = 0; i<num_blocks; i++)
-      {
-         for (int j = 0; j<num_blocks; j++)
-         {
-            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
-            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
-            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
-         }
-      }
-
-
-      X = 0.;
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(tdof_offsets);
-
-
-      if (!static_cond)
-      {
-         HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(0,0));
-         solver_p->SetPrintLevel(0);
-         solver_p->SetSystemsOptions(dim);
-         HypreBoomerAMG * solver_u = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(1,1));
-         solver_u->SetPrintLevel(0);
-         solver_u->SetSystemsOptions(dim);
-         M->SetDiagonalBlock(0,solver_p);
-         M->SetDiagonalBlock(1,solver_u);
-         M->SetDiagonalBlock(num_blocks,solver_p);
-         M->SetDiagonalBlock(num_blocks+1,solver_u);
-      }
-
-
-      HypreBoomerAMG * solver_hatp = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(skip,skip));
-      // amg->SetCycleNumSweeps(5, 5);
-      solver_hatp->SetPrintLevel(0);
-      
-      HypreSolver * solver_hatu = nullptr;
-      if (dim == 2)
-      {
-         solver_hatu = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),hatu_fes);
-         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
-      }
-      else
-      {
-         solver_hatu = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1), hatu_fes);
-         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
-      }
-      
-
-      M->SetDiagonalBlock(skip,solver_hatp);
-      M->SetDiagonalBlock(skip+1,solver_hatu);
-      M->SetDiagonalBlock(skip+num_blocks,solver_hatp);
-      M->SetDiagonalBlock(skip+num_blocks+1,solver_hatu);
-
-      StopWatch chrono;
-      
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-7);
-      cg.SetAbsTol(1e-7);
-      cg.SetMaxIter(10000);
-      cg.SetPrintLevel(0);
-      cg.SetPreconditioner(*M); 
-      cg.SetOperator(blockA);
-      chrono.Clear();
-      chrono.Start();
-      cg.Mult(B, X);
-      chrono.Stop();
-      delete M;
-
-      int ne = pmesh.GetNE();
-      MPI_Allreduce(MPI_IN_PLACE,&ne,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
-      int ne_x = (dim == 2) ? (int)sqrt(ne) : (int)cbrt(ne);
-      ostringstream oss;
-      double pcg_time = chrono.RealTime();
-      if (myid == 0)
-      {
-         if (dim == 2)
-         {
-            oss << ne_x << " x " << ne_x ;
-         }
-         else
-         {
-            oss << ne_x << " x " << ne_x << " x " << ne_x ;
-         }
-      }
-
-      int num_iter = cg.GetNumIterations();
-
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      double maxresidual = residuals.Max(); 
-      double globalresidual = residual * residual; 
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      globalresidual = sqrt(globalresidual);
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-      ParComplexGridFunction p(p_fes);
-      p.real().MakeRef(p_fes,x.GetData());
-      p.imag().MakeRef(p_fes,&x.GetData()[offsets.Last()]);
-
-      ParComplexGridFunction u(u_fes);
-      u.real().MakeRef(u_fes,&x.GetData()[offsets[1]]);
-      u.imag().MakeRef(u_fes,&x.GetData()[offsets.Last()+offsets[1]]);
-
-
-      // Error in pressure 
-      ParComplexGridFunction pgf_ex(p_fes);
-      FunctionCoefficient p_ex_r(p_exact_r);
-      FunctionCoefficient p_ex_i(p_exact_i);
-      pgf_ex.ProjectCoefficient(p_ex_r, p_ex_i);
-
-      double p_err_r = p.real().ComputeL2Error(p_ex_r);
-      double p_err_i = p.imag().ComputeL2Error(p_ex_i);
-      double p_error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
-      double p_norm_r = pgf_ex.real().ComputeL2Error(zero);
-      double p_norm_i = pgf_ex.imag().ComputeL2Error(zero);
-      double p_norm = sqrt(p_norm_r*p_norm_r + p_norm_i*p_norm_i);
-
-      // Error in velocity
-      ParComplexGridFunction ugf_ex(u_fes);
-      VectorFunctionCoefficient u_ex_r(dim,u_exact_r);
-      VectorFunctionCoefficient u_ex_i(dim,u_exact_i);
-
-      double u_err_r = u.real().ComputeL2Error(u_ex_r);
-      double u_err_i = u.imag().ComputeL2Error(u_ex_i);
-      double u_error = sqrt(u_err_r*u_err_r + u_err_i*u_err_i);
-      double u_norm_r = pgf_ex.real().ComputeL2Error(vzero);
-      double u_norm_i = pgf_ex.imag().ComputeL2Error(vzero);
-      double u_norm = sqrt(u_norm_r*u_norm_r + u_norm_i*u_norm_i);
-
-
-      double L2Error = sqrt(p_error*p_error + u_error*u_error);
-      double L2norm = sqrt(p_norm*p_norm + u_norm*u_norm);
-
-      double rel_err = L2Error/L2norm;
-
-      int dofs = p_fes->GlobalTrueVSize()
-               + u_fes->GlobalTrueVSize()
-               + hatp_fes->GlobalTrueVSize()
-               + hatu_fes->GlobalTrueVSize();
-
-
-      double rate_err = (it) ? dim*log(err0/rel_err)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (it) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = rel_err;
-      res0 = globalresidual;
-      dof0 = dofs;
-
-      if (myid == 0)
-      {
-         mfem::out << std::right << std::setw(5) << it << " | " 
-                  << std::setw(16) << oss.str() << " | " 
-                  << std::setw(10) <<  dof0 << " | " 
-                  << std::setprecision(0) << std::fixed
-                  << std::setw(2) <<  2*rnum << " π  | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::scientific <<  err0 << " | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::fixed <<  rel_err * 100. << " | " 
-                  << std::setprecision(2) 
-                  << std::setw(6) << std::fixed << rate_err << " | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::scientific <<  res0 << " | " 
-                  << std::setprecision(2) 
-                  << std::setw(6) << std::fixed << rate_res << " | " 
-                  << std::setw(6) << std::fixed << num_iter << " | " 
-                  << std::setprecision(5) 
-                  << std::setw(8) << std::fixed << pcg_time << " | " 
-                  << std::scientific 
-                  << std::endl;
-      }   
-
-      if (visualization)
-      {
-         p_out_r << "parallel " << num_procs << " " << myid << "\n";
-         p_out_r.precision(8);
-         p_out_r << "solution\n" << pmesh << p.real() <<
-                  "window_title 'Real Numerical presure' "
-                  << flush;
-
-         p_out_i << "parallel " << num_procs << " " << myid << "\n";
-         p_out_i.precision(8);
-         p_out_i << "solution\n" << pmesh << p.imag() <<
-                  "window_title 'Imag Numerical presure' "
-                  << flush;         
-      }
-
-      if (it == pr)
-         break;
-
-      pmesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();   
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double p_exact_r(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_r(x,p,dp,d2p);
-   return p;
-}
-
-double p_exact_i(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_i(x,p,dp,d2p);
-   return p;
-}
-
-double hatp_exact_r(const Vector & X)
-{
-   return p_exact_r(X);
-}
-
-double hatp_exact_i(const Vector & X)
-{
-   return p_exact_i(X);
-}
-
-void gradp_exact_r(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   double p,d2p;
-   acoustics_solution_r(x,p,grad,d2p);
-}
-
-void gradp_exact_i(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   double p,d2p;
-   acoustics_solution_i(x,p,grad,d2p);
-}
-
-double d2_exact_r(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_r(x,p,dp,d2p);
-   return d2p;
-}
-
-double d2_exact_i(const Vector &x)
-{
-   double p,d2p;
-   Vector dp;
-   acoustics_solution_i(x,p,dp,d2p);
-   return d2p;
-}
-
-//  u = - ∇ p / (i ω )
-//    = i (∇ p_r + i * ∇ p_i)  / ω 
-//    = - ∇ p_i / ω + i ∇ p_r / ω 
-void u_exact_r(const Vector &x, Vector & u)
-{
-   gradp_exact_i(x,u);
-   u *= -1./omega;
-}
-
-void u_exact_i(const Vector &x, Vector & u)
-{
-   gradp_exact_r(x,u);
-   u *= 1./omega;
-}
-
-void hatu_exact_r(const Vector & X, Vector & hatu)
-{
-   u_exact_r(X,hatu);
-}
-void hatu_exact_i(const Vector & X, Vector & hatu)
-{
-   u_exact_i(X,hatu);
-}
-
-//  ∇⋅u = i Δ p / ω
-//      = i (Δ p_r + i * Δ p_i)  / ω 
-//      = - Δ p_i / ω + i Δ p_r / ω 
-
-double divu_exact_r(const Vector &x)
-{
-   return -d2_exact_i(x)/omega;
-}
-
-double divu_exact_i(const Vector &x)
-{
-   return d2_exact_r(x)/omega;
-}
-
-// f = ∇⋅u + i ω p 
-// f_r = ∇⋅u_r - ω p_i  
-double rhs_func_r(const Vector &x)
-{
-   double p = p_exact_i(x);
-   double divu = divu_exact_r(x);
-   return divu - omega * p;
-}
-
-// f_i = ∇⋅u_i + ω p_r
-double rhs_func_i(const Vector &x)
-{
-   double p = p_exact_r(x);
-   double divu = divu_exact_i(x);
-   return divu + omega * p;
-}
-
-
-void acoustics_solution_r(const Vector & X, double & p, 
-                          Vector &dp, double & d2p)
-{
-   complex<double> zp, d2zp;
-   vector<complex<double>> dzp;
-   acoustics_solution(X,zp,dzp,d2zp);
-   p = zp.real();
-   d2p = d2zp.real();
-   dp.SetSize(X.Size());
-   for (int i = 0; i<X.Size(); i++)
-   {
-      dp[i] = dzp[i].real();
-   }
-}                           
-
-void acoustics_solution_i(const Vector & X, double & p, 
-                          Vector &dp, double & d2p)
-{
-   complex<double> zp, d2zp;
-   vector<complex<double>> dzp;
-   acoustics_solution(X,zp,dzp,d2zp);
-   p = zp.imag();
-   d2p = d2zp.imag();
-   dp.SetSize(X.Size());
-   for (int i = 0; i<X.Size(); i++)
-   {
-      dp[i] = dzp[i].imag();
-   }
-}
-
-
-void acoustics_solution(const Vector & X, complex<double> & p, vector<complex<double>> & dp, 
-         complex<double> & d2p)
-{
-   dp.resize(X.Size());
-   complex<double> zi = complex<double>(0., 1.);
-   switch (prob)
-   {
-   case plane_wave:
-   {
-      double beta = omega/std::sqrt((double)X.Size());
-      complex<double> alpha = beta * zi * X.Sum();
-      p = exp(-alpha);
-      d2p = - dim * beta * beta * p;
-      for (int i = 0; i<X.Size(); i++)
-      {
-         dp[i] = - zi * beta * p;
-      }
-   }
-      break;
-   default:
-   {
-      double rk = omega;
-      double alpha = 45 * M_PI/180.;
-      double sina = sin(alpha); 
-      double cosa = cos(alpha);
-      // shift the origin
-      double xprim=X(0) + 0.1; 
-      double yprim=X(1) + 0.1;
-
-      double  x = xprim*sina - yprim*cosa;
-      double  y = xprim*cosa + yprim*sina;
-      double  dxdxprim = sina, dxdyprim = -cosa;
-      double  dydxprim = cosa, dydyprim =  sina;
-      //wavelength
-      double rl = 2.*M_PI/rk;
-
-      // beam waist radius
-      double w0 = 0.05;
-
-      // function w
-      double fact = rl/M_PI/(w0*w0);
-      double aux = 1. + (fact*y)*(fact*y);
-
-      double w = w0*sqrt(aux);
-      double dwdy = w0*fact*fact*y/sqrt(aux);
-      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
-
-      double phi0 = atan(fact*y);
-      double dphi0dy = cos(phi0)*cos(phi0)*fact;
-      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
-
-      double r = y + 1./y/(fact*fact);
-      double drdy = 1. - 1./(y*y)/(fact*fact);
-      double d2rdydy = 2./(y*y*y)/(fact*fact);
-
-      // pressure
-      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
-
-      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
-      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
-      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
-      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
-      complex<double> zd2edydx = zd2edxdy;
-      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
-                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
-
-      double pf = pow(2.0/M_PI/(w*w),0.25);
-      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
-      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
-                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
-
-
-      complex<double> zp = pf*exp(ze);
-      complex<double> zdpdx = zp*zdedx;
-      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
-      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
-      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
-      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
-      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
-
-      p = zp;
-      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
-      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
-
-      d2p = (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
-          + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
-   }
-   break;
-   }
-
-}
@@ -1,271 +0,0 @@
-//                 MFEM DPG_strong acoustics Example
-//
-// Compile with: make strong_dpg
-//
-// Definite/Indefinite Helmholtz
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// Strong DPG formulation
-// (p,u) ∈ H^1(Ω) × H(div,Ω)
-// 
-//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
-//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
-//                    p = p_0, in ∂Ω
-// 
-// ------------------------------------
-// |   |      p    |     u    |  RHS  | 
-// ------------------------------------
-// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
-// |   |           |          |       |
-// | v | (∇ p, v)  | -ω (u,v) |       |
-
-// where (q,v) ∈ L^2  × (L^2)^dim
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");                  
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   for (int i = 0; i < ref; i++ )
-   {
-      mesh.UniformRefinement();
-   }
-
-   // Define spaces
-   // H1 space for p
-   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
-   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
-
-   // H(div) for u
-   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
-   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->SetTestFECollVdim(1,dim);
-   
-   a->StoreMatrices(true);
-
-//  ± ω (p, q)
-#ifdef DEFINITE
-//  ω (p, q)
-   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
-#else
-// -ω (p, q)
-   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
-#endif
-
-// -(∇⋅u, q) 
-   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
-
-// -ω (u,v)
-   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
-
-// (∇ p, v)
-   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
-
-// (v,δv)
-   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
-    
-// (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   FunctionCoefficient p_ex(p_exact);
-   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
-   VectorFunctionCoefficient u_ex(dim,u_exact);
-   FunctionCoefficient divu_ex(divu_exact);
-   GridFunction p_gf, u_gf;   
-   GridFunction pex_gf(p_fes);   
-
-   Array<int> offsets(3);
-   offsets[0] = 0;
-   offsets[1] = p_fes->GetVSize();
-   offsets[2] = u_fes->GetVSize();
-   offsets.PartialSum();
-   BlockVector x(offsets);
-   x = 0.0;
-
-   p_gf.MakeRef(p_fes,x.GetBlock(0));
-   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
-
-   u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-   a->Assemble();
-
-   OperatorPtr Ah;
-   Vector X,B;
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = Ah.As<BlockMatrix>();
-
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete M;
-
-   a->RecoverFEMSolution(X,x);
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream p_out;
-      socketstream u_out;
-      p_out.open(vishost, visport);
-      u_out.open(vishost, visport);
-      p_out.precision(8);
-      p_out << "solution\n" << mesh << p_gf <<
-               "window_title 'Numerical p' "
-               << flush;
-
-      u_out.precision(8);
-      u_out << "solution\n" << mesh << u_gf <<
-            "window_title 'Numerical flux' "
-            << flush;
-   }
-
-   return 0;
-}
-
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
@@ -1,546 +0,0 @@
-//                   MFEM Ultraweak DPG acoustics example
-//
-// Compile with: make uw_dpg
-//
-// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂ = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := -u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-double divu_exact(const Vector &x);
-double hatp_exact(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-enum prob_type
-{
-   plane_wave,
-   gaussian_beam  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   int iprob = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");    
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: plane wave, 1: Gaussian beam");                                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 1) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-
-   // ± ω (p,q)
-#ifdef DEFINITE
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
-#else
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
-#endif   
-
-   // (u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-   // -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
-
-   // - ω (u,v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
-
-   // < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-   // < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   // test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -ω (∇q,δv)
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
-      // -ω (v,δq)
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
-
-#ifdef DEFINITE
-      // - ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // - ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
-#else
-      // ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
-      // ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-#endif
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-
-   FunctionCoefficient hatpex(hatp_exact);
-   FunctionCoefficient pex(p_exact);
-   VectorFunctionCoefficient uex(dim,u_exact);
-   Array<int> elements_to_refine;
-   GridFunction hatp_gf;
-
-
-   socketstream p_out;
-   // socketstream u_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out.open(vishost, visport);
-      // u_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-   // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
-      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      for (int i=0; i<A->NumRowBlocks(); i++)
-      {
-         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-      }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-8);
-      cg.SetMaxIter(20000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-      
-      GridFunction p_gf;
-      p_gf.MakeRef(p_fes,x.GetBlock(0));
-
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-      GridFunction pex_gf(p_fes);
-      GridFunction uex_gf(u_fes);
-      pex_gf.ProjectCoefficient(pex);
-      uex_gf.ProjectCoefficient(uex);
-
-
-      // Error
-      int dofs = X.Size();
-      double p_err = p_gf.ComputeL2Error(pex);
-      double p_norm = pex_gf.ComputeL2Error(zero);
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = uex_gf.ComputeL2Error(vzero);
-
-      double L2Error = sqrt(p_err*p_err + u_err*u_err);
-      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = residual;
-      dof0 = dofs;
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::endl;
-
-      if (visualization)
-      {
-         p_out.precision(8);
-         p_out << "solution\n" << mesh << p_gf <<
-                  "window_title 'Numerical presure' "
-                  << flush;
-
-         // u_out.precision(8);
-         // u_out << "solution\n" << mesh << u_gf <<
-         //       "window_title 'Numerical velocity' "
-         //       << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   double p, d2p;
-   Vector dp;
-   acoustics_solution(x,p,dp,d2p);
-   return p;
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   double p, d2p;
-   acoustics_solution(x,p,u,d2p);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   double p, d2p;
-   Vector dp;
-   acoustics_solution(x,p,dp,d2p);
-   return d2p/omega;
-}
-
-double hatp_exact(const Vector & X)
-{
-   return p_exact(X);
-}
-
-void hatu_exact(const Vector & X, Vector & hatu)
-{
-   u_exact(X,hatu);
-   hatu *= -1.;
-}
-
-void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
-{
-   dp.SetSize(X.Size());
-   switch (prob)
-   {
-   case plane_wave:
-   {
-      p = sin(omega*X.Sum());
-      dp = omega * cos(omega * X.Sum());
-      d2p = -dim * omega * omega * sin(omega*X.Sum());
-   }
-      break;
-   default:
-   {
-      double rk = omega;
-      double alpha = 45 * M_PI/180.;
-      double sina = sin(alpha); 
-      double cosa = cos(alpha);
-      // shift the origin
-      double xprim=X(0) + 0.1; 
-      double yprim=X(1) + 0.1;
-
-      double  x = xprim*sina - yprim*cosa;
-      double  y = xprim*cosa + yprim*sina;
-      double  dxdxprim = sina, dxdyprim = -cosa;
-      double  dydxprim = cosa, dydyprim =  sina;
-      //wavelength
-      double rl = 2.*M_PI/rk;
-
-      // beam waist radius
-      double w0 = 0.05;
-
-      // function w
-      double fact = rl/M_PI/(w0*w0);
-      double aux = 1. + (fact*y)*(fact*y);
-
-      double w = w0*sqrt(aux);
-      double dwdy = w0*fact*fact*y/sqrt(aux);
-      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
-
-      double phi0 = atan(fact*y);
-      double dphi0dy = cos(phi0)*cos(phi0)*fact;
-      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
-
-      double r = y + 1./y/(fact*fact);
-      double drdy = 1. - 1./(y*y)/(fact*fact);
-      double d2rdydy = 2./(y*y*y)/(fact*fact);
-
-      // pressure
-      complex<double> zi = complex<double>(0., 1.);
-      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
-
-      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
-      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
-      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
-      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
-      complex<double> zd2edydx = zd2edxdy;
-      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
-                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
-
-      double pf = pow(2.0/M_PI/(w*w),0.25);
-      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
-      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
-                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
-
-
-      complex<double> zp = pf*exp(ze);
-      complex<double> zdpdx = zp*zdedx;
-      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
-      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
-      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
-      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
-      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
-
-      p = zp.real();
-      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
-      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
-
-      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
-            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
-   }
-   break;
-   }
-
-}
@@ -1,525 +0,0 @@
-//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
-//
-// Compile with: make uw_dpgp
-//
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-//
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-// 
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂ = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := -u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-double hatp_exact(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int sr = 0;
-   int pr = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   bool static_cond = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");      
-   args.AddOption(&sr, "-sref", "--serial_ref",
-                  "Number of parallel refinements.");                                              
-   args.AddOption(&pr, "-pref", "--parallel_ref",
-                  "Number of parallel refinements.");  
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                    
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   omega = 2.0 * M_PI * rnum;
-
-   Mesh mesh(mesh_file, 1, 1);
-   for (int i = 0; i<sr; i++)
-   {
-      mesh.UniformRefinement();
-   }
-   dim = mesh.Dimension();
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   Array<ParFiniteElementSpace * > trial_fes; 
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   Array<FiniteElementCollection * > test_fec; 
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-
-   // Integrators
-
-   // ± ω (p,q)
-#ifdef DEFINITE
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
-#else
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
-#endif   
-
-// (u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
-
-// - ω (u,v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   // test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -ω (∇q,δv)
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
-      // -ω (v,δq)
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
-
-#ifdef DEFINITE
-      // - ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // - ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
-#else
-      // ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
-      // ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-#endif
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
-   }   
-
-   // RHS
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-
-   FunctionCoefficient hatpex(hatp_exact);
-   FunctionCoefficient pex(p_exact);
-   VectorFunctionCoefficient uex(dim,u_exact);
-   Array<int> elements_to_refine;
-   ParGridFunction hatp_gf;
-
-
-
-
-   socketstream p_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out.open(vishost, visport);
-   }
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   if (myid == 0)
-   {
-      mfem::out << "\n Refinement |" 
-               << "    Dofs    |" 
-               << "   ω   |" 
-               << "  L2 Error  |" 
-               << " Relative % |" 
-               << "  Rate  |" 
-               << "  Residual  |" 
-               << "  Rate  |" 
-               << " PCG it |" << endl;
-      mfem::out << " --------------------"      
-               <<  "---------------------"    
-               <<  "---------------------"    
-               <<  "---------------------"    
-               <<  "----------------" << endl;   
-   }
-
-
-   for (int i = 0; i<pr; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int j = 0; j < ess_tdof_list.Size(); j++)
-      {
-         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
-      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
-
-      Vector X,B;
-      OperatorPtr Ah;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-
-      int skip = 0;
-      if (!static_cond)
-      {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         // amg0->SetRelaxType(16);
-         // amg1->SetRelaxType(16);
-         M->SetDiagonalBlock(0,amg0);
-         M->SetDiagonalBlock(1,amg1);
-         skip = 2;
-      }
-
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
-      amg2->SetPrintLevel(0);
-      // amg2->SetRelaxType(16);
-      M->SetDiagonalBlock(skip,amg2);
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatu_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatu_fes);
-      }
-      M->SetDiagonalBlock(skip+1,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-7);
-      cg.SetMaxIter(20000);
-      cg.SetPrintLevel(0);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      int num_iter = cg.GetNumIterations();
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      
-      double maxresidual = residuals.Max(); 
-      double globalresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      globalresidual = sqrt(globalresidual);
-
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-
-      ParGridFunction p_gf;
-      p_gf.MakeRef(p_fes,x.GetBlock(0));
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-
-      ParGridFunction pex_gf(p_fes);
-      ParGridFunction uex_gf(u_fes);
-      pex_gf.ProjectCoefficient(pex);
-      uex_gf.ProjectCoefficient(uex);
-
-      int dofs = p_fes->GlobalTrueVSize()
-               + u_fes->GlobalTrueVSize()
-               + hatp_fes->GlobalTrueVSize()
-               + hatu_fes->GlobalTrueVSize();
-
-      double p_err = p_gf.ComputeL2Error(pex);
-      double p_norm = pex_gf.ComputeL2Error(zero);
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = uex_gf.ComputeL2Error(vzero);
-
-      double L2Error = sqrt(p_err*p_err + u_err*u_err);
-      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = globalresidual;
-      dof0 = dofs;
-
-      std::ios oldState(nullptr);
-      if (myid == 0)
-      {
-         mfem::out << std::right << std::setw(11) << i << " | " 
-                   << std::setw(10) <<  dof0 << " | " 
-                   << std::setprecision(0) << std::fixed
-                   << std::setw(2) <<  2*rnum << " π  | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::scientific <<  err0 << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                   << std::setprecision(2) 
-                   << std::setw(6) << std::fixed << rate_err << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::scientific <<  res0 << " | " 
-                   << std::setprecision(2) 
-                   << std::setw(6) << std::fixed << rate_res << " | " 
-                   << std::setw(6) << std::fixed << num_iter << " | " 
-                   << std::setprecision(5) 
-                   << std::scientific 
-                   << std::endl;
-      }    
-
-      if (visualization)
-      {
-         p_out << "parallel " << num_procs << " " << myid << "\n";
-         p_out.precision(8);
-         p_out << "solution\n" << pmesh << p_gf <<
-                  "window_title 'Numerical pressure' "
-                  << flush;
-      }
-
-      if (i == pr)
-         break;
-
-      pmesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
-
-double hatp_exact(const Vector & X)
-{
-   return p_exact(X);
-}
-
-void hatu_exact(const Vector & X, Vector & hatu)
-{
-   u_exact(X,hatu);
-   hatu *= -1.;
-}
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/convection-diffusion,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = uw_dpg
-PAR_EXAMPLES = uw_dpgp
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
@@ -1,652 +0,0 @@
-//                     MFEM Ultraweak DPG example
-//
-// Compile with: make uw_dpg
-//
-// sample runs 
-// ./uw_dpg  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
-//     - εΔu + ∇⋅(βu) = f,   in Ω
-//                  u = u_0, on ∂Ω
-
-// First Order System
-
-//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
-//        1/ε σ - ∇u  = 0,   in Ω
-//                  u = u_0, on ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, σ̂ ∈ H^-1/2  
-// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
-//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
-//                                        û = u_0  on ∂Ω 
-
-// Note: 
-// f̂ := βu - σ
-// û := -u
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
-// -------------------------------------------------------------
-// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
-// |   |           |           |           |          |         |
-// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
-
-// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   polynomial,
-   EJ,   
-   general  
-};
-
-enum test_norm_type
-{
-   standard,   
-   adjoint_graph,
-   robust  
-};
-
-prob_type prob;
-test_norm_type test_norm;
-Vector beta;
-double epsilon;
-// Function returns the solution u, and gradient du and the Laplacian d2u
-void solution(const Vector & x, double & u, Vector & du, double & d2u);
-double exact_u(const Vector & X);
-void exact_sigma(const Vector & X, Vector & sigma);
-double exact_hatu(const Vector & X);
-void exact_hatf(const Vector & X, Vector & hatf);
-double f_exact(const Vector & X);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool visualization = true;
-   int iprob = 0;
-   int itest_norm = 0;
-   double theta = 0.7;
-   epsilon = 1e0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&epsilon, "-eps", "--epsilon",
-                  "Epsilon coefficient");               
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");        
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                             
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: polynomial, 1: EJ ,2: General");           
-   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
-                  " 0: Standard, 1: Adjoint Graph, 2: Robust");       
-   args.AddOption(&beta, "-beta", "--beta",
-                  "Vector Coefficient beta");                                              
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 2) { iprob = 2; }
-   prob = (prob_type)iprob;
-   test_norm = (test_norm_type)itest_norm;
-
-   if (prob == prob_type::EJ)
-   {
-      mesh_file = "../../../data/inline-quad.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   if (beta.Size() == 0)
-   {
-      beta.SetSize(dim);
-      beta[0] = 1.;
-      beta[1] = 0.;
-   }
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatf_fes = new FiniteElementSpace(&mesh,hatf_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient eps(epsilon);
-   ConstantCoefficient eps1(1./epsilon);
-   ConstantCoefficient negeps1(-1./epsilon);
-   ConstantCoefficient eps2(1/(epsilon*epsilon));
-
-   ConstantCoefficient negeps(-epsilon);
-   VectorConstantCoefficient betacoeff(beta);
-   Vector negbeta = beta;
-   negbeta.Neg();
-
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-
-
-   DenseMatrix bbt(beta.Size());
-   MultVVt(beta, bbt); 
-   MatrixConstantCoefficient bbtcoeff(bbt);
-
-
-   VectorConstantCoefficient negbetacoeff(negbeta);
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatf_fes);
-   test_fec.Append(v_fec);
-   test_fec.Append(tau_fec);
-
-
-   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
-   FiniteElementSpace *coeff_fes = new FiniteElementSpace(&mesh,coeff_fec);
-   GridFunction c1_gf, c2_gf;
-   GridFunctionCoefficient c1_coeff(&c1_gf);
-   GridFunctionCoefficient c2_coeff(&c2_gf);
-
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //-(βu , ∇v)
-   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
-   
-   // (σ,∇ v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-   // (u ,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
-
-   // 1/ε (σ,τ)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-   // <f̂ ,v> 
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   switch (test_norm)
-   {
-      case standard:
-      {   
-         // (∇v,∇δv)
-         mfem::out << "\n Test norm: Standard" << endl;
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-      }
-      break;
-      case adjoint_graph:
-      {
-         mfem::out << "\n Test norm: Adjoint Graph" << endl;
-         // (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-         // 1/ε^2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
-         // 1/ε (∇v, δτ)
-         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
-         // - (β ⋅ ∇v,∇⋅δτ)  
-         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
-         // 1/ε (τ,∇δv) 
-         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
-         // -(β ∇⋅τ ,∇⋅δv)  
-         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
-      }
-      break;
-      default:
-      {
-         mfem::out << "\n Test norm: Robust" << endl;
-         c1_gf.SetSpace(coeff_fes);
-         c2_gf.SetSpace(coeff_fes);
-         Array<int> dofs;
-         for (int i =0; i < mesh.GetNE(); i++)
-         {
-            double volume = mesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            // double c2 = 1.;
-            coeff_fes->GetElementDofs(i,dofs);
-            c1_gf.SetSubVector(dofs,c1);
-            c2_gf.SetSubVector(dofs,c2);
-         }
-         // c1 (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
-         // ε (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // c2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-      }
-      break;
-   }
-
-
-   FunctionCoefficient f(f_exact);
-   // if (prob != prob_type::EJ)
-   // {
-      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
-   // }
-
-   FunctionCoefficient hatuex(exact_hatu);
-   VectorFunctionCoefficient hatfex(dim,exact_hatf);
-   Array<int> elements_to_refine;
-   FunctionCoefficient uex(exact_u);
-   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
-   GridFunction hatu_gf;
-   GridFunction hatf_gf;
-
-   // socketstream uex_out;
-   socketstream u_out;
-   // socketstream sigma_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      // uex_out.open(vishost, visport);
-      // sigma_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-   for (int i = 0; i<=ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list_uhat;
-      Array<int> ess_tdof_list_fhat;
-      Array<int> ess_bdr_uhat;
-      Array<int> ess_bdr_fhat;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr_uhat.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr_fhat.SetSize(mesh.bdr_attributes.Max());
-         // ess_bdr_uhat = 1;
-         // ess_bdr_fhat = 0;
-         ess_bdr_uhat = 0;
-         ess_bdr_fhat = 1;
-         ess_bdr_uhat[1] = 1;
-         ess_bdr_fhat[1] = 0;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
-         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
-      }
-
-      // shift the ess_tdofs
-      int n = ess_tdof_list_uhat.Size();
-      int m = ess_tdof_list_fhat.Size();
-      Array<int> ess_tdof_list(n+m);
-      for (int j = 0; j < n; j++)
-      {
-         ess_tdof_list[j] = ess_tdof_list_uhat[j] 
-                          + u_fes->GetTrueVSize() 
-                          + sigma_fes->GetTrueVSize();
-      }
-      for (int j = 0; j < m; j++)
-      {
-         ess_tdof_list[j+n] = ess_tdof_list_fhat[j] 
-                            + u_fes->GetTrueVSize() 
-                            + sigma_fes->GetTrueVSize()
-                            + hatu_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatf_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-
-      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
-
-      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
-      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      // BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      // M->owns_blocks = 1;
-      // for (int i=0; i<A->NumRowBlocks(); i++)
-      // {
-      //    M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-      // }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(200000);
-      cg.SetPrintLevel(0);
-      // cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      // delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }   
-
-      GridFunction uex_gf(u_fes);
-      uex_gf.ProjectCoefficient(uex);
-
-      GridFunction sigmaex_gf(sigma_fes);
-      sigmaex_gf.ProjectCoefficient(sigmaex);
-
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      GridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      int dofs = X.Size();
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = uex_gf.ComputeL2Error(zero);
-      // mfem::out << "u_err = " << u_err << endl;
-      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
-      double sigma_norm = sigmaex_gf.ComputeL2Error(vzero);
-      // mfem::out << "sigma_err = " << sigma_err << endl;
-      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
-      double L2norm = sqrt(u_norm * u_norm + sigma_norm * sigma_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = residual;
-      dof0 = dofs;
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::endl;
-
-
-      if (visualization)
-      {
-         // uex_out.precision(8);
-         // uex_out << "solution\n" << mesh << uex_gf <<
-         //          "window_title 'Exact u' "
-         //          << flush;
-         u_out.precision(8);
-         u_out << "solution\n" << mesh << u_gf <<
-                  "window_title 'Numerical u' "
-                  << flush;
-         // sigma_out.precision(8);
-         // sigma_out << "solution\n" << mesh << sigma_gf <<
-         //       "window_title 'Numerical flux' "
-         //       << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-      if (test_norm == test_norm_type::robust)
-      {
-         coeff_fes->Update();
-         c1_gf.Update();
-         c2_gf.Update();
-         Array<int> dofs;
-         for (int i = 0; i < mesh.GetNE(); i++)
-         {
-            double volume = mesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            // double c2 = 1.;
-            coeff_fes->GetElementDofs(i,dofs);
-            c1_gf.SetSubVector(dofs,c1);
-            c2_gf.SetSubVector(dofs,c2);
-         }
-      }
-
-   }
-
-   delete coeff_fes;
-   delete coeff_fec;
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatf_fes;
-   delete hatf_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fes;
-   delete sigma_fec;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
-
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u)
-{
-   double x = X[0];
-   double y = X[1];
-   double z = 0.;
-   if (X.Size() == 3) z = X[2];
-   du.SetSize(X.Size());
-   du = 0.;
-   d2u = 0.;
-
-   switch(prob)
-   {
-      case polynomial:
-      {
-         int n=2;
-         int m=2;
-         u = pow(x,n)*pow(y,m);
-         du[0] = n * pow(x,n-1) * pow(y,m);
-         du[1] = m * pow(x,n) * pow(y,m-1);
-         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
-             + m * (m-1) * pow(x,n) * pow(y,m-2);
-      }
-      break;
-      case EJ:
-      {
-         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
-         double r1 = (1. + alpha) / (2.*epsilon);
-         double r2 = (1. - alpha) / (2.*epsilon);
-         double denom = exp(-r2) - exp(-r1);
-
-
-         double g1 = exp(r2*(x-1.));
-         double g1_x = r2*g1;
-         double g1_xx = r2*g1_x;
-         double g2 = exp(r1*(x-1.));
-         double g2_x = r1*g2;
-         double g2_xx = r1*g2_x;
-         double g = g1-g2;
-         double g_x = g1_x - g2_x;
-         double g_xx = g1_xx - g2_xx;
-
-
-         u = g * cos(M_PI * y)/denom;
-         double u_x = g_x * cos(M_PI * y)/denom;
-         double u_xx = g_xx * cos(M_PI * y)/denom;
-         double u_y = -M_PI * g * sin(M_PI*y)/denom;
-         double u_yy = -M_PI * M_PI * u;
-         du[0] = u_x;
-         du[1] = u_y;
-         d2u = u_xx + u_yy;
-
-      }
-      break;
-      default:
-      {
-         double alpha = M_PI * (x + y + z);
-         u = sin(alpha);
-         du.SetSize(X.Size());
-         for (int i = 0; i<du.Size(); i++)
-         {
-            du[i] = M_PI * cos(alpha);
-         }
-         d2u = - M_PI*M_PI * u * du.Size();
-      }
-      break;
-   }
-}
-
-
-double exact_u(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return u;
-}
-
-void exact_sigma(const Vector & X, Vector & sigma)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   // σ = ε ∇ u
-   sigma = du;
-   sigma *= epsilon;
-}
-
-double exact_hatu(const Vector & X)
-{
-   return -exact_u(X);
-}
-
-void exact_hatf(const Vector & X, Vector & hatf)
-{
-   Vector sigma;
-   exact_sigma(X,sigma);
-   double u = exact_u(X);
-   hatf.SetSize(X.Size());
-   for (int i = 0; i<hatf.Size(); i++)
-   {
-      hatf[i] = beta[i] * u - sigma[i];
-   }
-}
-
-double f_exact(const Vector & X)
-{
-   // f = - εΔu + ∇⋅(βu)
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-
-   double s = 0;
-   for (int i = 0; i<du.Size(); i++)
-   {
-      s += beta[i] * du[i];
-   }
-   return -epsilon * d2u + s;
-}
@@ -1,704 +0,0 @@
-//                                MFEM Ultraweak DPG example
-//
-// Compile with: make  uw_dpgp
-//
-// sample runs 
-// mpirun -np 6 ./uw_dpgp  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
-//     - εΔu + ∇⋅(βu) = f,   in Ω
-//                  u = u_0, on ∂Ω
-
-// First Order System
-
-//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
-//        1/ε σ - ∇u  = 0,   in Ω
-//                  u = u_0, on ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, f̂ ∈ H^-1/2  
-// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
-//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
-//                                        û = u_0  on ∂Ω 
-
-// Note: 
-// f̂ := βu - σ
-// û := -u
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
-// -------------------------------------------------------------
-// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
-// |   |           |           |           |          |         |
-// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
-
-// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   polynomial,
-   EJ,   
-   general  
-};
-
-enum test_norm_type
-{
-   standard,   
-   adjoint_graph,
-   robust  
-};
-
-prob_type prob;
-test_norm_type test_norm;
-Vector beta;
-double epsilon;
-// Function returns the solution u, and gradient du and the Laplacian d2u
-void solution(const Vector & x, double & u, Vector & du, double & d2u);
-double exact_u(const Vector & X);
-void exact_sigma(const Vector & X, Vector & sigma);
-double exact_hatu(const Vector & X);
-void exact_hatf(const Vector & X, Vector & hatf);
-double f_exact(const Vector & X);
-
-
-int main(int argc, char *argv[])
-{
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool visualization = true;
-   int iprob = 0;
-   int itest_norm = 0;
-   double theta = 0.7;
-   bool static_cond = false;
-   epsilon = 1e0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&epsilon, "-eps", "--epsilon",
-                  "Epsilon coefficient");               
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                           
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: lshape, 1: General");                         
-   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
-                  " 0: Standard, 1: Adjoint Graph, 2: Robust");     
-   args.AddOption(&beta, "-beta", "--beta",
-                  "Vector Coefficient beta");          
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                            
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   if (iprob > 2) { iprob = 2; }
-   prob = (prob_type)iprob;
-
-   test_norm = (test_norm_type)itest_norm;
-
-   if (prob == prob_type::EJ)
-   {
-      mesh_file = "../../../data/inline-quad.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   if (beta.Size() == 0)
-   {
-      beta.SetSize(dim);
-      beta[0] = 1.;
-      beta[1] = 0.;
-   }
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatf_fes = new ParFiniteElementSpace(&pmesh,hatf_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient eps(epsilon);
-   ConstantCoefficient eps1(1./epsilon);
-   ConstantCoefficient negeps1(-1./epsilon);
-   ConstantCoefficient eps2(1/(epsilon*epsilon));
-
-   ConstantCoefficient negeps(-epsilon);
-   VectorConstantCoefficient betacoeff(beta);
-   Vector negbeta = beta;
-   negbeta.Neg();
-
-   DenseMatrix bbt(beta.Size());
-   MultVVt(beta, bbt); 
-   MatrixConstantCoefficient bbtcoeff(bbt);
-
-
-   VectorConstantCoefficient negbetacoeff(negbeta);
-   // Normal equation weak formulation
-   Array<ParFiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatf_fes);
-   test_fec.Append(v_fec);
-   test_fec.Append(tau_fec);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //-(βu , ∇v)
-   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
-   
-   // (σ,∇ v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-   // (u ,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
-
-   // 1/ε (σ,τ)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-   // <f̂ ,v> 
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
-   ParFiniteElementSpace *coeff_fes = new ParFiniteElementSpace(&pmesh,coeff_fec);
-   ParGridFunction c1_gf, c2_gf;
-   GridFunctionCoefficient c1_coeff(&c1_gf);
-   GridFunctionCoefficient c2_coeff(&c2_gf);
-
-   switch (test_norm)
-   {
-      case standard:
-      {   
-         if (myid == 0)
-         {
-            mfem::out << "\n Test norm: Standard" << endl;
-         }
-         // (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-      }
-      break;
-      case adjoint_graph:
-      {
-         if (myid == 0)
-         {
-            mfem::out << "\n Test norm: Adjoint Graph" << endl;
-         }
-         // (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-         // 1/ε^2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
-         // 1/ε (∇v, δτ)
-         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
-         // - (β ⋅ ∇v,∇⋅δτ)  
-         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
-         // 1/ε (τ,∇δv) 
-         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
-         // -(β ∇⋅τ ,∇⋅δv)  
-         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
-      }
-      break;
-      default:
-      {
-         if (myid == 0)
-         {
-            mfem::out << "\n Test norm: Robust" << endl;
-         }
-         c1_gf.SetSpace(coeff_fes);
-         c2_gf.SetSpace(coeff_fes);
-         Array<int> dofs;
-         for (int i =0; i < pmesh.GetNE(); i++)
-         {
-            double volume = pmesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            coeff_fes->GetElementDofs(i,dofs);
-            c1_gf.SetSubVector(dofs,c1);
-            c2_gf.SetSubVector(dofs,c2);
-         }
-         // c1 (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
-         // ε (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // c2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-      }
-      break;
-   }
-
-
-   FunctionCoefficient f(f_exact);
-   // if (prob != prob_type::EJ)
-   // {
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
-   // }
-
-   FunctionCoefficient hatuex(exact_hatu);
-   VectorFunctionCoefficient hatfex(dim,exact_hatf);
-   Array<int> elements_to_refine;
-   FunctionCoefficient uex(exact_u);
-   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
-
-   ParGridFunction hatu_gf;
-   ParGridFunction hatf_gf;
-
-   // socketstream uex_out;
-   socketstream u_out;
-   // socketstream sigma_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      // uex_out.open(vishost, visport);
-      // sigma_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   if (myid == 0)
-   {
-      mfem::out << " Refinement |" 
-               << "    Dofs    |" 
-               << "  L2 Error  |" 
-               << "  Rate  |" 
-               << "  Residual  |" 
-               << "  Rate  |" 
-               << "  CG iter  |" << endl;
-      mfem::out << " --------------------"      
-               <<  "-------------------"    
-               <<  "-------------------"    
-               <<  "-------------------" << endl;   
-   }
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list_uhat;
-      Array<int> ess_tdof_list_fhat;
-      Array<int> ess_bdr_uhat;
-      Array<int> ess_bdr_fhat;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr_uhat.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr_fhat.SetSize(pmesh.bdr_attributes.Max());
-         // ess_bdr_uhat = 1;
-         // ess_bdr_fhat = 0;
-         ess_bdr_uhat = 0;
-         ess_bdr_fhat = 1;
-         ess_bdr_uhat[1] = 1;
-         ess_bdr_fhat[1] = 0;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
-         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
-      }
-
-      // shift the ess_tdofs
-      int n = ess_tdof_list_uhat.Size();
-      int m = ess_tdof_list_fhat.Size();
-      Array<int> ess_tdof_list(n+m);
-      for (int j = 0; j < n; j++)
-      {
-         ess_tdof_list[j] = ess_tdof_list_uhat[j] 
-                          + u_fes->GetTrueVSize() 
-                          + sigma_fes->GetTrueVSize();
-      }
-      for (int j = 0; j < m; j++)
-      {
-         ess_tdof_list[j+n] = ess_tdof_list_fhat[j] 
-                            + u_fes->GetTrueVSize() 
-                            + sigma_fes->GetTrueVSize()
-                            + hatu_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatf_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
-
-      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
-      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
-      {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M->SetDiagonalBlock(0,amg0);
-         M->SetDiagonalBlock(1,amg1);
-         skip = 2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
-      amg2->SetPrintLevel(0);
-      M->SetDiagonalBlock(skip,amg2);
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
-      }
-      M->SetDiagonalBlock(skip+1,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-6);
-      cg.SetMaxIter(200000);
-      cg.SetPrintLevel(-1);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      int num_iter = cg.GetNumIterations();
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      double maxresidual = residuals.Max(); 
-
-      double gresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&gresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      gresidual = sqrt(gresidual);
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      ParGridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      int dofs = u_fes->GlobalTrueVSize()
-               + sigma_fes->GlobalTrueVSize()
-               + hatu_fes->GlobalTrueVSize()
-               + hatf_fes->GlobalTrueVSize();
-
-      double u_err = u_gf.ComputeL2Error(uex);
-      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
-      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/gresidual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = gresidual;
-      dof0 = dofs;
-
-      if (myid == 0)
-      {
-         mfem::out << std::right << std::setw(11) << i << " | " 
-                  << std::setw(10) <<  dof0 << " | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::scientific <<  err0 << " | " 
-                  << std::setprecision(2) 
-                  << std::setw(6) << std::fixed << rate_err << " | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::scientific <<  res0 << " | " 
-                  << std::setprecision(2) 
-                  << std::setw(6) << std::fixed << rate_res << " | " 
-                  << std::setw(6) << std::fixed << num_iter << " | " 
-                  << std::resetiosflags(std::ios::showbase)
-                  << std::endl;
-      }
-
-
-      if (visualization)
-      {
-         // uex_out.precision(8);
-         // uex_out << "parallel " << num_procs << " " << myid << "\n";
-         // uex_out << "solution\n" << pmesh << uex_gf <<
-         //          "window_title 'Exact u' "
-         //          << flush;
-
-         u_out << "parallel " << num_procs << " " << myid << "\n";
-         u_out.precision(8);
-         u_out << "solution\n" << pmesh << u_gf <<
-                  "window_title 'Numerical u' "
-                  << flush;
-
-         // sigma_out << "parallel " << num_procs << " " << myid << "\n";
-         // sigma_out.precision(8);
-         // sigma_out << "solution\n" << pmesh << sigma_gf <<
-         //       "window_title 'Numerical flux' "
-         //       << flush;
-      }
-
-      if (i == ref-1)
-         break;
-
-      pmesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-      if (test_norm == test_norm_type::robust)
-      {
-         coeff_fes->Update();
-         c1_gf.Update();
-         c2_gf.Update();
-         Array<int> edofs;
-         for (int i = 0; i < pmesh.GetNE(); i++)
-         {
-            double volume = pmesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            coeff_fes->GetElementDofs(i,edofs);
-            c1_gf.SetSubVector(edofs,c1);
-            c2_gf.SetSubVector(edofs,c2);
-         }
-      }
-   }
-
-   delete coeff_fes;
-   delete coeff_fec;
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatf_fes;
-   delete hatf_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete sigma_fes;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u)
-{
-   double x = X[0];
-   double y = X[1];
-   double z = 0.;
-   if (X.Size() == 3) z = X[2];
-   du.SetSize(X.Size());
-   du = 0.;
-   d2u = 0.;
-
-   switch(prob)
-   {
-      case polynomial:
-      {
-         int n=2;
-         int m=2;
-         u = pow(x,n)*pow(y,m);
-         du[0] = n * pow(x,n-1) * pow(y,m);
-         du[1] = m * pow(x,n) * pow(y,m-1);
-         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
-             + m * (m-1) * pow(x,n) * pow(y,m-2);
-      }
-      break;
-      case EJ:
-      {
-         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
-         double r1 = (1. + alpha) / (2.*epsilon);
-         double r2 = (1. - alpha) / (2.*epsilon);
-         double denom = exp(-r2) - exp(-r1);
-
-
-         double g1 = exp(r2*(x-1.));
-         double g1_x = r2*g1;
-         double g1_xx = r2*g1_x;
-         double g2 = exp(r1*(x-1.));
-         double g2_x = r1*g2;
-         double g2_xx = r1*g2_x;
-         double g = g1-g2;
-         double g_x = g1_x - g2_x;
-         double g_xx = g1_xx - g2_xx;
-
-
-         u = g * cos(M_PI * y)/denom;
-         double u_x = g_x * cos(M_PI * y)/denom;
-         double u_xx = g_xx * cos(M_PI * y)/denom;
-         double u_y = -M_PI * g * sin(M_PI*y)/denom;
-         double u_yy = -M_PI * M_PI * u;
-         du[0] = u_x;
-         du[1] = u_y;
-         d2u = u_xx + u_yy;
-
-      }
-      break;
-      default:
-      {
-         double alpha = M_PI * (x + y + z);
-         u = sin(alpha);
-         du.SetSize(X.Size());
-         for (int i = 0; i<du.Size(); i++)
-         {
-            du[i] = M_PI * cos(alpha);
-         }
-         d2u = - M_PI*M_PI * u * du.Size();
-      }
-      break;
-   }
-}
-
-
-double exact_u(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return u;
-}
-
-void exact_sigma(const Vector & X, Vector & sigma)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   // σ = ε ∇ u
-   sigma = du;
-   sigma *= epsilon;
-}
-
-double exact_hatu(const Vector & X)
-{
-   return -exact_u(X);
-}
-
-void exact_hatf(const Vector & X, Vector & hatf)
-{
-   Vector sigma;
-   exact_sigma(X,sigma);
-   double u = exact_u(X);
-   hatf.SetSize(X.Size());
-   for (int i = 0; i<hatf.Size(); i++)
-   {
-      hatf[i] = beta[i] * u - sigma[i];
-   }
-}
-
-double f_exact(const Vector & X)
-{
-   // f = - εΔu + ∇⋅(βu)
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-
-   double s = 0;
-   for (int i = 0; i<du.Size(); i++)
-   {
-      s += beta[i] * du[i];
-   }
-   return -epsilon * d2u + s;
-}
@@ -1,203 +0,0 @@
-//                 MFEM Fosls 1
-//
-// Compile with: make blkfosls
-//
-//     - Δ u = f, in Ω
-//         u = 0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// FOSLS:
-//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
-
-
-// -------------------------------------------------
-// |   |    u    |         σ           |    RHS    |
-// -------------------------------------------------
-// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
-// |   |         |                     |           |
-// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
-
-// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec0 = new H1_FECollection(order, dim);
-   FiniteElementCollection *fec1 = new RT_FECollection(order-1, dim);
-   FiniteElementSpace fespace0(&mesh, fec0);
-   FiniteElementSpace fespace1(&mesh, fec1);
-
-   Array<FiniteElementSpace *> fespaces(2);
-   fespaces[0] = &fespace0;
-   fespaces[1] = &fespace1;
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   BlockBilinearForm a(fespaces);
-   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
-
-   cout << "H1 fespace = " << fespace0.GetVSize() << endl;
-   cout << "RT fespace = " << fespace1.GetVSize() << endl;
-
-   FiniteElementCollection *fec2 = new RT_Trace_FECollection(order-1, dim);
-   FiniteElementSpace RT_trace_fes(&mesh, fec2);
-   cout << "RT trace = " << RT_trace_fes.GetVSize() << endl;
-
-   // for (int i = 0; i<mesh.GetNE(); i++)
-   // {
-   //    // const FiniteElement * fe = fespace1.GetFE(i);
-   //    // fespace1.GetTraceElement()
-   //    Array<int> faces, ori;
-   //    mesh.GetElementEdges(i, faces, ori);
-   //    for (int f = 0; f<faces.Size(); f++)
-   //    {
-   //       const FiniteElement * fe_trace = RT_trace_fes.GetFaceElement(faces[f]);
-   //       cout << fe_trace->GetDof() << endl;
-   //       Array<int> face_dofs;
-   //       RT_trace_fes.GetFaceDofs(faces[f],face_dofs);
-   //       cout << "face dofs = " << endl;
-   //       face_dofs.Print();
-   //    }
-
-   //    // cout << fe->GetGeomType() << endl;
-   //    Array<int> vdofs;
-   //    RT_trace_fes.GetElementVDofs(i, vdofs);
-   //    cout << "trace dofs = " << endl;
-   //    vdofs.Print();
-   //    fespace1.GetElementVDofs(i, vdofs);
-   //    cout << "elem dofs = " << endl;
-   //    vdofs.Print();
-   //    cin.get();
-   // }
-
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   Array2D<BilinearFormIntegrator * > blfi(2,2);
-   blfi(0,0) = new DiffusionIntegrator(one);
-   blfi(0,1) = new MixedVectorWeakDivergenceIntegrator(one);
-   blfi(1,0) = new MixedVectorGradientIntegrator(negone);
-
-   BilinearFormIntegrator * divdiv = new DivDivIntegrator(one);
-   BilinearFormIntegrator * mass = new VectorFEMassIntegrator(one);
-   SumIntegrator * suminteg = new SumIntegrator();
-   suminteg->AddIntegrator(divdiv);
-   suminteg->AddIntegrator(mass);
-   blfi(1,1) = suminteg;
-
-   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
-   integ->SetIntegrators(blfi);
-   a.AddDomainIntegrator(integ);
-   a.Assemble();
-
-
-   BlockLinearForm b(fespaces);
-
-   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
-   Array<LinearFormIntegrator * > lfi(2);
-   lfi[0] = nullptr;
-   lfi[1] = new VectorFEDomainLFDivIntegrator(negone);
-   lininteg->SetIntegrators(lfi);
-   b.AddDomainIntegrator(lininteg);
-   b.Assemble();
-
-
-   // need to implement blkgridfunction later but for now Vector would do
-   int size = 0;
-   for (int i = 0; i<fespaces.Size(); i++)
-   {
-      size += fespaces[i]->GetVSize();
-   }
-
-   Vector x(size);
-   x = 0.0;
-
-   OperatorPtr A;
-   Vector X,B;
-   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
-
-   GSSmoother M((SparseMatrix&)(*A));
-   CGSolver cg;
-   cg.SetRelTol(1e-6);
-   cg.SetMaxIter(200);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   a.RecoverFEMSolution(X,b,x);
-
-   GridFunction u_gf, sigma_gf;
-   double *data = x.GetData();
-   u_gf.MakeRef(fespaces[0],&data[0]);
-   sigma_gf.MakeRef(fespaces[1],&data[fespaces[0]->GetVSize()]);
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << u_gf <<
-                "window_title 'Numerical u' "
-                << flush;
-      socketstream sols_sock(vishost, visport);
-      sols_sock.precision(8);
-      sols_sock << "solution\n" << mesh << sigma_gf <<
-                "window_title 'Numerical sigma' "
-                << flush;
-   }
-
-   delete fec0;
-
-   return 0;
-}
@@ -1,223 +0,0 @@
-//                   MFEM Fosls example
-//
-// Compile with: make fosls
-//
-//     - Δ u = f, in Ω
-//         u = 0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// FOSLS:
-//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
-
-
-// -------------------------------------------------
-// |   |    u    |         σ           |    RHS    |
-// -------------------------------------------------
-// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
-// |   |         |                     |           |
-// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
-
-// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   FiniteElementCollection *H1fec = new H1_FECollection(order,dim);
-   FiniteElementSpace *H1fes = new FiniteElementSpace(&mesh, H1fec);
-
-   FiniteElementCollection *RTfec = new RT_FECollection(order-1,dim);
-   FiniteElementSpace *RTfes = new FiniteElementSpace(&mesh, RTfec);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   // Linear forms
-   LinearForm b_0(H1fes);
-   // (f,∇⋅τ )
-   LinearForm b_1(RTfes);
-   b_1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(negone));
-
-   // Bilinear forms
-   // (∇u,∇v)
-   BilinearForm a_00(H1fes);
-   a_00.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   //   -(σ,∇v)
-   MixedBilinearForm a_01(RTfes, H1fes);
-   a_01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(
-                               one)); // (-1 is included)
-
-   //    // -(∇u,τ)
-   // MixedBilinearForm()
-   MixedBilinearForm a_10(H1fes, RTfes);
-   a_10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negone));
-
-   // (∇⋅σ, ∇⋅τ) + (σ,τ)
-
-   BilinearForm a_11(RTfes);
-   a_11.AddDomainIntegrator(new DivDivIntegrator(one));
-   a_11.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      H1fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   Array<int> block_Toffsets(3);
-   block_Toffsets[0] = 0;
-   block_Toffsets[1] = H1fes->GetTrueVSize();
-   block_Toffsets[2] = RTfes->GetTrueVSize();
-   block_Toffsets.PartialSum();
-
-   Vector rhs_H1(H1fes->GetVSize()); rhs_H1 = 0.;
-   Vector rhs_RT(RTfes->GetVSize()); rhs_RT = 0.;
-
-   Vector x_H1(H1fes->GetVSize()); x_H1 = 0.;
-   Vector x_RT(RTfes->GetVSize()); x_RT = 0.;
-
-
-   Vector RHS_H1(H1fes->GetTrueVSize()); RHS_H1 = 0.0;
-   Vector RHS_RT(RTfes->GetTrueVSize()); RHS_RT = 0.0;
-
-   Vector X_H1(H1fes->GetTrueVSize()); X_H1 = 0.0;
-   Vector X_RT(RTfes->GetTrueVSize()); X_RT = 0.0;
-
-
-   b_0.Update(H1fes,rhs_H1,0);
-   b_0.Assemble();
-
-   b_1.Update(RTfes,rhs_RT,0);
-   b_1.Assemble();
-
-
-   // Assembly and BC
-   a_00.Assemble();
-   SparseMatrix A_00;
-   a_00.FormLinearSystem(ess_tdof_list,x_H1,rhs_H1,
-                         A_00,X_H1,RHS_H1);
-
-   a_01.Assemble();
-   SparseMatrix A_01;
-   Array<int> empty;
-   a_01.FormRectangularSystemMatrix(empty, ess_tdof_list,A_01);
-
-
-   a_10.Assemble();
-   SparseMatrix A_10;
-
-   a_10.FormRectangularLinearSystem(ess_tdof_list,empty,x_H1,rhs_RT,
-                                    A_10,X_H1,RHS_RT);
-
-   a_11.Assemble();
-   SparseMatrix A_11;
-   a_11.FormSystemMatrix(empty,A_11);
-
-
-   BlockMatrix BlockA(block_Toffsets);
-   BlockA.SetBlock(0,0,&A_00);
-   BlockA.SetBlock(0,1,&A_01);
-   BlockA.SetBlock(1,0,&A_10);
-   BlockA.SetBlock(1,1,&A_11);
-
-
-   BlockVector RHS(block_Toffsets);
-   RHS.GetBlock(0) = RHS_H1;
-   RHS.GetBlock(1) = RHS_RT;
-
-   BlockVector X(block_Toffsets);
-   X.GetBlock(0) = X_H1;
-   X.GetBlock(1) = X_RT;
-
-
-   SparseMatrix * A = BlockA.CreateMonolithic();
-
-   GSSmoother M(*A);
-
-   CGSolver cg;
-   cg.SetRelTol(1e-6);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(RHS, X);
-
-   GridFunction u_gf(H1fes), sigma_gf(RTfes);
-   u_gf = 0.;
-   sigma_gf = 0.;
-
-   const SparseMatrix * P = H1fes->GetConformingProlongation();
-   if (P)
-   {
-      a_00.RecoverFEMSolution(X.GetBlock(0),rhs_H1,u_gf);
-      a_11.RecoverFEMSolution(X.GetBlock(1),rhs_RT,sigma_gf);
-   }
-   else
-   {
-      u_gf.MakeRef(X.GetBlock(0),0);
-      sigma_gf.MakeRef(X.GetBlock(1),0);
-   }
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << u_gf <<
-                "window_title 'Numerical u' "
-                << flush;
-      socketstream sols_sock(vishost, visport);
-      sols_sock.precision(8);
-      sols_sock << "solution\n" << mesh << sigma_gf <<
-                "window_title 'Numerical sigma' "
-                << flush;
-   }
-
-   return 0;
-}
@@ -1,61 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/diffusion,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = blkfosls fosls primal_dpg \
-	 				uw_dpg 
-PAR_EXAMPLES = uw_dpgp 
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-	rm -rf ParaView
-
-clean-exec:
@@ -1,179 +0,0 @@
-//                     MFEM primal_dpg example
-//
-// Compile with: make primal_dpg
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");   
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   // mesh.UniformRefinement();
-
-   // 3. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   H1_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace H1fes(&mesh, &fec);
-
-   RT_Trace_FECollection trace_fec(order-1, mesh.Dimension());
-   FiniteElementSpace RTtrace_fes(&mesh, &trace_fec);
-
-   int dim = mesh.Dimension();
-   int test_order = order;
-   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
-   {
-      test_order++;
-   }
-
-   test_order++;
-
-   H1_FECollection test_fec(test_order,mesh.Dimension());
-
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fecs; 
-
-   trial_fes.Append(&H1fes);
-   trial_fes.Append(&RTtrace_fes);
-   test_fecs.Append(&test_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
-
-
-   ConstantCoefficient one(1.0);
-   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
-   a->AddTrialIntegrator(new TraceIntegrator,1,0);
-
-   BilinearFormIntegrator * diffusion = new DiffusionIntegrator(one);
-   BilinearFormIntegrator * mass = new MassIntegrator(one);
-   a->AddTestIntegrator(diffusion,0,0);
-   a->AddTestIntegrator(mass,0,0);
-
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(one),0);
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   Vector X,B;
-   OperatorPtr Ah;
-
-   int size = H1fes.GetVSize() + RTtrace_fes.GetVSize();
-
-   Vector x(size);
-   x = 0.0;
-
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-6);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   delete M;
-
-
-   a->RecoverFEMSolution(X,x);
-
-   GridFunction u_gf;
-   double *data = x.GetData();
-   u_gf.MakeRef(&H1fes,data);
-
-   GridFunction s_gf;
-   s_gf.MakeRef(&RTtrace_fes,&data[H1fes.GetVSize()]);
-
-
-
-   RT_FECollection RTfec(order-1, mesh.Dimension());
-   FiniteElementSpace RTfes(&mesh, &RTfec);
-
-   GridFunction sigma_gf(&RTfes);
-   sigma_gf = 0.0;
-   for (int i = 0; i<mesh.GetNE(); i++)
-   {
-      Array<int> strace_dofs;
-      Array<int> trace_dofs;
-      Vector dofs;
-      RTtrace_fes.GetElementDofs(i,trace_dofs);
-      strace_dofs.SetSize(trace_dofs.Size());
-      // shift dofs;
-      for (int j = 0; j< trace_dofs.Size(); j++)
-      {
-         int offset = trace_dofs[j] < 0 ? -H1fes.GetVSize() : H1fes.GetVSize();
-         strace_dofs[j] = offset + trace_dofs[j];
-      }
-      x.GetSubVector(strace_dofs, dofs);
-      sigma_gf.SetSubVector(trace_dofs,dofs);
-
-   }
-
-
-
-
-
-   ParaViewDataCollection paraview_dc("DPG_example", &mesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(order);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("field",&u_gf);
-   paraview_dc.RegisterField("flux",&sigma_gf);
-   // paraview_dc.RegisterField("flux",&s_gf);
-   paraview_dc.Save();
-
-
-
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream solu_sock(vishost, visport);
-   solu_sock.precision(8);
-   solu_sock << "solution\n" << mesh << u_gf <<
-             "window_title 'Numerical u' "
-             << flush;
-
-   socketstream soltrace_sock(vishost, visport);
-   soltrace_sock.precision(8);
-   soltrace_sock << "solution\n" << mesh << sigma_gf <<
-                 "window_title 'Flux sigma_n' "
-                 << flush;
-
-
-
-}
@@ -1,403 +0,0 @@
-//                       MFEM Ultraweak DPG example
-//
-// Compile with: make uw_dpg
-//
-// sample runs 
-// ./uw_dpg -m ../lshape2.mesh -o 2 -ref 20 -graph-norm -do 1 -prob 0
-
-//     - Δ u = f,   in Ω
-//         u = u_0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, σ̂ ∈ H^-1/2  
-// -(u , ∇⋅τ) - (σ , τ) + < û, τ⋅n> = 0,      ∀ τ ∈ H(div,Ω)      
-//  (σ , ∇ v) + < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
-//                                û = 0       on ∂Ω 
-
-// Note: 
-// û := u
-// σ̂ := -σ
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
-// -------------------------------------------------------------
-// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
-// |   |           |           |           |         |         |
-// | v |           |  (σ,∇ v)  |           |  <σ̂,v>  |  (f,v)  |  
-
-// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   lshape,   
-   general  
-};
-
-prob_type prob;
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u);
-
-
-double exact_u(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return u;
-}
-
-void exact_sigma(const Vector & X, Vector & sigma)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   // σ = ∇ u
-   sigma = du;
-}
-
-double exact_hatu(const Vector & X)
-{
-   return exact_u(X);
-}
-
-void exact_hatsigma(const Vector & X, Vector & hatsigma)
-{
-   exact_sigma(X,hatsigma);
-   hatsigma *= -1.;
-}
-
-double f_exact(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return -d2u;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool adjoint_graph_norm = false;
-   bool visualization = true;
-   int iprob = 0;
-   bool static_cond = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");               
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space"); 
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: lshape, 1: General");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                  
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 1) { iprob = 1; }
-   prob = (prob_type)iprob;
-
-   if (prob == prob_type::lshape)
-   {
-      mesh_file = "../lshape2.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   mesh.UniformRefinement();
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatsigma_fes = new FiniteElementSpace(&mesh,hatsigma_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatsigma_fes);
-
-   test_fec.Append(tau_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //  -(u,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
-
-   // -(σ,τ) 
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negone)),1,0);
-
-   // (σ,∇ v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
-
-   //  <σ̂,v> 
-   a->AddTrialIntegrator(new TraceIntegrator,3,1);
-
-   // test integrators (space-induced norm for H(div) × H1)
-   // (∇⋅τ,∇⋅δτ)
-   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
-   // (τ,δτ)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   // (∇v,∇δv)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new MassIntegrator(one),1,1);
-
-   // additional terms for adjoint graph norm
-   if (adjoint_graph_norm)
-   {
-      // -(∇v,δτ) 
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
-      // -(τ,∇δv) 
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
-      // (τ,δτ)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f(f_exact);
-   if (prob == prob_type::general)
-   {
-      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),1);
-   }
-
-   FunctionCoefficient hatuex(exact_hatu);
-   Array<int> elements_to_refine;
-   GridFunction hatu_gf;
-
-
-   socketstream u_out;
-   // socketstream sigma_out;
-   socketstream mesh_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      // sigma_out.open(vishost, visport);
-      mesh_out.open(vishost, visport);
-   }
-
-
-   for (int iref = 0; iref<ref; iref++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-   // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatsigma_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      for (int i=0; i<A->NumRowBlocks(); i++)
-      {
-         M->SetDiagonalBlock(i,new GSSmoother(A->GetBlock(i,i)));
-      }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      cout << "Residual = " << residual << endl;
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      double theta = 0.7;
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-      
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      GridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      if (visualization)
-      {
-         u_out.precision(8);
-         string keys = (iref == 0) ? "keys em\n" : "keys";
-         u_out << "solution\n" << mesh << u_gf 
-               << "window_title 'Numerical u' "
-                  << flush;
-
-         // sigma_out.precision(8);
-         // sigma_out << "solution\n" << mesh << sigma_gf <<
-         //       "window_title 'Numerical flux' "
-         //       << flush;
-
-         mesh_out.precision(8);
-         mesh_out << "mesh\n" << mesh 
-                  << keys
-                  << "window_title 'Mesh' "
-                  << flush;
-
-      }
-
-      mesh.GeneralRefinement(elements_to_refine);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatsigma_fes;
-   delete hatsigma_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete sigma_fes;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
-
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u)
-{
-   double x = X[0];
-   double y = X[1];
-   double z = 0.;
-   if (X.Size() == 3) z = X[2];
-   du.SetSize(X.Size());
-   du = 0.;
-   d2u = 0.;
-
-   switch(prob)
-   {
-      case lshape:
-      {
-         double r = sqrt(x*x + y*y);
-         double alpha = 2./3.;
-         double theta = atan2(y,x);
-         if (theta < 0) theta += 2*M_PI;
-         u = pow(r,alpha) * sin(alpha * theta);
-      }
-      break;
-      default:
-      {
-         double alpha = M_PI * (x + y + z);
-         u = sin(alpha);
-         du.SetSize(X.Size());
-         for (int i = 0; i<du.Size(); i++)
-         {
-            du[i] = M_PI * cos(alpha);
-         }
-         d2u = - M_PI*M_PI * u * du.Size();
-      }
-      break;
-   }
-}
@@ -1,404 +0,0 @@
-//                 MFEM UW DPG parallel example
-//
-// Compile with: make poisson_fosls
-//
-//     - Δ u = f, in Ω
-//         u = 0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, σ̂ ∈ H^-1/2  
-// -(u , ∇⋅τ) + < û, τ⋅n> - (σ , τ) = 0,      ∀ τ ∈ H(div,Ω)      
-//  (σ , ∇ v) - < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
-//            û = 0        on ∂Ω 
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
-// -------------------------------------------------------------
-// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
-// |   |           |           |           |         |         |
-// | v |           |  (σ,∇ v)  |           | -<σ̂,v>  |  (f,v)  |  
-
-// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   lshape,   
-   general  
-};
-
-prob_type prob;
-
-double exact(const Vector & X)
-{
-   double x = X[0];
-   double y = X[1];
-
-   double r = sqrt(x*x + y*y);
-   double alpha = 2./3.;
-   double theta = atan2(y,x);
-   if (theta < 0) theta += 2*M_PI;
-
-   return pow(r,alpha) * sin(alpha * theta);
-}
-
-void gradexact(const Vector & X, Vector & grad)
-{
-   grad.SetSize(2);
-   double x = X[0];
-   double y = X[1];
-
-   double r = sqrt(x*x + y*y);
-   double alpha = 2./3.;
-   double theta = atan2(y,x);
-   if (theta < 0) theta += 2*M_PI;
-
-   double r_x = x/r;
-   double r_y = y/r;
-   double theta_x = - y / (r*r);
-   double theta_y = x / (r*r);
-   double beta = alpha * pow(r,alpha - 1.);
-   grad[0] = beta*(r_x * sin(alpha*theta) + r * theta_x * cos(alpha*theta));
-   grad[1] = beta*(r_y * sin(alpha*theta) + r * theta_y * cos(alpha*theta));
-}
-
-
-int main(int argc, char *argv[])
-{
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool adjoint_graph_norm = false;
-   bool visualization = true;
-   int iprob = 0;
-   bool static_cond = false;
-   double theta = 0.7;
-
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");    
-   args.AddOption(&theta, "-theta", "--theta_factor",
-                  "Refinement factor");                              
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");   
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: lshape, 1: General");       
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                                            
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   if (iprob > 1) { iprob = 1; }
-   prob = (prob_type)iprob;
-
-   if (prob == prob_type::lshape)
-   {
-      mesh_file = "../lshape2.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   mesh.UniformRefinement();
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,hatsigma_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   // Normal equation weak formulation
-   Array<ParFiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatsigma_fes);
-
-   test_fec.Append(tau_fec);
-   test_fec.Append(v_fec);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //  -(u,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
-
-   // -(σ,τ) 
-   TransposeIntegrator * mass = new TransposeIntegrator(new VectorFEMassIntegrator(negone));
-   a->AddTrialIntegrator(mass,1,0);
-
-   // (σ,∇ v)
-   TransposeIntegrator * grad = new TransposeIntegrator(new GradientIntegrator(one));
-   a->AddTrialIntegrator(grad,1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
-
-   // -<σ̂,v> (sign is included in σ̂)
-   a->AddTrialIntegrator(new TraceIntegrator,3,1);
-
-   // test integrators (space-induced norm for H(div) × H1)
-   // (∇⋅τ,∇⋅δτ)
-   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
-   // (τ,δτ)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   // (∇v,∇δv)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new MassIntegrator(one),1,1);
-
-   // additional terms for adjoint graph norm
-   if (adjoint_graph_norm)
-   {
-      // -(∇v,δτ) 
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
-      // -(τ,∇δv) 
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
-      // (τ,δτ)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   }
-   // RHS
-   if (prob == prob_type::general)
-   {
-      a->AddDomainLFIntegrator(new DomainLFIntegrator(one),1);
-   }
-
-   FunctionCoefficient uex(exact);
-   Array<int> elements_to_refine;
-   ParGridFunction hatu_gf;
-
-
-   socketstream u_out;
-   socketstream sigma_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      sigma_out.open(vishost, visport);
-   }
-
-   for (int i = 0; i<ref; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatsigma_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      if (prob == prob_type::lshape)
-      {
-         hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-         hatu_gf.ProjectBdrCoefficient(uex,ess_bdr);
-      }
-
-      Vector X,B;
-      OperatorPtr Ah;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
-      {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M->SetDiagonalBlock(0,amg0);
-         M->SetDiagonalBlock(1,amg1);
-         skip=2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
-      amg2->SetPrintLevel(0);
-      M->SetDiagonalBlock(skip,amg2);
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
-      }
-      M->SetDiagonalBlock(skip+1,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      
-      double maxresidual = residuals.Max(); 
-      double globalresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      globalresidual = sqrt(globalresidual);
-
-      if (myid == 0)
-      {
-         cout << "Global Residual = " << globalresidual << endl;
-      }
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      ParGridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      if (visualization)
-      {
-         u_out << "parallel " << num_procs << " " << myid << "\n";
-         u_out.precision(8);
-         u_out << "solution\n" << pmesh << u_gf <<
-                  "window_title 'Numerical u' "
-                  << flush;
-
-         sigma_out << "parallel " << num_procs << " " << myid << "\n";
-         sigma_out.precision(8);
-         sigma_out << "solution\n" << pmesh << sigma_gf <<
-                  "window_title 'Numerical flux' "
-                  << flush;
-      }
-
-
-      if (i == ref-1)
-      {
-         break;
-      }
-
-      pmesh.GeneralRefinement(elements_to_refine);
-
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatsigma_fes;
-   delete hatsigma_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete sigma_fes;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/grad-div,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = primal_dpg 
-PAR_EXAMPLES = 
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
@@ -1,176 +0,0 @@
-//                                MFEM primal dpg example for grad-dic problem
-//
-// Compile with: make primal_dpg
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, F, and r.h.s., f. See below for implementation.
-void F_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   kappa = freq * M_PI;
-
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   // mesh.UniformRefinement();
-
-   RT_FECollection fec(order-1, mesh.Dimension());
-   FiniteElementSpace RTfes(&mesh, &fec);
-
-   H1_Trace_FECollection trace_fec(order, mesh.Dimension());
-   FiniteElementSpace H1trace_fes(&mesh, &trace_fec);
-
-   int dim = mesh.Dimension();
-   int test_order = order;
-   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
-   {
-      test_order++;
-   }
-
-   test_order++;
-
-   RT_FECollection test_fec(test_order,mesh.Dimension());
-
-   Array<FiniteElementSpace *> trial_fes; 
-   Array<FiniteElementCollection * > test_fecs; 
-   
-   trial_fes.Append(&RTfes);
-   trial_fes.Append(&H1trace_fes);
-   test_fecs.Append(&test_fec);
-
-
-   GridFunction rt_gf(&RTfes);
-   VectorFunctionCoefficient F(dim, F_exact);
-   rt_gf.ProjectCoefficient(F);
-
-   Vector x(RTfes.GetVSize()+H1trace_fes.GetVSize());
-   x = 0.;
-   x.SetVector(rt_gf,0);
-
-
-   ConstantCoefficient alpha(1.0);
-   ConstantCoefficient beta(1.0);
-   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
-   a->AddTrialIntegrator(new DivDivIntegrator(alpha),0,0);
-   a->AddTrialIntegrator(new VectorFEMassIntegrator(beta),0,0);
-   a->AddTrialIntegrator(new NormalTraceIntegrator,1,0);
-   a->AddTestIntegrator(new DivDivIntegrator(alpha),0,0);
-   a->AddTestIntegrator(new VectorFEMassIntegrator(beta),0,0);
-
-
-   VectorFunctionCoefficient f(dim, f_exact);
-   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
-   a->Assemble();
-
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      RTfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   Vector X,B;
-
-   OperatorPtr Ah;
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   delete M;
-
-   a->RecoverFEMSolution(X,x);
-
-   // GridFunction u_gf;
-   double *data = x.GetData();
-   rt_gf.MakeRef(&RTfes,data);
-
-   GridFunction exact_gf(&RTfes);
-   exact_gf.ProjectCoefficient(F);
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream solu_sock(vishost, visport);
-   solu_sock.precision(8);
-   solu_sock << "solution\n" << mesh << rt_gf <<
-             "window_title 'Numerical u' "
-             << flush;
-
-   socketstream soltrace_sock(vishost, visport);
-   soltrace_sock.precision(8);
-   soltrace_sock << "solution\n" << mesh << exact_gf <<
-                 "window_title 'Exact' "
-                 << flush;
-
-}
-
-
-// The exact solution (for non-surface meshes)
-void F_exact(const Vector &p, Vector &F)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
-
-   F(0) = cos(kappa*x)*sin(kappa*y);
-   F(1) = cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      F(2) = 0.0;
-   }
-}
-
-// The right hand side
-void f_exact(const Vector &p, Vector &f)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
-
-   double temp = 1 + 2*kappa*kappa;
-
-   f(0) = temp*cos(kappa*x)*sin(kappa*y);
-   f(1) = temp*cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      f(2) = 0;
-   }
-}
@@ -1,51 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-3
-1 3 0 1 4 3
-1 3 3 4 7 6
-1 3 1 2 5 4
-
-boundary
-8
-1 1 0 1
-1 1 1 2
-1 1 2 5
-2 1 5 4
-2 1 4 7
-1 1 7 6
-1 1 6 3
-1 1 3 0
-
-vertices
-8
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 1
-
-1 1
-1 -0
-1 -1
-0 1
-0 -0
-0 -1
-1 1
-1 -0
@@ -1,907 +0,0 @@
-//                   MFEM Ultraweak DPG Maxwell example
-//
-// Compile with: make complex_uw_dpg
-//
-
-//      ∇×(1/μ ∇×E) - ω^2 ϵ E = Ĵ ,   in Ω
-//                E×n = E_0, on ∂Ω
-
-// First Order System
-
-//  i ω μ H + ∇ × E = 0,   in Ω
-// -i ω ϵ E + ∇ × H = J,   in Ω
-//            E × n = E_0, on ∂Ω
-
-// note: Ĵ = -iωJ
-// in 2D 
-// E is vector valued and H is scalar. 
-//    (∇ × E, F) = (E, ∇ × F) + < n × E , F>
-// or (∇ ⋅ AE , F) = (AE, ∇ F) + < AE ⋅ n, F>
-// where A = A = [0 1; -1 0];
-
-// UW-DPG:
-// 
-// in 3D 
-// E,H ∈ (L^2(Ω))^3 
-// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)  
-//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)      
-// -i ω ϵ (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)      
-//                                   Ê × n = E_0     on ∂Ω 
-// -------------------------------------------------------------------------
-// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
-// -------------------------------------------------------------------------
-// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
-// |   |              |             |              |              |         |
-// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |  
-// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
-
-// in 2D 
-// E ∈ L^2(Ω)^2, H ∈ L^2(Ω)
-// Ê ∈ H^-1/2(Ω)(Γ_h), Ĥ ∈ H^1/2(Γ_h)  
-//  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H^1      
-// -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)      
-//                                    Ê = E_0     on ∂Ω 
-// -------------------------------------------------------------------------
-// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
-// -------------------------------------------------------------------------
-// | F |  (E,∇ × F)   | i ω μ (H,F) |   < Ê, F >   |              |         |
-// |   |              |             |              |              |         |
-// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < Ĥ, G × n > |  (J,G)  |  
-
-// where (F,G) ∈  H^1 × H(curl,Ω)
-
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-void E_exact_r(const Vector &x, Vector & E_r);
-void E_exact_i(const Vector &x, Vector & E_i);
-
-void H_exact_r(const Vector &x, Vector & H_r);
-void H_exact_i(const Vector &x, Vector & H_i);
-
-
-void  rhs_func_r(const Vector &x, Vector & J_r);
-void  rhs_func_i(const Vector &x, Vector & J_i);
-
-void curlE_exact_r(const Vector &x, Vector &curlE_r);
-void curlE_exact_i(const Vector &x, Vector &curlE_i);
-void curlH_exact_r(const Vector &x,Vector &curlH_r);
-void curlH_exact_i(const Vector &x,Vector &curlH_i);
-
-void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
-void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
-
-void hatE_exact_r(const Vector & X, Vector & hatE_r);
-void hatE_exact_i(const Vector & X, Vector & hatE_i);
-
-void hatH_exact_r(const Vector & X, Vector & hatH_r);
-void hatH_exact_i(const Vector & X, Vector & hatH_i);
-
-double hatH_exact_scalar_r(const Vector & X);
-double hatH_exact_scalar_i(const Vector & X);
-
-void maxwell_solution(const Vector & X, 
-                      std::vector<complex<double>> &E, 
-                      std::vector<complex<double>> &curlE, 
-                      std::vector<complex<double>> &curlcurlE);
-
-void maxwell_solution_r(const Vector & X, Vector &E_r, 
-                      Vector &curlE_r, 
-                      Vector &curlcurlE_r);
-
-void maxwell_solution_i(const Vector & X, Vector &E_i, 
-                      Vector &curlE_i, 
-                      Vector &curlcurlE_i);     
-
-int dim;
-int dimc;
-double omega;
-double mu = 1.0;
-double epsilon = 1.0;
-
-enum prob_type
-{
-   polynomial,
-   plane_wave,
-   fichera_oven  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   const char *mesh_file = "../../../data/inline-hex.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   bool static_cond = false;
-   int iprob = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");    
-   args.AddOption(&mu, "-mu", "--permeability",
-                  "Permeability of free space (or 1/(spring constant)).");
-   args.AddOption(&epsilon, "-eps", "--permittivity",
-                  "Permittivity of free space (or mass constant).");                  
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: polynomial, 1: plane wave, 2: Gaussian beam");                     
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");       
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                                            
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 2) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-   omega = 2.*M_PI*rnum;
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-   dimc = (dim == 3) ? 3 : 1;
-   int test_order = order+delta_order;
-
-   // Define spaces
-   // L2 space for E
-   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *E_fes = new FiniteElementSpace(&mesh,E_fec,dim);
-
-   // Vector L2 space for H 
-   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *H_fes = new FiniteElementSpace(&mesh,H_fec, dimc); 
-
-   // H^-1/2 (curl) space for Ê   
-   FiniteElementCollection * hatE_fec = nullptr;
-   FiniteElementCollection * hatH_fec = nullptr; 
-   FiniteElementCollection * F_fec = nullptr;
-   if (dim == 3)
-   {
-      hatE_fec = new ND_Trace_FECollection(order,dim);
-      hatH_fec = new ND_Trace_FECollection(order,dim);   
-      F_fec = new ND_FECollection(test_order, dim);
-   }
-   else
-   {
-      hatE_fec = new RT_Trace_FECollection(order-1,dim);
-      hatH_fec = new H1_Trace_FECollection(order,dim);   
-      F_fec = new H1_FECollection(test_order, dim);
-   } 
-   FiniteElementSpace *hatE_fes = new FiniteElementSpace(&mesh,hatE_fec);
-   FiniteElementSpace *hatH_fes = new FiniteElementSpace(&mesh,hatH_fec);
-
-   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
-
-
-   mfem::out << "E_fes space true dofs = " << E_fes->GetTrueVSize() << endl;
-   mfem::out << "H_fes space true dofs = " << H_fes->GetTrueVSize() << endl;
-   mfem::out << "hatE_fes space true dofs = " << hatE_fes->GetTrueVSize() << endl;
-   mfem::out << "hatH_fes space true dofs = " << hatH_fes->GetTrueVSize() << endl;
-
-
-//    // Coefficients
-   Vector dim_zero(dim); dim_zero = 0.0;
-   Vector dimc_zero(dimc); dimc_zero = 0.0;
-   VectorConstantCoefficient E_zero(dim_zero);
-   VectorConstantCoefficient H_zero(dimc_zero);
-
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
-   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
-   ConstantCoefficient muomeg(mu*omega);
-   ConstantCoefficient negepsomeg(-epsilon*omega);
-   ConstantCoefficient epsomeg(epsilon*omega);
-   ConstantCoefficient negmuomeg(-mu*omega);
-
-   DenseMatrix rot_mat(2);
-   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
-   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
-   MatrixConstantCoefficient rot(rot_mat);
-   ScalarMatrixProductCoefficient epsrot(epsomeg,rot);
-   ScalarMatrixProductCoefficient negepsrot(negepsomeg,rot);
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(E_fes);
-   trial_fes.Append(H_fes);
-   trial_fes.Append(hatE_fes);
-   trial_fes.Append(hatH_fes);
-
-   test_fec.Append(F_fec);
-   test_fec.Append(G_fec);
-
-   ComplexNormalEquations * a = new ComplexNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices();
-
-   // (E,∇ × F)
-   a->AddTrialIntegrator(new TransposeIntegrator(new CurlIntegrator(one)),nullptr,0,0);
-
-   // -i ω ϵ (E , G)
-   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(negepsomeg)),0,1);
-
-   // i ω μ (H, F)
-   if (dim == 3)
-   {
-      a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
-   }
-   else
-   {
-      a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(muomeg),1,0);
-   }
-   //  (H,∇ × G) 
-   a->AddTrialIntegrator(new TransposeIntegrator(new CurlIntegrator(one)),nullptr,1,1);
-
-   // < n×Ê,F>
-   if (dim == 3)
-   {
-      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
-   }
-   else
-   {
-      a->AddTrialIntegrator(new TraceIntegrator,nullptr,2,0);
-   }
-
-   // < n×Ĥ ,G>
-   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
-
-
-   // test integrators 
-
-   //space-induced norm for H(curl) × H(curl)
-   if (dim == 3)
-   {
-      // (∇×F,∇×δF)
-      a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
-      // (F,δF)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
-   }
-   else
-   {
-      // (∇F,∇δF)
-      a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
-      // (F,δF)
-      a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
-   }
-
-   // (∇×G ,∇× δG)
-   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
-   // (G,δG)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      if(dim == 3)
-      {
-         // μ^2 ω^2 (F,δF)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
-         // -i ω μ (F,∇ × δG) = (F, ω μ ∇ × δ G)
-         a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
-         // -i ω ϵ (∇ × F, δG)
-         a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(negepsomeg),0,1);
-         // i ω μ (∇ × G,δF)
-         a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(epsomeg),1,0);
-         // i ω ϵ (G, ∇ × δF )
-         a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(muomeg),1,0);
-         // ϵ^2 ω^2 (G,δG)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);
-      }
-      else
-      {
-         // μ^2 ω^2 (F,δF)
-         a->AddTestIntegrator(new MassIntegrator(mu2omeg2),nullptr,0,0);
-
-         // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
-         a->AddTestIntegrator(nullptr,
-            new TransposeIntegrator(new CurlIntegrator(negmuomeg)),0,1);
-
-         // -i ω ϵ (∇ × F, δG) = i (- ω ϵ A ∇ F,δG), A = [0 1; -1; 0]
-         a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negepsrot),0,1);   
-
-         // i ω μ (∇ × G,δF) = i (ω μ ∇ × G, δF )
-         a->AddTestIntegrator(nullptr,new CurlIntegrator(muomeg),1,0);
-
-         // i ω ϵ (G, ∇ × δF ) =  i (ω ϵ G, A ∇ δF) = i ( G , ω ϵ A ∇ δF) 
-         a->AddTestIntegrator(nullptr,
-                   new TransposeIntegrator(new MixedVectorGradientIntegrator(epsrot)),1,0);
-
-         // or    i ( ω ϵ A^t G, ∇ δF) = i (- ω ϵ A G, ∇ δF)
-         // a->AddTestIntegrator(nullptr,
-                  //  new MixedVectorWeakDivergenceIntegrator(epsrot),1,0);
-         // ϵ^2 ω^2 (G,δG)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);            
-      }
-   }
-
-   // RHS
-   VectorFunctionCoefficient f_rhs_r(dim,rhs_func_r);
-   VectorFunctionCoefficient f_rhs_i(dim,rhs_func_i);
-   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f_rhs_r),
-                            new VectorFEDomainLFIntegrator(f_rhs_i),1);
-   
-   
-   VectorFunctionCoefficient hatEex_r(dim,hatE_exact_r);
-   VectorFunctionCoefficient hatEex_i(dim,hatE_exact_i);
-
-   VectorFunctionCoefficient hatHex_r(dimc,hatH_exact_r);
-   VectorFunctionCoefficient hatHex_i(dimc,hatH_exact_i);
-
-   FunctionCoefficient hatH_2D_ex_r(hatH_exact_scalar_r);
-   FunctionCoefficient hatH_2D_ex_i(hatH_exact_scalar_i);
-
-
-   Array<int> elements_to_refine;
-
-   socketstream E_out_r;
-   socketstream Eex_out_r;
-   // socketstream E_out_i;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      E_out_r.open(vishost, visport);
-      Eex_out_r.open(vishost, visport);
-      // E_out_i.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-         // hatH_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int j = 0; j < ess_tdof_list.Size(); j++)
-      {
-         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
-                           // + hatE_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = E_fes->GetVSize();
-      offsets[2] = H_fes->GetVSize();
-      offsets[3] = hatE_fes->GetVSize();
-      offsets[4] = hatH_fes->GetVSize();
-      offsets.PartialSum();
-
-      Vector x(2*offsets.Last());
-      x = 0.;
-      double * xdata = x.GetData();
-
-      ComplexGridFunction hatE_gf(hatE_fes);
-      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
-      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
-
-      ComplexGridFunction hatH_gf(hatH_fes);
-      hatH_gf.real().MakeRef(hatH_fes,&xdata[offsets[3]]);
-      hatH_gf.imag().MakeRef(hatH_fes,&xdata[offsets.Last()+ offsets[3]]);
-
-      if (dim == 3)
-      {
-         hatE_gf.ProjectBdrCoefficientTangent(hatEex_r,hatEex_i, ess_bdr);
-         // hatH_gf.ProjectBdrCoefficientTangent(hatHex_r,hatHex_i, ess_bdr);
-      }
-      else
-      {
-         hatE_gf.ProjectBdrCoefficientNormal(hatEex_r,hatEex_i, ess_bdr);
-         // hatH_gf.ProjectBdrCoefficient(hatH_2D_ex_r,hatH_2D_ex_i, ess_bdr);
-
-      }
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
-
-      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
-
-      SparseMatrix * Ar = dynamic_cast<BlockMatrix *>(&Ahc->real())->CreateMonolithic();
-      SparseMatrix * Ai = dynamic_cast<BlockMatrix *>(&Ahc->imag())->CreateMonolithic();
-
-      ComplexSparseMatrix Ac(Ar,Ai,true,true);
-      SparseMatrix * A = Ac.GetSystemMatrix();
-
-      UMFPackSolver umf(*A);
-      umf.Mult(B,X);
-
-      delete A;
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-      double residual = residuals.Norml2();
-
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-
-      ComplexGridFunction E(E_fes);
-      E.real().MakeRef(E_fes,x.GetData());
-      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
-
-      VectorFunctionCoefficient E_ex_r(dim,E_exact_r);
-      VectorFunctionCoefficient E_ex_i(dim,E_exact_i);
-
-      ComplexGridFunction H(H_fes);
-      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
-      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
-
-      VectorFunctionCoefficient H_ex_r(dimc,H_exact_r);
-      VectorFunctionCoefficient H_ex_i(dimc,H_exact_i);
-      
-
-      int dofs = X.Size()/2;
-
-      double E_err_r = E.real().ComputeL2Error(E_ex_r);
-      double E_err_i = E.imag().ComputeL2Error(E_ex_i);
-      double H_err_r = H.real().ComputeL2Error(H_ex_r);
-      double H_err_i = H.imag().ComputeL2Error(H_ex_i);
-
-      double L2Error = sqrt(  E_err_r*E_err_r + E_err_i*E_err_i 
-                            + H_err_r*H_err_r + H_err_i*H_err_i );
-
-      ComplexGridFunction Egf_ex(E_fes);
-      ComplexGridFunction Hgf_ex(H_fes);
-      Egf_ex.ProjectCoefficient(E_ex_r, E_ex_i);
-      Hgf_ex.ProjectCoefficient(H_ex_r, H_ex_i);
-
-   
-
-      double E_norm_r = Egf_ex.real().ComputeL2Error(E_zero);
-      double E_norm_i = Egf_ex.imag().ComputeL2Error(E_zero);
-      double H_norm_r = Hgf_ex.real().ComputeL2Error(H_zero);
-      double H_norm_i = Hgf_ex.imag().ComputeL2Error(H_zero);
-
-      double L2norm = sqrt(  E_norm_r*E_norm_r + E_norm_i*E_norm_i 
-                           + H_norm_r*H_norm_r + H_norm_i*H_norm_i );
-
-      double rel_err = L2Error/L2norm;
-
-
-      double rate_err = (i) ? dim*log(err0/rel_err)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = rel_err;
-      res0 = residual;
-      dof0 = dofs;
-
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  rel_err*100 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::setw(10) << std::scientific 
-                << std::endl;
-
-      if (visualization)
-      {
-         E_out_r.precision(8);
-         E_out_r << "solution\n" << mesh << E.real() <<
-                  "window_title 'Real Numerical Electric field' "
-                  << flush;
-
-         // E_out_i.precision(8);
-         // E_out_i << "solution\n" << mesh << E.imag() <<
-         //          "window_title 'Imag Numerical Electric field' "
-         //          << flush;         
-
-
-
-
-         Eex_out_r.precision(8);
-         Eex_out_r << "solution\n" << mesh << Egf_ex.real()  
-                  << "window_title 'Real Exact Electric field' " 
-                  << flush;
-         // socketstream E_i_sock(vishost, visport);
-         // E_i_sock.precision(8);
-         // E_i_sock << "solution\n" << mesh << Egf_ex.imag()  
-         //          << "window_title 'Imag Exact Electric field' " 
-         //          << flush;
-
-
-      }
-
-      if (i == ref-1)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete F_fec;
-   delete G_fec;
-   delete hatH_fes;
-   delete hatH_fec;
-   delete hatE_fes;
-   delete hatE_fec;
-   delete H_fec;
-   delete E_fec;
-   delete H_fes;
-   delete E_fes;
-
-   return 0;
-}                                       
-
-void E_exact_r(const Vector &x, Vector & E_r)
-{
-   Vector curlE_r;
-   Vector curlcurlE_r;
-
-   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
-}
-
-void E_exact_i(const Vector &x, Vector & E_i)
-{
-   Vector curlE_i;
-   Vector curlcurlE_i;
-
-   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
-}
-
-void curlE_exact_r(const Vector &x, Vector &curlE_r)
-{
-   Vector E_r;
-   Vector curlcurlE_r;
-
-   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
-}
-
-void curlE_exact_i(const Vector &x, Vector &curlE_i)
-{
-   Vector E_i;
-   Vector curlcurlE_i;
-
-   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
-}
-
-void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
-{
-   Vector E_r;
-   Vector curlE_r;
-   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
-}
-
-void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
-{
-   Vector E_i;
-   Vector curlE_i;
-   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
-}
-
-
-void H_exact_r(const Vector &x, Vector & H_r)
-{
-   // H = i ∇ × E / ω μ  
-   // H_r = - ∇ × E_i / ω μ  
-   Vector curlE_i;
-   curlE_exact_i(x,curlE_i);
-   H_r.SetSize(dimc);
-   for (int i = 0; i<dimc; i++)
-   {
-      H_r(i) = - curlE_i(i) / (omega * mu);
-   }
-}
-
-void H_exact_i(const Vector &x, Vector & H_i)
-{
-   // H = i ∇ × E / ω μ  
-   // H_i =  ∇ × E_r / ω μ  
-   Vector curlE_r;
-   curlE_exact_r(x,curlE_r);
-   H_i.SetSize(dimc);
-   for (int i = 0; i<dimc; i++)
-   {
-      H_i(i) = curlE_r(i) / (omega * mu);
-   }
-}
-
-void curlH_exact_r(const Vector &x,Vector &curlH_r)
-{
-   // ∇ × H_r = - ∇ × ∇ × E_i / ω μ  
-   Vector curlcurlE_i;
-   curlcurlE_exact_i(x,curlcurlE_i);
-   curlH_r.SetSize(dim);
-   for (int i = 0; i<dim; i++)
-   {
-      curlH_r(i) = -curlcurlE_i(i) / (omega * mu);
-   }
-}
-
-void curlH_exact_i(const Vector &x,Vector &curlH_i)
-{
-   // ∇ × H_i = ∇ × ∇ × E_r / ω μ  
-   Vector curlcurlE_r;
-   curlcurlE_exact_r(x,curlcurlE_r);
-   curlH_i.SetSize(dim);
-   for (int i = 0; i<dim; i++)
-   {
-      curlH_i(i) = curlcurlE_r(i) / (omega * mu);
-   }
-}
-
-
-void hatE_exact_r(const Vector & x, Vector & hatE_r)
-{
-   if (dim == 3)
-   {
-      E_exact_r(x,hatE_r);
-   }
-   else
-   {
-      Vector E_r;
-      E_exact_r(x,E_r);
-      hatE_r.SetSize(hatE_r.Size());
-      // rotate E_hat
-      hatE_r[0] = E_r[1];
-      hatE_r[1] = -E_r[0];
-   }
-}
-
-void hatE_exact_i(const Vector & x, Vector & hatE_i)
-{
-   if (dim == 3)
-   {
-      E_exact_i(x,hatE_i);
-   }
-   else
-   {
-      Vector E_i;
-      E_exact_i(x,E_i);
-      hatE_i.SetSize(hatE_i.Size());
-      // rotate E_hat
-      hatE_i[0] = E_i[1];
-      hatE_i[1] = -E_i[0];
-   }
-}
-
-
-void hatH_exact_r(const Vector & x, Vector & hatH_r)
-{
-   H_exact_r(x,hatH_r);
-}
-
-void hatH_exact_i(const Vector & x, Vector & hatH_i)
-{
-   H_exact_i(x,hatH_i);
-}
-
-double hatH_exact_scalar_r(const Vector & x)
-{
-   Vector hatH_r;
-   H_exact_r(x,hatH_r);
-   return hatH_r[0];
-}
-
-double hatH_exact_scalar_i(const Vector & x)
-{
-   Vector hatH_i;
-   H_exact_i(x,hatH_i);
-   return hatH_i[0];
-}
-
-// J = -i ω ϵ E + ∇ × H 
-// J_r + iJ_i = -i ω ϵ (E_r + i E_i) + ∇ × (H_r + i H_i) 
-void  rhs_func_r(const Vector &x, Vector & J_r)
-{
-   // J_r = ω ϵ E_i + ∇ × H_r
-   Vector E_i, curlH_r;
-   E_exact_i(x,E_i);
-   curlH_exact_r(x,curlH_r);
-   J_r.SetSize(dim);
-   for (int i = 0; i<dim; i++)
-   {
-      J_r(i) = omega * epsilon * E_i(i) + curlH_r(i);
-   }
-}
-
-void  rhs_func_i(const Vector &x, Vector & J_i)
-{
-   // J_i = - ω ϵ E_r + ∇ × H_i
-   Vector E_r, curlH_i;
-   E_exact_r(x,E_r);
-   curlH_exact_i(x,curlH_i);
-   J_i.SetSize(dim);
-   for (int i = 0; i<dim; i++)
-   {
-      J_i(i) = -omega * epsilon * E_r(i) + curlH_i(i);
-   }
-}
-
-
-void maxwell_solution(const Vector & X, std::vector<complex<double>> &E, 
-                      std::vector<complex<double>> &curlE, 
-                      std::vector<complex<double>> &curlcurlE)
-{
-   double x = X(0);
-   double y = X(1);
-   double z;
-   if (dim == 3) z = X(2);
-
-   E.resize(dim);
-   curlE.resize(dimc);
-   curlcurlE.resize(dim);
-   switch (prob)
-   {
-   case prob_type::polynomial:
-   {
-      if (dim == 3)
-      {
-         E[0] = y * z * (1.0 - y) * (1.0 - z);
-         E[1] = x * y * z * (1.0 - x) * (1.0 - z);
-         E[2] = x * y * (1.0 - x) * (1.0 - y);
-         curlE[0] = (1.0 - x) * x * (y*(2.0*z-3.0)+1.0);
-         curlE[1] = 2.0*(1.0 - y)*y*(x-z);
-         curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
-         curlcurlE[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
-         curlcurlE[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
-         curlcurlE[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
-      }
-      else
-      {
-         E[0] = y * (1.0 - y);
-         E[1] = x * y * (1.0 - x);
-         curlE[0] = y*(3.0 - 2*x) - 1.0;
-         curlcurlE[0] = 3.0 - 2*x;
-         curlcurlE[1] = 2.0*y;
-      }
-   }
-      break;
-
-   case prob_type::plane_wave:
-   {
-      std::complex<double> zi(0,1);
-      std::complex<double> pw = exp(-zi * omega * (X.Sum()));
-      E[0] = pw;
-      E[1] = 0.0;
-      if (dim == 3)
-      {
-         E[2] = 0.0;
-         curlE[0] = 0.0;
-         curlE[1] = -zi * omega * pw;
-         curlE[2] =  zi * omega * pw;
-
-         curlcurlE[0] = 2.0 * omega * omega * pw;
-         curlcurlE[1] = - omega * omega * pw;
-         curlcurlE[2] = - omega * omega * pw;
-      }
-      else
-      {
-         curlE[0] = zi * omega * pw;
-         curlcurlE[0] =   omega * omega * pw;
-         curlcurlE[1] = - omega * omega * pw ;
-      }
-   }
-      break;
-
-   default:
-      MFEM_ABORT("Fichera 'oven' problem not implemented yet");
-      break;
-   }
-   
-}
-
-
-void maxwell_solution_r(const Vector & X, Vector &E_r, 
-                        Vector &curlE_r, 
-                        Vector &curlcurlE_r)
-{
-   E_r.SetSize(dim);
-   curlE_r.SetSize(dimc);
-   curlcurlE_r.SetSize(dim);
-
-   std::vector<complex<double>> E;
-   std::vector<complex<double>> curlE;
-   std::vector<complex<double>> curlcurlE;
-
-   maxwell_solution(X,E,curlE,curlcurlE);
-   for (int i = 0; i<dim ; i++)
-   {
-      E_r(i) = E[i].real();
-      curlcurlE_r(i) = curlcurlE[i].real();
-   }
-   for (int i = 0; i<dimc; i++)
-   {
-      curlE_r(i) = curlE[i].real();
-   }
-}
-
-
-void maxwell_solution_i(const Vector & X, Vector &E_i, 
-                      Vector &curlE_i, 
-                      Vector &curlcurlE_i)
-{
-   E_i.SetSize(dim);
-   curlE_i.SetSize(dimc);
-   curlcurlE_i.SetSize(dim);
-
-   std::vector<complex<double>> E;
-   std::vector<complex<double>> curlE;
-   std::vector<complex<double>> curlcurlE;
-
-   maxwell_solution(X,E,curlE,curlcurlE);
-   for (int i = 0; i<dim; i++)
-   {
-      E_i(i) = E[i].imag();
-      curlcurlE_i(i) = curlcurlE[i].imag();
-   }
-   for (int i = 0; i<dimc; i++)
-   {
-      curlE_i(i) = curlE[i].imag();
-   }
-}  
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = complex_uw_dpg complex_uw_dpg_2D
-PAR_EXAMPLES = pcomplex_uw_dpg
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
@@ -1,253 +0,0 @@
-// Test integrator
-// (∇ × E, F) 
-
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-void E_exact_r(const Vector &x, Vector & E_r);
-
-void curlE_exact_r(const Vector &x, Vector &curlE_r);
-
-void maxwell_solution(const Vector & X, 
-                      std::vector<complex<double>> &E, 
-                      std::vector<complex<double>> &curlE, 
-                      std::vector<complex<double>> &curlcurlE);
-
-void maxwell_solution_r(const Vector & X, Vector &E_r, 
-                      Vector &curlE_r, 
-                      Vector &curlcurlE_r);
-
-int dim;
-int dimc;
-double omega;
-
-enum prob_type
-{
-   polynomial,
-   plane_wave,
-   fichera_oven  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   const char *mesh_file = "../../../data/inline-hex.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   bool static_cond = false;
-   int iprob = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");    
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: polynomial, 1: plane wave, 2: Gaussian beam");                     
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");       
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                                            
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 2) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-   omega = 2.*M_PI*rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-   dimc = (dim == 3) ? 3 : 1;
-
-   // Define spaces
-   // L2 space for E
-   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
-   FiniteElementSpace *E_fes = new FiniteElementSpace(&mesh,E_fec);
-
-
-   FiniteElementCollection *curlE_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *curlE_fes = new FiniteElementSpace(&mesh,curlE_fec,dimc);
-
-   mfem::out << "E_fes space true dofs = " << E_fes->GetTrueVSize() << endl;
-   mfem::out << "curlE_fes space true dofs = " << curlE_fes->GetTrueVSize() << endl;
-
-
-   GridFunction E_gf(E_fes);
-   VectorFunctionCoefficient E_cf(dim,E_exact_r);
-   E_gf.ProjectCoefficient(E_cf);
-
-   GridFunction curlE_gf(curlE_fes);
-   VectorFunctionCoefficient curlE_cf(dimc,curlE_exact_r);
-   curlE_gf.ProjectCoefficient(curlE_cf);
-
-
-   char vishost[] = "localhost";
-   int visport = 19916;
-   socketstream E_sock(vishost, visport);
-   E_sock.precision(8);
-   E_sock << "solution\n"
-               << mesh << E_gf 
-               << "window_title 'Exact E'" << flush;
-
-   socketstream curlE_sock(vishost, visport);
-   curlE_sock.precision(8);
-   curlE_sock << "solution\n"
-               << mesh << curlE_gf 
-               << "window_title 'Exact curlE'" << flush;
-
-
-
-   MixedBilinearForm a(E_fes,curlE_fes);
-   a.AddDomainIntegrator(new CurlIntegrator());
-   a.Assemble();
-   Array<int> empty;
-   SparseMatrix A;
-   a.FormRectangularSystemMatrix(empty,empty,A);
-
-
-   Vector curl_load(A.Height());
-   A.Mult(E_gf,curl_load);
-   BilinearForm m(curlE_fes);
-   m.AddDomainIntegrator(new VectorMassIntegrator);
-   m.Assemble();
-   SparseMatrix M;
-   m.FormSystemMatrix(empty, M);
-
-
-   GSSmoother prec(M);
-   PCG(M, prec, curl_load, curlE_gf, 1, 200, 1e-12, 0.0);
-
-
-   socketstream curlE2_sock(vishost, visport);
-   curlE2_sock.precision(8);
-   curlE2_sock << "solution\n"
-               << mesh << curlE_gf 
-               << "window_title 'Numerical curlE'" << flush;
-
-
-   delete E_fec;
-   delete E_fes;
-   delete curlE_fec;
-   delete curlE_fes;
-
-   return 0;
-}                                       
-
-void E_exact_r(const Vector &x, Vector & E_r)
-{
-   Vector curlE_r;
-   Vector curlcurlE_r;
-
-   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
-}
-
-void curlE_exact_r(const Vector &x, Vector &curlE_r)
-{
-   Vector E_r;
-   Vector curlcurlE_r;
-
-   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
-}
-
-
-
-void maxwell_solution(const Vector & X, std::vector<complex<double>> &E, 
-                      std::vector<complex<double>> &curlE, 
-                      std::vector<complex<double>> &curlcurlE)
-{
-   double x = X(0);
-   double y = X(1);
-   double z;
-   if (dim == 3)
-   {
-      z = X(2);
-   } 
-
-   E.resize(dim);
-   curlE.resize(dimc);
-   curlcurlE.resize(dim);
-
-   if (dim == 3)
-   {
-      E[0] = y * z * (1.0 - y) * (1.0 - z);
-      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
-      E[2] = x * y * (1.0 - x) * (1.0 - y);
-      
-      curlE[0] = (1.0 - x) * x * (y*(2.0*z-3.0)+1.0);
-      curlE[1] = 2.0*(1.0 - y)*y*(x-z);
-      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
-      
-      curlcurlE[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
-      curlcurlE[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
-      curlcurlE[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
-   }
-   else if (dim == 2)
-   {
-      double c = 2.0*M_PI;
-      E[0] = sin(c * y);
-      E[1] = sin(c * x);
-      curlE[0] = c * (cos(c*x) - cos(c*y));
-      curlcurlE[0] = c*c * sin(c*y);
-      curlcurlE[1] = c*c * sin(c*x);
-   }
-   else
-   {
-      MFEM_ABORT("Dimension cannot be 1");
-   }
-
-}
-
-void maxwell_solution_r(const Vector & X, Vector &E_r, 
-                        Vector &curlE_r, 
-                        Vector &curlcurlE_r)
-{
-   E_r.SetSize(dim);
-   curlE_r.SetSize(dimc);
-   curlcurlE_r.SetSize(dim);
-
-   std::vector<complex<double>> E;
-   std::vector<complex<double>> curlE;
-   std::vector<complex<double>> curlcurlE;
-
-   maxwell_solution(X,E,curlE,curlcurlE);
-   for (int i = 0; i<dim; i++)
-   {
-      E_r(i) = E[i].real();
-      curlcurlE_r(i) = curlcurlE[i].real();
-   }
-   for (int i = 0; i<dimc; i++)
-   {
-      curlE_r(i) = curlE[i].real();
-   }
-}
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
-//               ex1 -fa -d cuda
+//             * ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -fa -d cuda
+//             * mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
@@ -195,7 +195,7 @@ int main(int argc, char *argv[])
   Array<int> ess_tdof_list(0);
   if (h1 && pmesh.bdr_attributes.Size())
   {
-      // For a continuous basis the linear system must be modifed to enforce an
+      // For a continuous basis the linear system must be modified to enforce an
      // essential (Dirichlet) boundary condition. In the DG case this is not
      // necessary as the boundary condition will only be enforced weakly.
      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
@@ -197,7 +197,6 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
-      B.EnsureMultTranspose();
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -72,8 +72,8 @@ int main(int argc, char *argv[])
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
   {
-      int ref_levels = 1;
-      // (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
@@ -147,8 +147,6 @@ int main(int argc, char *argv[])
   F.AddDomainIntegrator(new DomainLFIntegrator(one));
   F.Assemble();

-
-
   // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
   //    the mixed bilinear form for the interfacial unknowns, Bhat,
   //    the inverse stiffness matrix on the discontinuous test space, Sinv,
@@ -189,17 +187,10 @@ int main(int argc, char *argv[])
   // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
   //    the normal equation operator, A = B^t Sinv B, and
   //    the normal equation right-hand-size, b = B^t Sinv F.
-   // BlockOperator B(offsets_test, offsets);
-
-   BlockMatrix B(offsets_test, offsets);
+   BlockOperator B(offsets_test, offsets);
   B.SetBlock(0,0,&matB0);
   B.SetBlock(0,1,&matBhat);
-   SparseMatrix * Bh = B.CreateMonolithic();
-
-   SparseMatrix * A = RAP(*Bh, matSinv, *Bh);
-
-
-   // RAPOperator A(B, matSinv, B);
+   RAPOperator A(B, matSinv, B);
   {
      Vector SinvF(s_test);
      matSinv.Mult(F,SinvF);
@@ -243,20 +234,7 @@ int main(int argc, char *argv[])
   // 10. Solve the normal equation system using the PCG iterative solver.
   //     Check the weighted norm of residual for the DPG least square problem.
   //     Wrap the primal variable in a GridFunction for visualization purposes.
-
-
-   // PCG(*A, P, b, x, 1, 200, 1e-12, 0.0);
-
-   GSSmoother M(*A);
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(b, x);
-
+   PCG(A, P, b, x, 1, 200, 1e-12, 0.0);

   {
      Vector LSres(s_test);
@@ -187,7 +187,6 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();

@@ -248,7 +248,10 @@ int main(int argc, char *argv[])
   //     constraints for non-conforming AMR, static condensation, etc.
   if (myid == 0) { cout << "matrix ... " << flush; }
   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
+   // Here we want to try out block-size aware AMG solver in PETSc.
+   // For that to work properly, we need a fully-compliant block-size
+   // structure and we do not skip zeros when assembling.
+   a->Assemble(use_petsc ? 0 : 1);

   Vector B, X;
   if (!use_petsc)
@@ -294,13 +297,14 @@ int main(int argc, char *argv[])
         cout << "done." << endl;
         cout << "Size of linear system: " << A.M() << endl;
      }
-      PetscPCGSolver *pcg = new PetscPCGSolver(A);
+      // Tell PETSc the matrix has a block structure
+      A.SetBlockSize(dim);

-      // The preconditioner for the PCG solver defined below is specified in the
-      // PETSc config file, rc_ex2p, since a Krylov solver in PETSc can also
-      // customize its preconditioner.
+      // The preconditioner for the PCG solver can be specified in the
+      // PETSc config file
+      PetscPCGSolver *pcg = new PetscPCGSolver(A);
      PetscPreconditioner *prec = NULL;
-      if (use_nonoverlapping)
+      if (use_nonoverlapping) // Specialized BDDC construction
      {
         // Compute dofs belonging to the natural boundary
         Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());
@@ -450,7 +450,7 @@ int main(int argc, char *argv[])
      for (int ti = 0; !done; )
      {
         // We cannot match exactly the time history of the Run method
-         // since we are explictly telling PETSc to use a time step
+         // since we are explicitly telling PETSc to use a time step
         double dt_real = min(dt, t_final - t);
         ode_solver->Step(*U, t, dt_real);
         ti++;
@@ -78,6 +78,7 @@ EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly -
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
+EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -109,6 +110,7 @@ endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_ASM))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p
@@ -1,8 +1,7 @@
 -ksp_converged_reason

 # GAMG is still not used at its best,
-# since we are not exploiting the
-# block size (Ordering::byVDIM) and the RBMs
+# since we are not exploiting the RBMs

 -ksp_view
 -pc_type gamg
@@ -0,0 +1,10 @@
+# Additive Schwarz with Overlap
+# This is not a good solver for elasticity
+# These options are here only to describe
+# the setup of the solver
+-ksp_converged_reason
+-ksp_view
+-ksp_max_it 10
+-pc_type asm
+-pc_asm_overlap 1
+-sub_pc_type icc
@@ -210,6 +210,9 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,

 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
   // 1. Parse command-line options.
   const char *mesh_file = "../../data/beam-quad.mesh";
   int ref_levels = 2;
@@ -215,10 +215,11 @@ void visualize(ostream &os, ParMesh *mesh, ParGridFunction *deformed_nodes,

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
   Mpi::Init(argc, argv);
   int myid = Mpi::WorldRank();
   Hypre::Init();
+   Sundials::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/beam-quad.mesh";
@@ -109,6 +109,9 @@ double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
   // 1. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
   int ref_levels = 2;
@@ -290,7 +293,10 @@ int main(int argc, char *argv[])
         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 11)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
         ode_solver = arkode; break;
      case 12:
         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
@@ -101,11 +101,12 @@ double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();
+   Sundials::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
@@ -327,7 +328,10 @@ int main(int argc, char *argv[])
         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 11)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
         ode_solver = arkode; break;
      case 12:
         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
@@ -140,6 +140,9 @@ public:

 int main(int argc, char *argv[])
 {
+   // 0. Initialize SUNDIALS.
+   Sundials::Init();
+
   // 1. Parse command-line options.
   problem = 0;
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
@@ -408,7 +411,7 @@ int main(int argc, char *argv[])
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(FEHLBERG_13_7_8);
+         arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
         ode_solver = arkode; break;
   }

@@ -152,11 +152,12 @@ public:

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI and HYPRE.
+   // 1. Initialize MPI, HYPRE, and SUNDIALS.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();
+   Sundials::Init();

   // 2. Parse command-line options.
   problem = 0;
@@ -487,7 +488,10 @@ int main(int argc, char *argv[])
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
+         if (ode_solver_type == 9)
+         {
+            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         }
         ode_solver = arkode; break;
   }

@@ -35,7 +35,7 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 if (MFEM_ENABLE_TESTING)
  # Command line options for the tests.
  # Example 1: Test SuperLU on the simple Poisson problem
-  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
+  set(EX1_COMMON_OPTS -m ../../data/star.mesh)
  set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})

  # Add the tests: one test per source file.
@@ -39,6 +39,7 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
+  dgmassinv.cpp
  doftrans.cpp
  eltrans.cpp
  estimators.cpp
@@ -72,6 +73,7 @@ set(SRCS
  linearform.cpp
  linearform_ext.cpp
  lininteg.cpp
+  lininteg_boundary.cpp
  lininteg_domain.cpp
  lininteg_domain_grad.cpp
  lor/lor.cpp
@@ -88,6 +90,7 @@ set(SRCS
  fespacehierarchy.cpp
  nonlininteg_vectorconvection.cpp
  nonlininteg_vectorconvection_mf.cpp
+  qfunction.cpp
  qinterp/det.cpp
  qinterp/eval_by_nodes.cpp
  qinterp/eval_by_vdim.cpp
@@ -95,6 +98,7 @@ set(SRCS
  qinterp/grad_by_vdim.cpp
  qinterp/grad_phys_by_nodes.cpp
  qinterp/grad_phys_by_vdim.cpp
+  qspace.cpp
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
@@ -136,10 +140,13 @@ set(HDRS
  bilinearform.hpp
  bilinearform_ext.hpp
  bilininteg.hpp
+  bilininteg_mass_pa.hpp
  coefficient.hpp
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
+  dgmassinv.hpp
+  dgmassinv_kernels.hpp
  doftrans.hpp
  eltrans.hpp
  estimators.hpp
@@ -189,9 +196,11 @@ set(HDRS
  nonlinearform.hpp
  nonlinearform_ext.hpp
  nonlininteg.hpp
+  qfunction.hpp
  qinterp/dispatch.hpp
  qinterp/eval.hpp
  qinterp/grad.hpp
+  qspace.hpp
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
@@ -136,7 +136,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
         ext = new MFBilinearFormExtension(this);
         break;
      default:
-         mfem_error("Unknown assembly level");
+         MFEM_ABORT("BilinearForm: unknown assembly level");
   }
 }

@@ -992,6 +992,7 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
      mat_e = new SparseMatrix(height);
   }

+   vdofs_.HostRead();
   for (int i = 0; i < vdofs_.Size(); i++)
   {
      int vdof = vdofs_[i];
@@ -26,7 +26,8 @@ namespace mfem
 {

 /** @brief Enumeration defining the assembly level for bilinear and nonlinear
-    form classes derived from Operator. */
+    form classes derived from Operator. For more details, see
+    https://mfem.org/howto/assembly_levels */
 enum class AssemblyLevel
 {
   /// In the case of a BilinearForm LEGACY corresponds to a fully assembled
@@ -177,7 +178,7 @@ public:
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE

-       This method must be called before assembly. */
+       If used, this method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

   /// Returns the assembly level
@@ -333,7 +334,7 @@ public:


   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
+        to it.  Used for transferring ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
@@ -774,7 +775,7 @@ public:
   SparseMatrix &SpMat() { return *mat; }

   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
+        to it.  Used for transferring ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /// Adds a domain integrator. Assumes ownership of @a bfi.
@@ -18,6 +18,8 @@
 #include "pgridfunc.hpp"
 #include "ceed/interface/util.hpp"

+#include "../general/nvtx.hpp"
+
 namespace mfem
 {

@@ -160,7 +162,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultMF(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -176,7 +178,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultMF(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -217,7 +219,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposeMF(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -233,7 +235,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposeMF(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -289,6 +291,10 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)

 void PABilinearFormExtension::Assemble()
 {
+#undef MFEM_NVTX_COLOR
+#define MFEM_NVTX_COLOR NavyBlue
+   NVTX("HO Assemble");
+
   SetupRestrictionOperators(L2FaceValues::DoubleValued);

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
@@ -383,6 +389,10 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,

 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
+#undef MFEM_NVTX_COLOR
+#define MFEM_NVTX_COLOR MediumSpringGreen
+   NVTX("HO Apply");
+
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

   const int iSz = integrators.Size();
@@ -418,7 +428,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -434,7 +444,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -475,7 +485,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposePA(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -491,7 +501,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -668,7 +678,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -699,7 +709,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -796,7 +806,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -827,7 +837,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -975,6 +985,9 @@ void FABilinearFormExtension::RAP(OperatorHandle &A)
 void FABilinearFormExtension::EliminateBC(const Array<int> &ess_dofs,
                                          OperatorHandle &A)
 {
+   MFEM_VERIFY(a->diag_policy == DiagonalPolicy::DIAG_ONE,
+               "Only DiagonalPolicy::DIAG_ONE supported with"
+               " FABilinearFormExtension.");
 #ifdef MFEM_USE_MPI
   if ( dynamic_cast<ParBilinearForm*>(a) )
   {
@@ -144,14 +144,6 @@ void BilinearFormIntegrator::AssembleFaceMatrix (
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AssembleTraceFaceMatrix(int elem,
-                                                     const FiniteElement &trial_face_fe,  const FiniteElement &test_fe,
-                                                     FaceElementTransformations &Trans, DenseMatrix &elmat)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleTraceFaceMatrix(...)\n"
-               "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AssembleFaceMatrix(
   const FiniteElement &trial_face_fe, const FiniteElement &test_fe1,
   const FiniteElement &test_fe2, FaceElementTransformations &Trans,
@@ -801,84 +793,6 @@ const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
 }


-void CurlIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe,
-   ElementTransformation &Trans,  DenseMatrix &elmat)
-{
-   int dim = trial_fe.GetDim();
-   int trial_dof = trial_fe.GetDof();
-   int test_dof = test_fe.GetDof();
-   int dimc = (dim == 3) ? 3 : 1;
-
-   MFEM_ASSERT(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
-               dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE,
-               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
-   MFEM_ASSERT(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
-               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
-               "Test finite element must be in H1/L2");
-
-   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
-
-   if (spaceH1)
-   {
-      dshape.SetSize(trial_dof,dim);
-      curlshape.SetSize(dim*trial_dof,1);
-      dimc = dim;
-   }
-   else
-   {
-      curlshape.SetSize(trial_dof,dimc);
-      elmat_comp.SetSize(test_dof, trial_dof);
-   }
-   elmat.SetSize(dimc * test_dof, trial_dof);
-   shape.SetSize(test_dof);
-   elmat = 0.0;
-
-   double c;
-   Vector d_col;
-   const IntegrationRule *ir = IntRule;
-
-   if (ir == NULL)
-   {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-   }
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint(&ip);
-      if (spaceH1)
-      {
-         trial_fe.CalcPhysDShape(Trans, dshape);
-         dshape.GradToCurl(curlshape);
-      }
-      else
-      {
-         trial_fe.CalcPhysCurlShape(Trans, curlshape);
-      }
-      test_fe.CalcPhysShape(Trans, shape);
-      c = ip.weight*Trans.Weight();
-      if (Q)
-      {
-         c *= Q->Eval(Trans, ip);
-      }
-      shape *= c;
-
-      for (int d = 0; d < dimc; ++d)
-      {
-         double * curldata = &(curlshape.GetData())[d*trial_dof];
-         for (int jj = 0; jj < trial_dof; ++jj)
-         {
-            for (int ii = 0; ii < test_dof; ++ii)
-            {
-               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
-            }
-         }
-      }
-   }
-}
-
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
  DenseMatrix &elmat )
@@ -2089,7 +2003,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
   }
 }

-
 void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
                                                const FiniteElement &test_fe,
                                                ElementTransformation &Trans,
@@ -2104,7 +2017,7 @@ void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
 #ifdef MFEM_THREAD_SAFE
   Vector D;
   DenseMatrix curlshape(tr_nd,dimc), curlshape_dFt(tr_nd,dimc), M;
-   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc), M;
+   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc);
 #else
   curlshape.SetSize(tr_nd,dimc);
   curlshape_dFt.SetSize(tr_nd,dimc);
@@ -2404,6 +2317,84 @@ double VectorCurlCurlIntegrator::GetElementEnergy(
   return 0.5 * energy;
 }

+void MixedCurlIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans, DenseMatrix &elmat)
+{
+   int dim = trial_fe.GetDim();
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dimc = (dim == 3) ? 3 : 1;
+
+   MFEM_VERIFY(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
+               (dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE),
+               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
+   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
+               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
+               "Test finite element must be in H1/L2");
+
+   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
+
+   if (spaceH1)
+   {
+      dshape.SetSize(trial_dof,dim);
+      curlshape.SetSize(dim*trial_dof,1);
+      dimc = dim;
+   }
+   else
+   {
+      curlshape.SetSize(trial_dof,dimc);
+      elmat_comp.SetSize(test_dof, trial_dof);
+   }
+   elmat.SetSize(dimc * test_dof, trial_dof);
+   shape.SetSize(test_dof);
+   elmat = 0.0;
+
+   double c;
+   Vector d_col;
+   const IntegrationRule *ir = IntRule;
+
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+      if (spaceH1)
+      {
+         trial_fe.CalcPhysDShape(Trans, dshape);
+         dshape.GradToCurl(curlshape);
+      }
+      else
+      {
+         trial_fe.CalcPhysCurlShape(Trans, curlshape);
+      }
+      test_fe.CalcPhysShape(Trans, shape);
+      c = ip.weight*Trans.Weight();
+      if (Q)
+      {
+         c *= Q->Eval(Trans, ip);
+      }
+      shape *= c;
+
+      for (int d = 0; d < dimc; ++d)
+      {
+         double * curldata = &(curlshape.GetData())[d*trial_dof];
+         for (int jj = 0; jj < trial_dof; ++jj)
+         {
+            for (int ii = 0; ii < test_dof; ++ii)
+            {
+               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+            }
+         }
+      }
+   }
+}
+

 void VectorFEMassIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -2769,7 +2760,6 @@ void DivDivIntegrator::AssembleElementMatrix2(
 #endif
   elmat.SetSize(te_nd,tr_nd);

-
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
@@ -4007,214 +3997,6 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
   }
 }

-void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
-                                              const FiniteElement &trial_face_fe,
-                                              const FiniteElement &test_fe,
-                                              FaceElementTransformations & Trans,
-                                              DenseMatrix &elmat)
-{
-   int i, j, face_ndof, ndof;
-   int order;
-
-   face_ndof = trial_face_fe.GetDof();
-   ndof = test_fe.GetDof();
-
-   face_shape.SetSize(face_ndof);
-   shape.SetSize(ndof);
-
-   elmat.SetSize(ndof, face_ndof);
-   elmat = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      order = test_fe.GetOrder();
-      order += trial_face_fe.GetOrder();
-      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
-      {
-         order += Trans.OrderW();
-      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
-   }
-
-   int iel = Trans.Elem1->ElementNo;
-   if (iel != elem)
-   {
-      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
-   }
-
-   double scale = 1.0;
-   if (iel != elem) { scale = -1.; }
-   for (int p = 0; p < ir->GetNPoints(); p++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-      // Trace finite element shape function
-      trial_face_fe.CalcPhysShape(Trans,face_shape);
-
-      // Finite element shape function
-      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
-      test_fe.CalcPhysShape(*eltrans, shape);
-
-      face_shape *= Trans.Weight()*ip.weight;
-      for (i = 0; i < ndof; i++)
-      {
-         for (j = 0; j < face_ndof; j++)
-         {
-            elmat(i, j) += scale * shape(i) * face_shape(j);
-         }
-      }
-   }
-}
-
-void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
-                                                    const FiniteElement &trial_face_fe,
-                                                    const FiniteElement &test_fe,
-                                                    FaceElementTransformations &Trans,
-                                                    DenseMatrix &elmat)
-{
-   int i, j, face_ndof, ndof, dim;
-   int order;
-
-   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE, "");
-
-   face_ndof = trial_face_fe.GetDof();
-   ndof = test_fe.GetDof();
-   dim = test_fe.GetDim();
-
-   face_shape.SetSize(face_ndof);
-   normal.SetSize(dim);
-   shape.SetSize(ndof,dim);
-   shape_n.SetSize(ndof);
-
-   elmat.SetSize(ndof, face_ndof);
-   elmat = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      order = test_fe.GetOrder();
-      order += trial_face_fe.GetOrder();
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
-   }
-
-   int iel = Trans.Elem1->ElementNo;
-   if (iel != elem)
-   {
-      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
-   }
-
-   double scale = 1.0;
-   if (iel != elem) { scale = -1.; }
-
-   for (int p = 0; p < ir->GetNPoints(); p++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(p);
-      Trans.SetAllIntPoints(&ip);
-      trial_face_fe.CalcPhysShape(Trans, face_shape);
-      CalcOrtho(Trans.Jacobian(),normal);
-      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
-      test_fe.CalcVShape(*etrans, shape);
-      shape.Mult(normal, shape_n);
-      face_shape *= ip.weight;
-
-      for (i = 0; i < ndof; i++)
-      {
-         for (j = 0; j < face_ndof; j++)
-         {
-            elmat(i, j) += scale * shape_n(i) * face_shape(j);
-         }
-      }
-   }
-}
-
-void TangentTraceIntegrator::AssembleTraceFaceMatrix(int elem,
-                                                     const FiniteElement &trial_face_fe,
-                                                     const FiniteElement &test_fe,
-                                                     FaceElementTransformations & Trans,
-                                                     DenseMatrix &elmat)
-{
-
-   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::H_CURL, "");
-
-   int face_ndof, ndof, dim;
-   int order;
-   dim = test_fe.GetDim();
-   if (dim == 3)
-   {
-      std::string msg =
-         "Trial space should be ND face trace and test space should be a ND vector field in 3D ";
-      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::H_CURL &&
-                  trial_face_fe.GetDim() == 2 && test_fe.GetDim() == 3, msg);
-   }
-   else
-   {
-      std::string msg =
-         "Trial space should be H1 edge trace and test space should be a ND vector field in 2D";
-      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE &&
-                  trial_face_fe.GetDim() == 1 && test_fe.GetDim() == 2, msg);
-   }
-   face_ndof = trial_face_fe.GetDof();
-   ndof = test_fe.GetDof();
-
-   int dimc = (dim == 3) ? 3 : 1;
-
-   face_shape.SetSize(face_ndof,dimc);
-   shape_n.SetSize(ndof,dimc);
-   shape.SetSize(ndof,dim);
-   normal.SetSize(dim);
-   DenseMatrix face_shape_n(face_ndof,dimc);
-
-   elmat.SetSize(ndof, face_ndof);
-   elmat = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      order = test_fe.GetOrder();
-      order += trial_face_fe.GetOrder();
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
-   }
-
-   int iel = Trans.Elem1->ElementNo;
-   if (iel != elem)
-   {
-      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
-   }
-
-   double scale = 1.0;
-   if (iel != elem) { scale = -1.; }
-   for (int p = 0; p < ir->GetNPoints(); p++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(p);
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-      // Trace finite element shape function
-      if (dim == 3)
-      {
-         trial_face_fe.CalcVShape(Trans,face_shape);
-      }
-      else
-      {
-         face_shape.GetColumnReference(0,temp);
-         trial_face_fe.CalcPhysShape(Trans,temp);
-      }
-      CalcOrtho(Trans.Jacobian(),normal);
-      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
-      test_fe.CalcVShape(*eltrans, shape);
-
-      // rotate
-      cross_product(normal, shape, shape_n);
-
-      const double w = scale*ip.weight;
-      AddMult_a_ABt(w,shape_n, face_shape, elmat);
-   }
-}
-
-
-

 void NormalInterpolator::AssembleElementMatrix2(
   const FiniteElement &dom_fe, const FiniteElement &ran_fe,
@@ -151,12 +151,6 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

-   virtual void AssembleTraceFaceMatrix(int elem,
-                                        const FiniteElement &trial_face_fe,
-                                        const FiniteElement &test_fe,
-                                        FaceElementTransformations &Trans,
-                                        DenseMatrix &elmat);
-
   /** Abstract method used for assembling TraceFaceIntegrators in a
       MixedBilinearForm. */
   virtual void AssembleFaceMatrix(const FiniteElement &trial_face_fe,
@@ -221,10 +215,10 @@ public:
                             function by any coefficients describing the
                             integrator.
       @param[in] ir  If passed (the default value is NULL), the implementation
-                        of the method will ignore the integration rule provided
-                        by the @a fluxelem parameter and, instead, compute the
-                        discrete flux at the points specified by the integration
-                        rule @a ir.
+                      of the method will ignore the integration rule provided
+                      by the @a fluxelem parameter and, instead, compute the
+                      discrete flux at the points specified by the integration
+                      rule @a ir.
    */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
@@ -2075,31 +2069,6 @@ public:
                                         ElementTransformation &Trans);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is a
-    scalar coefficient, and v is a vector with components v_i in the L2 or H1 space.
-    u can be in H(curl) (2D or 3D) or it can be a scalar H1.
-    Note: If u is scalar H1 then curl u = [0 1; -1 0] grad u */
-class CurlIntegrator : public BilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-
-private:
-   Vector shape;
-   DenseMatrix dshape;
-   DenseMatrix curlshape;
-   DenseMatrix elmat_comp;
-public:
-   CurlIntegrator() : Q{NULL} { }
-   CurlIntegrator(Coefficient *q_) :  Q{q_} { }
-   CurlIntegrator(Coefficient &q) :  Q{&q} { }
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-};
-
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
    can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
@@ -2205,6 +2174,7 @@ public:
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
 class MassIntegrator: public BilinearFormIntegrator
 {
+   friend class DGMassInverse;
 protected:
 #ifndef MFEM_THREAD_SAFE
   Vector shape, te_shape;
@@ -2639,6 +2609,35 @@ public:
                                   const Vector &elfun);
 };

+/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
+    an optional scalar coefficient, and v is a vector with components v_i in
+    the L2 or H1 space. This integrator handles 3 cases:
+    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
+    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
+    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
+        2D vector field with components v_i in L^2 or H^1 space.
+    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
+class MixedCurlIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+
+private:
+   Vector shape;
+   DenseMatrix dshape;
+   DenseMatrix curlshape;
+   DenseMatrix elmat_comp;
+public:
+   MixedCurlIntegrator() : Q{NULL} { }
+   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
+   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+};
+
 /** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
@@ -2780,10 +2779,12 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
+
   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
+
   const Coefficient *GetCoefficient() const { return Q; }
 };

@@ -3300,88 +3301,6 @@ public:
                                   DenseMatrix &elmat);
 };

-
-/** Integrator for the DPG form: < v, w > over a face (the interface) where
-    the trial variable v is defined on the interface
-    (H^-1/2 i.e., v:=u⋅n normal trace of H(div))
-    and the test variable w is in an H1-conforming space. */
-class TraceIntegrator : public BilinearFormIntegrator
-{
-private:
-   Vector face_shape, shape;
-public:
-   TraceIntegrator() { }
-   void AssembleTraceFaceMatrix(int elem,
-                                const FiniteElement &trial_face_fe,
-                                const FiniteElement &test_fe,
-                                FaceElementTransformations &Trans,
-                                DenseMatrix &elmat);
-};
-
-/** Integrator for the form: < v, w.n > over a face (the interface) where
-    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
-    and the test variable w is in an H(div)-conforming space. */
-class NormalTraceIntegrator : public BilinearFormIntegrator
-{
-private:
-   Vector face_shape, normal, shape_n;
-   DenseMatrix shape;
-
-public:
-   NormalTraceIntegrator() { }
-   virtual void AssembleTraceFaceMatrix(int ielem,
-                                        const FiniteElement &trial_face_fe,
-                                        const FiniteElement &test_fe,
-                                        FaceElementTransformations &Trans,
-                                        DenseMatrix &elmat);
-};
-
-
-/** Integrator for the form: < v, w × n > over a face (the interface)
- *  In 3D the trial variable v is defined on the interface (H^-1/2(curl), trace of H(curl))
- *  In 2D it's defined on the interface (H^1/2, trace of H1)
- *  The test variable w is in an H(curl)-conforming space. */
-class TangentTraceIntegrator : public BilinearFormIntegrator
-{
-private:
-   DenseMatrix face_shape, shape, shape_n;
-   Vector normal;
-   Vector temp;
-
-   void cross_product(const Vector & x, const DenseMatrix & Y, DenseMatrix & Z)
-   {
-      int dim = x.Size();
-      MFEM_VERIFY(Y.Width() == dim, "Size missmatch");
-      int dimc = dim == 3 ? dim : 1;
-      int h = Y.Height();
-      Z.SetSize(h,dimc);
-      if (dim == 3)
-      {
-         for (int i = 0; i<h; i++)
-         {
-            Z(i,0) = x(2) * Y(i,1) - x(1) * Y(i,2);
-            Z(i,1) = x(0) * Y(i,2) - x(2) * Y(i,0);
-            Z(i,2) = x(1) * Y(i,0) - x(0) * Y(i,1);
-         }
-      }
-      else
-      {
-         for (int i = 0; i<h; i++)
-         {
-            Z(i,0) = x(1) * Y(i,0) - x(0) * Y(i,1);
-         }
-      }
-   }
-
-public:
-   TangentTraceIntegrator() { }
-   void AssembleTraceFaceMatrix(int elem,
-                                const FiniteElement &trial_face_fe,
-                                const FiniteElement &test_fe,
-                                FaceElementTransformations &Trans,
-                                DenseMatrix &elmat);
-};
-
 /** Abstract class to serve as a base for local interpolators to be used in the
    DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"

@@ -1408,66 +1409,10 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * ne, mt);
-   Vector vel;
-   if (VectorConstantCoefficient *cQ =
-          dynamic_cast<VectorConstantCoefficient*>(Q))
-   {
-      vel = cQ->GetVec();
-   }
-   else if (VectorGridFunctionCoefficient *vgfQ =
-               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
-   {
-      vel.SetSize(dim * nq * ne, mt);

-      const GridFunction *gf = vgfQ->GetGridFunction();
-      const FiniteElementSpace &gf_fes = *gf->FESpace();
-      const QuadratureInterpolator *qi(gf_fes.GetQuadratureInterpolator(*ir));
-      const bool use_tensor_products = UsesTensorBasis(gf_fes);
-      const ElementDofOrdering ordering = use_tensor_products ?
-                                          ElementDofOrdering::LEXICOGRAPHIC :
-                                          ElementDofOrdering::NATIVE;
-      const Operator *R = gf_fes.GetElementRestriction(ordering);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);

-      Vector xe(R->Height(), mt);
-      xe.UseDevice(true);
-
-      R->Mult(*gf, xe);
-      qi->SetOutputLayout(QVectorLayout::byVDIM);
-      qi->DisableTensorProducts(!use_tensor_products);
-      qi->Values(xe,vel);
-   }
-   else if (VectorQuadratureFunctionCoefficient* vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      vel.SetSize(dim * nq * ne);
-      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      DenseMatrix MQ_ir;
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         Q->Eval(MQ_ir, T, *ir);
-         for (int q = 0; q < nq; ++q)
-         {
-            for (int i = 0; i < dim; ++i)
-            {
-               C(i,q,e) = MQ_ir(i,q);
-            }
-         }
-      }
-   }
   PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
                     vel, alpha, pa_data);
 }
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "restriction.hpp"

 using namespace std;
@@ -161,88 +162,24 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
-   Vector vel;
-   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
-                                        (u))
-   {
-      vel = c_u->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient* qf_u =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
-   {
-      // Assumed to be in lexicographical ordering
-      const QuadratureFunction &qFun = qf_u->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
+   FaceQuadratureSpace qs(*mesh, *ir, type);
+   CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);
+
+   CoefficientVector r(qs, CoefficientStorage::COMPRESSED);
+   if (rho == nullptr)
   {
-      vel.SetSize(dim * nq * nf);
-      auto C = Reshape(vel.HostWrite(), dim, nq, nf);
-      Vector Vq(dim);
-      int f_ind = 0;
-      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
-      {
-         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse())
-         {
-            // We skip nonconforming coarse faces as they are treated
-            // by the corresponding nonconforming fine faces.
-            continue;
-         }
-         else if ( face.IsOfFaceType(type) )
-         {
-            const int mask = FaceElementTransformations::HAVE_ELEM1 |
-                             FaceElementTransformations::HAVE_LOC1;
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                      quad1D, q);
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               u->Eval(Vq, *T.Elem1, eip1);
-               for (int i = 0; i < dim; ++i)
-               {
-                  C(i,iq,f_ind) = Vq(i);
-               }
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+      r.SetConstant(1.0);
   }
-   Vector r;
-   if (rho==nullptr)
+   else if (ConstantCoefficient *const_rho = dynamic_cast<ConstantCoefficient*>
+                                             (rho))
   {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
+      r.SetConstant(const_rho->constant);
   }
   else if (QuadratureFunctionCoefficient* qf_rho =
               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
   {
-      const QuadratureFunction &qFun = qf_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      r.MakeRef(qf_rho->GetQuadFunction());
   }
   else
   {
@@ -254,45 +191,42 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
      {
         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse())
+         if (face.IsNonconformingCoarse() || !face.IsOfFaceType(type))
         {
            // We skip nonconforming coarse faces as they are treated
            // by the corresponding nonconforming fine faces.
            continue;
         }
-         else if ( face.IsOfFaceType(type) )
+         FaceElementTransformations &T =
+            *fes.GetMesh()->GetFaceElementTransformations(f);
+         for (int q = 0; q < nq; ++q)
         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
-            for (int q = 0; q < nq; ++q)
+            // Convert to lexicographic ordering
+            int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                   quad1D, q);
+
+            T.SetAllIntPoints(&ir->IntPoint(q));
+            const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+            const IntegrationPoint &eip2 = T.GetElement2IntPoint();
+            double rq;
+
+            if (face.IsBoundary())
            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                      quad1D, q);
-
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-               double rq;
-
-               if ( face.IsBoundary() )
-               {
-                  rq = rho->Eval(*T.Elem1, eip1);
-               }
-               else
-               {
-                  double udotn = 0.0;
-                  for (int d=0; d<dim; ++d)
-                  {
-                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
-                  }
-                  if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
-                  else { rq = rho->Eval(*T.Elem1, eip1); }
-               }
-               C(iq,f_ind) = rq;
+               rq = rho->Eval(*T.Elem1, eip1);
            }
-            f_ind++;
+            else
+            {
+               double udotn = 0.0;
+               for (int d=0; d<dim; ++d)
+               {
+                  udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+               }
+               if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
+               else { rq = rho->Eval(*T.Elem1, eip1); }
+            }
+            C(iq,f_ind) = rq;
         }
+         f_ind++;
      }
      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
   }
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;
@@ -390,120 +391,21 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   int coeffDim = 1;
-   Vector coeff;
-   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
-   {
-      MFEM_VERIFY(SMQ->GetSize() == dim, "");
-      coeffDim = symmDims;
-      coeff.SetSize(symmDims * nq * ne);

-      DenseSymmetricMatrix sym_mat;
-      sym_mat.SetSize(dim);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);

-      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
+   if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (VQ) { coeff.Project(*VQ); }
+   else if (Q) { coeff.Project(*Q); }
+   else { coeff.SetConstant(1.0); }

-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
-            int cnt = 0;
-            for (int i=0; i<dim; ++i)
-               for (int j=i; j<dim; ++j, ++cnt)
-               {
-                  C(cnt, p, e) = sym_mat(i,j);
-               }
-         }
-      }
-   }
-   else if (MQ)
-   {
-      symmetric = false;
-      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dims*dims);
+   const int pa_size = symmetric ? symmDims : dims*dims;

-      coeffDim = MQfullDim;
-
-      coeff.SetSize(MQfullDim * nq * ne);
-
-      DenseMatrix mat;
-      mat.SetSize(dim);
-
-      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            MQ->Eval(mat, *tr, ir->IntPoint(p));
-            for (int i=0; i<dim; ++i)
-               for (int j=0; j<dim; ++j)
-               {
-                  C(j+(i*dim), p, e) = mat(i,j);
-               }
-         }
-      }
-   }
-   else if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "");
-      coeffDim = VQ->GetVDim();
-      coeff.SetSize(coeffDim * nq * ne);
-      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      Vector DM(coeffDim);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            VQ->Eval(DM, *tr, ir->IntPoint(p));
-            for (int i=0; i<coeffDim; ++i)
-            {
-               C(i, p, e) = DM[i];
-            }
-         }
-      }
-   }
-   else if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
+   pa_data.SetSize(pa_size * nq * ne, mt);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne, ir->GetWeights(),
                    geom->J, coeff, pa_data);
 }

@@ -1784,7 +1686,7 @@ static void PADiffusionApply(const int dim,
         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+            // default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }

@@ -1802,7 +1704,14 @@ static void PADiffusionApply(const int dim,
         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
-         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         case 0x33: return SmemPADiffusionApply3D<3,3>(NE,symm,B,G,D,X,Y);
+         case 0x44: return SmemPADiffusionApply3D<4,4>(NE,symm,B,G,D,X,Y);
+         case 0x55: return SmemPADiffusionApply3D<5,5>(NE,symm,B,G,D,X,Y);
+         case 0x66: return SmemPADiffusionApply3D<6,6>(NE,symm,B,G,D,X,Y);
+         case 0x77: return SmemPADiffusionApply3D<7,7>(NE,symm,B,G,D,X,Y);
+         case 0x88: return SmemPADiffusionApply3D<8,8>(NE,symm,B,G,D,X,Y);
+         case 0x99: return SmemPADiffusionApply3D<9,9>(NE,symm,B,G,D,X,Y);
+            // default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel: 0x"<<std::hex << id << std::dec);
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"

 using namespace std;

@@ -209,44 +210,8 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
               "PA requires test and trial space to have same number of quadrature points!");
   pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());

-   Vector coeff;
-
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *trial_fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);

   PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
                   ne, ir->GetWeights(), geom->J, coeff, pa_data);
@@ -865,4 +830,3 @@ void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 }

 } // namespace mfem
-
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qspace.hpp"

 using namespace std;

@@ -967,8 +968,6 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dim = mesh->Dimension();
   MFEM_VERIFY(dim == 2 || dim == 3, "");

-   const int dimc = (dim == 3) ? 3 : 1;
-
   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -978,88 +977,19 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }

-   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
-
-   symmetric = (SMQ || MQ == NULL);
-
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+   const int sym_dims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? sym_dims : dim*dim);
   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   Vector coeff(coeffDim * ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
-   {
-      Vector DM(DQ ? coeffDim : 0);
-      DenseMatrix GM;
-      DenseSymmetricMatrix SM;
-
-      if (DQ)
-      {
-         MFEM_VERIFY(coeffDim == dimc, "");
-      }
-      if (SMQ)
-      {
-         SM.SetSize(dimc);
-         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
-      }
-      else if (MQ)
-      {
-         GM.SetSize(dimc);
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            if (SMQ)
-            {
-               SMQ->Eval(SM, *tr, ir->IntPoint(p));
-
-               int cnt = 0;
-               for (int i=0; i<dimc; ++i)
-                  for (int j=i; j<dimc; ++j, ++cnt)
-                  {
-                     coeffh(cnt, p, e) = SM(i,j);
-                  }
-
-            }
-            else if (MQ)
-            {
-               MQ->Eval(GM, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dimc; ++i)
-                  for (int j=0; j<dimc; ++j)
-                  {
-                     coeffh(j+(i*dimc), p, e) = GM(i,j);
-                  }
-
-            }
-            else if (DQ)
-            {
-               DQ->Eval(DM, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = DM[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
-
   if (el->GetDerivType() != mfem::FiniteElement::CURL)
   {
      MFEM_ABORT("Unknown kernel.");
@@ -1067,7 +997,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   if (dim == 3)
   {
-      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
+      PACurlCurlSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J, coeff,
                        pa_data);
   }
   else
@@ -2780,7 +2710,7 @@ void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
   }
 }

-// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply3D(const int D1D,
                      const int Q1D,
@@ -2970,7 +2900,7 @@ void PAHcurlH1Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(curl), integrated
+// Apply to x corresponding to DOFs in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose3D(const int D1D,
                               const int Q1D,
@@ -3169,7 +3099,7 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply2D(const int D1D,
                      const int Q1D,
@@ -3293,7 +3223,7 @@ void PAHcurlH1Apply2D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(curl), integrated
+// Apply to x corresponding to DOFs in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose2D(const int D1D,
                               const int Q1D,
@@ -3489,20 +3419,8 @@ void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), nq, ne);
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeffh(p, e) = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   if (dim == 2)
   {
@@ -3593,38 +3511,11 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   const int ndata = curlSpaces ? (coeffDim == 1 ? 1 : 9) : symmDims;
   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   Vector coeff(coeffDim * nq * ne);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ)
-   {
-      Vector V(coeffDim);
-      if (DQ)
-      {
-         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-
-         for (int p=0; p<nq; ++p)
-         {
-            if (DQ)
-            {
-               DQ->Eval(V, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = V[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }

   if (testType == mfem::FiniteElement::CURL &&
       trialType == mfem::FiniteElement::CURL && dim == 3)
@@ -3652,7 +3543,7 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3D(const int D1D,
@@ -4015,7 +3906,7 @@ static void PAHcurlL2Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void SmemPAHcurlL2Apply3D(const int D1D,
@@ -4325,7 +4216,7 @@ static void SmemPAHcurlL2Apply3D(const int D1D,
   ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3D(const int D1D,
@@ -4681,7 +4572,7 @@ static void PAHcurlHdivApply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(div) (test), integrated against the
+// Apply to x corresponding to DOFs in H(div) (test), integrated against the
 // curl of H(curl) trial functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3DTranspose(const int D1D,
@@ -5146,38 +5037,11 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace

   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   Vector coeff(coeffDim * nq * ne);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ)
-   {
-      Vector V(coeffDim);
-      if (DQ)
-      {
-         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-
-         for (int p=0; p<nq; ++p)
-         {
-            if (DQ)
-            {
-               DQ->Eval(V, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = V[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }

   if (trialType == mfem::FiniteElement::CURL && dim == 3)
   {
@@ -5203,7 +5067,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
   }
 }

-// Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
+// Apply to x corresponding to DOFs in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3DTranspose(const int D1D,
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qspace.hpp"

 using namespace std;

@@ -1513,19 +1514,8 @@ void DivDivIntegrator::AssemblePA(const FiniteElementSpace &fes)

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
   {
@@ -1783,19 +1773,8 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   if (test_el->GetMapType() == FiniteElement::INTEGRAL)
   {
@@ -1818,7 +1797,7 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
 }

-// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply3D(const int D1D,
                            const int Q1D,
@@ -1981,7 +1960,7 @@ static void PAHdivL2Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply2D(const int D1D,
                            const int Q1D,
@@ -12,7 +12,9 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/mass/mass.hpp"
+#include "bilininteg_mass_pa.hpp"

 using namespace std;

@@ -60,43 +62,10 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(ne*nq, mt);
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
   if (dim==2)
   {
@@ -590,85 +559,18 @@ static void PAMassApply2D(const int NE,
                          const int d1d = 0,
                          const int q1d = 0)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b_.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto X = Reshape(x_.Read(), D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
   MFEM_FORALL(e, NE,
   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            sol_x[qy] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = X(dx,dy,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx)* s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double d2q = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += d2q * sol_x[qx];
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] *= D(qx,qy,e);
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0.0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xy[qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double q2d = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx,dy,e) += q2d * sol_x[dx];
-            }
-         }
-      }
+      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
   });
 }

@@ -690,108 +592,13 @@ static void SmemPAMassApply2D(const int NE,
   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
   MFEM_VERIFY(D1D <= MD1, "");
   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   const auto b = b_.Read();
+   const auto D = d_.Read();
+   const auto x = x_.Read();
+   auto Y = y_.ReadWrite();
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double BBt[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) BBt;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            X[dy][dx] = x(dx,dy,e);
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][dy] = b(q,dy);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double dq = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               dq += X[dy][dx] * B[qx][dx];
-            }
-            DQ[dy][qx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double qq = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               qq += DQ[dy][qx] * B[qy][dy];
-            }
-            QQ[qy][qx] = qq * D(qx, qy, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[dy][q] = b(q,dy);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dq = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dq += QQ[qy][qx] * Bt[dx][qx];
-            }
-            QD[qy][dx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dd = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               dd += (QD[qy][dx] * Bt[dy][qy]);
-            }
-            Y(dx, dy, e) += dd;
-         }
-      }
+      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
   });
 }

@@ -805,134 +612,18 @@ static void PAMassApply3D(const int NE,
                          const int d1d = 0,
                          const int q1d = 0)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b_.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
   MFEM_FORALL(e, NE,
   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         double sol_xy[max_Q1D][max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double sol_x[max_Q1D];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] = 0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = X(dx,dy,dz,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] += B(qx,dx) * s;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy = B(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] += wy * sol_x[qx];
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double wz = B(qz,dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double sol_xy[max_D1D][max_D1D];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] = 0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double sol_x[max_D1D];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] = 0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double s = sol_xyz[qz][qy][qx];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] += Bt(dx,qx) * s;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double wy = Bt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_xy[dy][dx] += wy * sol_x[dx];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            const double wz = Bt(dz,qz);
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-               }
-            }
-         }
-      }
+      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
   });
 }

@@ -953,213 +644,13 @@ static void SmemPAMassApply3D(const int NE,
   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
   MFEM_VERIFY(D1D <= M1D, "");
   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   auto b = b_.Read();
+   auto d = d_.Read();
+   auto x = x_.Read();
+   auto y = y_.ReadWrite();
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sDQ[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sDQ;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               X[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-         MFEM_FOREACH_THREAD(dx,x,Q1D)
-         {
-            B[dx][dy] = b(dx,dy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += X[dz][dy][dx] * B[qx][dx];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
-               {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += u[dz];
-            }
-         }
-      }
+      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
   });
 }

@@ -0,0 +1,632 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_MASS_PA_HPP
+#define MFEM_BILININTEG_MASS_PA_HPP
+
+#include "../config/config.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+template <bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void PAMassApply2D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
+   auto X = ConstDeviceCube(x_, D1D, D1D, NE);
+   auto Y = DeviceCube(y_, D1D, D1D, NE);
+
+   if (!ACCUMULATE)
+   {
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y(dx, dy, e) = 0.0;
+         }
+      }
+   }
+
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
+   double sol_xy[max_Q1D][max_Q1D];
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         sol_xy[qy][qx] = 0.0;
+      }
+   }
+   for (int dy = 0; dy < D1D; ++dy)
+   {
+      double sol_x[max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         sol_x[qy] = 0.0;
+      }
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         const double s = X(dx,dy,e);
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_x[qx] += B(qx,dx)* s;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         const double d2q = B(qy,dy);
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] += d2q * sol_x[qx];
+         }
+      }
+   }
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         sol_xy[qy][qx] *= D(qx,qy,e);
+      }
+   }
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      double sol_x[max_D1D];
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         sol_x[dx] = 0.0;
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         const double s = sol_xy[qy][qx];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] += Bt(dx,qx) * s;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         const double q2d = Bt(dy,qy);
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y(dx,dy,e) += q2d * sol_x[dx];
+         }
+      }
+   }
+}
+
+template<int T_D1D, int T_Q1D, int T_NBZ, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply2D_Element(const int e,
+                               const int NE,
+                               const double *b_,
+                               const double *d_,
+                               const double *x_,
+                               double *y_,
+                               int d1d = 0,
+                               int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
+   auto x = ConstDeviceCube(x_, D1D, D1D, NE);
+   auto Y = DeviceCube(y_, D1D, D1D, NE);
+
+   const int tidz = MFEM_THREAD_ID(z);
+
+   MFEM_SHARED double BBt[MQ1*MD1];
+   double (*B)[MD1] = (double (*)[MD1]) BBt;
+   double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
+   MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
+   MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
+   double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
+   double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
+   double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
+   double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
+
+
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         X[dy][dx] = x(dx,dy,e);
+      }
+   }
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            B[q][dy] = b(q,dy);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double dq = 0.0;
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            dq += X[dy][dx] * B[qx][dx];
+         }
+         DQ[dy][qx] = dq;
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double qq = 0.0;
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            qq += DQ[dy][qx] * B[qy][dy];
+         }
+         QQ[qy][qx] = qq * D(qx, qy, e);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[dy][q] = b(q,dy);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double dq = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dq += QQ[qy][qx] * Bt[dx][qx];
+         }
+         QD[qy][dx] = dq;
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double dd = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            dd += (QD[qy][dx] * Bt[dy][qy]);
+         }
+         if (ACCUMULATE)
+         {
+            Y(dx, dy, e) += dd;
+         }
+         else
+         {
+            Y(dx, dy, e) = dd;
+         }
+      }
+   }
+}
+
+template <bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void PAMassApply3D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d,
+                           const int q1d)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
+   auto X = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
+   auto Y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
+
+   if (!ACCUMULATE)
+   {
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx, dy, dz, e) = 0.0;
+            }
+         }
+      }
+   }
+
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
+   double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xyz[qz][qy][qx] = 0.0;
+         }
+      }
+   }
+   for (int dz = 0; dz < D1D; ++dz)
+   {
+      double sol_xy[max_Q1D][max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double sol_x[max_Q1D];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_x[qx] = 0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s = X(dx,dy,dz,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] += B(qx,dx) * s;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double wy = B(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] += wy * sol_x[qx];
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         const double wz = B(qz,dz);
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+            }
+         }
+      }
+   }
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
+         }
+      }
+   }
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      double sol_xy[max_D1D][max_D1D];
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_xy[dy][dx] = 0;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double sol_x[max_D1D];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] = 0;
+         }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double s = sol_xyz[qz][qy][qx];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] += Bt(dx,qx) * s;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            const double wy = Bt(dy,qy);
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_xy[dy][dx] += wy * sol_x[dx];
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         const double wz = Bt(dz,qz);
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
+            }
+         }
+      }
+   }
+}
+
+template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply3D_Element(const int e,
+                               const int NE,
+                               const double *b_,
+                               const double *d_,
+                               const double *x_,
+                               double *y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
+{
+   constexpr int D1D = T_D1D ? T_D1D : d1d;
+   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto d = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
+   auto x = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
+   auto y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
+
+   MFEM_SHARED double sDQ[MQ1*MD1];
+   double (*B)[MD1] = (double (*)[MD1]) sDQ;
+   double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
+   MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
+   MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
+   double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
+   double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
+   double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
+   double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
+   double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
+   double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            X[dz][dy][dx] = x(dx,dy,dz,e);
+         }
+      }
+      MFEM_FOREACH_THREAD(dx,x,Q1D)
+      {
+         B[dx][dy] = b(dx,dy);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] += X[dz][dy][dx] * B[qx][dx];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            DDQ[dz][dy][qx] = u[dz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            DQQ[dz][qy][qx] = u[dz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(di,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(q,x,Q1D)
+      {
+         Bt[di][q] = b(q,di);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            QQD[qz][qy][dx] = u[qz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            QDD[qz][dy][dx] = u[qz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            if (ACCUMULATE)
+            {
+               y(dx,dy,dz,e) += u[dz];
+            }
+            else
+            {
+               y(dx,dy,dz,e) = u[dz];
+            }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;
@@ -175,43 +176,9 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(!VQ && !MQ,
               "Only scalar coefficient supported for partial assembly for VectorDiffusionIntegrator");
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto Co = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            Co(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);

   const Array<double> &w = ir->GetWeights();
   const Vector &j = geom->J;
@@ -11,6 +11,7 @@

 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
+#include "qspace.hpp"
 #include "gridfunc.hpp"

 namespace mfem
@@ -793,140 +794,63 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   trial_fetype = trial_el->GetDerivType();
   test_fetype = test_el->GetDerivType();

-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
-
-   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
-
-   symmetric = (SMQ || MQ == NULL);
-
   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);

+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
+
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+
   if ((trial_curl && test_div) || (trial_div && test_curl))
-      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
+      pa_data.SetSize((coeff_dim == 1 ? 1 : dim*dim) * nq * ne,
                      Device::GetMemoryType());
   else
-      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+      pa_data.SetSize((symmetric ? symmDims : dims*dims) * nq * ne,
                      Device::GetMemoryType());

-   Vector coeff;
-
-   auto *qf_c = dynamic_cast<QuadratureFunctionCoefficient*>(Q);
-   if (qf_c)
-   {
-      const QuadratureFunction &qf = qf_c->GetQuadFunction();
-      qf.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction&>(qf), 0);
-   }
-   else
-   {
-      coeff.SetSize(coeffDim * ne * nq);
-      coeff = 1.0;
-      auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      if (Q || DQ || MQ)
-      {
-         Vector DM(DQ ? coeffDim : 0);
-         DenseMatrix M;
-         DenseSymmetricMatrix SM;
-
-         if (DQ)
-         {
-            MFEM_VERIFY(coeffDim == dim, "");
-         }
-         if (SMQ)
-         {
-            MFEM_VERIFY(SMQ->GetSize() == dim, "");
-            SM.SetSize(dim);
-         }
-         else if (MQ)
-         {
-            MFEM_VERIFY(coeffDim == MQdim, "");
-            MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-            M.SetSize(dim);
-         }
-
-         for (int e=0; e<ne; ++e)
-         {
-            ElementTransformation *tr = mesh->GetElementTransformation(e);
-            for (int p=0; p<nq; ++p)
-            {
-               if (SMQ)
-               {
-                  SMQ->Eval(SM, *tr, ir->IntPoint(p));
-                  int cnt = 0;
-                  for (int i=0; i<dim; ++i)
-                     for (int j=i; j<dim; ++j, ++cnt)
-                     {
-                        coeffh(cnt, p, e) = SM(i,j);
-                     }
-               }
-               else if (MQ)
-               {
-                  MQ->Eval(M, *tr, ir->IntPoint(p));
-
-                  for (int i=0; i<dim; ++i)
-                     for (int j=0; j<dim; ++j)
-                     {
-                        coeffh(j+(i*dim), p, e) = M(i,j);
-                     }
-               }
-               else if (DQ)
-               {
-                  DQ->Eval(DM, *tr, ir->IntPoint(p));
-                  for (int i=0; i<coeffDim; ++i)
-                  {
-                     coeffh(i, p, e) = DM[i];
-                  }
-               }
-               else
-               {
-                  coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-               }
-            }
-         }
-      }
-   }
-
   if (trial_curl && test_curl && dim == 3)
   {
-      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                         coeff, pa_data);
   }
   else if (trial_curl && test_curl && dim == 2)
   {
-      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup2D<2>(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                            coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 3)
   {
-      PAHdivSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 2)
   {
-      PAHdivSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup2D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
            test_fel->GetOrder() == trial_fel->GetOrder())
   {
-      if (coeffDim == 1)
+      if (coeff_dim == 1)
      {
-         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+         PAHcurlL2Setup(nq, coeff_dim, ne, ir->GetWeights(), coeff, pa_data);
      }
      else
      {
         const bool tr = (trial_div && test_curl);
         if (dim == 3)
-            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup3D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
                               geom->J, coeff, pa_data);
         else
-            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup2D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
                               geom->J, coeff, pa_data);
      }
   }
@@ -1168,19 +1092,8 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace

   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   // Use the same setup functions as VectorFEMassIntegrator.
   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
@@ -1,512 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-BlockBilinearForm::BlockBilinearForm(Array<FiniteElementSpace *> & fespaces_) :
-   Matrix(0), fespaces(fespaces_)
-{
-   height = 0;
-   nblocks = fespaces.Size();
-   dof_offsets.SetSize(nblocks+1);
-   tdof_offsets.SetSize(nblocks+1);
-   dof_offsets[0] = 0;
-   tdof_offsets[0] = 0;
-   for (int i =0; i<nblocks; i++)
-   {
-      dof_offsets[i+1] = fespaces[i]->GetVSize();
-      tdof_offsets[i+1] = fespaces[i]->GetTrueVSize();
-   }
-   dof_offsets.PartialSum();
-   tdof_offsets.PartialSum();
-   height = dof_offsets[nblocks];
-   width = height;
-   mat = mat_e = NULL;
-   extern_bfs = 0;
-   element_matrices = NULL;
-   diag_policy = DIAG_KEEP;
-}
-
-
-// Allocate appropriate SparseMatrix and assign it to mat
-void BlockBilinearForm::AllocMat()
-{
-   mat = new SparseMatrix(height);
-}
-
-void BlockBilinearForm::BuildProlongation()
-{
-   P = new BlockMatrix(dof_offsets, tdof_offsets);
-   R = new BlockMatrix(tdof_offsets, dof_offsets);
-   for (int i = 0; i<nblocks; i++)
-   {
-      const SparseMatrix *P_ = fespaces[i]->GetConformingProlongation();
-      const SparseMatrix *R_ = fespaces[i]->GetRestrictionMatrix();
-      P->SetBlock(i,i,const_cast<SparseMatrix*>(P_));
-      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
-   }
-}
-
-void BlockBilinearForm::ConformingAssemble()
-{
-   Finalize(0);
-   MFEM_ASSERT(mat, "the BilinearForm is not assembled");
-
-   if (!P) { BuildProlongation(); }
-
-   SparseMatrix * Pm = P->CreateMonolithic();
-
-   SparseMatrix *Pt = Transpose(*Pm);
-
-   SparseMatrix *PtA = mfem::Mult(*Pt, *mat);
-   delete mat;
-   if (mat_e)
-   {
-      SparseMatrix *PtAe = mfem::Mult(*Pt, *mat_e);
-      delete mat_e;
-      mat_e = PtAe;
-   }
-   delete Pt;
-   mat = mfem::Mult(*PtA, *Pm);
-   delete PtA;
-   if (mat_e)
-   {
-      SparseMatrix *PtAeP = mfem::Mult(*mat_e, *Pm);
-      delete mat_e;
-      mat_e = PtAeP;
-   }
-   delete Pm;
-   height = mat->Height();
-   width = mat->Width();
-}
-
-void BlockBilinearForm::Mult(const Vector &x, Vector &y) const
-{
-   // TODO
-}
-
-
-double& BlockBilinearForm::Elem (int i, int j)
-{
-   return mat -> Elem(i,j);
-}
-
-const double& BlockBilinearForm::Elem (int i, int j) const
-{
-   return mat -> Elem(i,j);
-}
-
-MatrixInverse * BlockBilinearForm::Inverse() const
-{
-   return mat -> Inverse();
-}
-
-void BlockBilinearForm::Finalize(int skip_zeros)
-{
-   mat->Finalize(skip_zeros);
-   if (mat_e) { mat_e->Finalize(skip_zeros); }
-}
-
-/// Adds new Block Domain Integrator. Assumes ownership of @a bfi.
-void BlockBilinearForm::AddDomainIntegrator(BlockBilinearFormIntegrator *bfi)
-{
-   domain_integs.Append(bfi);
-}
-
-/// Assembles the form i.e. sums over all domain integrators.
-void BlockBilinearForm::Assemble(int skip_zeros)
-{
-   ElementTransformation *eltrans;
-   DofTransformation * doftrans_j, *doftrans_k;
-   Mesh *mesh = fespaces[0] -> GetMesh();
-   DenseMatrix elmat, *elmat_p;
-   int nblocks = fespaces.Size();
-   Array<const FiniteElement *> fe(nblocks);
-   Array<int> vdofs_j, vdofs_k;
-   Array<int> offsetvdofs_j;
-   Array<int> elementblockoffsets(nblocks+1);
-   elementblockoffsets[0] = 0;
-   Array<int> blockoffsets(nblocks+1);
-   blockoffsets[0] = 0;
-   for (int i =0; i<nblocks; i++)
-   {
-      blockoffsets[i+1] = fespaces[i]->GetVSize();
-   }
-   blockoffsets.PartialSum();
-   // mfem::out << "blockoffsets = " ; blockoffsets.Print();
-
-   if (mat == NULL)
-   {
-      AllocMat();
-   }
-
-   if (domain_integs.Size())
-   {
-      // loop through elements
-      for (int i = 0; i < mesh -> GetNE(); i++)
-      {
-         if (element_matrices)
-         {
-            elmat_p = &(*element_matrices)(i);
-         }
-         else
-         {
-            elmat.SetSize(0);
-            for (int k = 0; k < domain_integs.Size(); k++)
-            {
-               for (int j = 0; j<nblocks; j++)
-               {
-                  fe[j] = fespaces[j]->GetFE(i);
-                  elementblockoffsets[j+1] = fe[j]->GetDof();
-               }
-               elementblockoffsets.PartialSum();
-               eltrans = mesh->GetElementTransformation(i);
-               domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
-               if (elmat.Size() == 0)
-               {
-                  elmat = elemmat;
-               }
-               else
-               {
-                  elmat += elemmat;
-               }
-            }
-         }
-         if (elmat.Size() == 0)
-         {
-            continue;
-         }
-         else
-         {
-            elmat_p = &elmat;
-         }
-         vdofs.SetSize(0);
-         for (int j = 0; j<nblocks; j++)
-         {
-            doftrans_j = fespaces[j]->GetElementVDofs(i, vdofs_j);
-            int jbeg = elementblockoffsets[j];
-            int jend = elementblockoffsets[j+1]-1;
-            int offset_j = blockoffsets[j];
-            offsetvdofs_j.SetSize(vdofs_j.Size());
-
-            for (int l = 0; l<vdofs_j.Size(); l++)
-            {
-               offsetvdofs_j[l] = vdofs_j[l]<0 ? -offset_j + vdofs_j[l]
-                                  :  offset_j + vdofs_j[l];
-            }
-            vdofs.Append(offsetvdofs_j);
-            for (int k = 0; k<nblocks; k++)
-            {
-               doftrans_k = fespaces[k]->GetElementVDofs(i, vdofs_k);
-               if (doftrans_k || doftrans_j)
-               {
-                  int kbeg = elementblockoffsets[k];
-                  int kend = elementblockoffsets[k+1]-1;
-                  DenseMatrix A;
-                  elmat_p->GetSubMatrix(jbeg,jend,kbeg, kend, A);
-                  TransformDual(doftrans_j, doftrans_k, A);
-                  elmat_p->SetSubMatrix(jbeg,kbeg,A);
-               }
-            }
-         }
-         mat->AddSubMatrix(vdofs,vdofs,*elmat_p, skip_zeros);
-      }
-   }
-
-}
-
-
-
-void BlockBilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                         Vector &x,
-                                         Vector &b, OperatorHandle &A, Vector &X,
-                                         Vector &B, int copy_interior)
-{
-   FormSystemMatrix(ess_tdof_list, A);
-
-   if (!P)
-   {
-      EliminateVDofsInRHS(ess_tdof_list, x, b);
-      X.MakeRef(x, 0, x.Size());
-      B.MakeRef(b, 0, b.Size());
-      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
-   }
-   else // non conforming space
-   {
-      B.SetSize(P->Width());
-      P->MultTranspose(b, B);
-      X.SetSize(R->Height());
-
-      mfem::out << "R height, width  = " << R->Height() <<" x "<< R->Width() <<
-                std::endl;
-
-      R->Mult(x, X);
-      EliminateVDofsInRHS(ess_tdof_list, X, B);
-      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
-   }
-
-}
-
-void BlockBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                         OperatorHandle &A)
-{
-   if (!mat_e)
-   {
-      const SparseMatrix *P_ = fespaces[0]->GetConformingProlongation();
-      if (P_) { ConformingAssemble(); }
-      EliminateVDofs(ess_tdof_list, diag_policy);
-      const int remove_zeros = 0;
-      Finalize(remove_zeros);
-   }
-   A.Reset(mat, false);
-}
-
-void BlockBilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
-                                           Vector &x)
-{
-   if (!P)
-   {
-      x.SyncMemory(X);
-   }
-   else
-   {
-      // Apply conforming prolongation
-      x.SetSize(P->Height());
-      P->Mult(X, x);
-   }
-}
-
-
-
-void BlockBilinearForm::ComputeElementMatrices()
-{
-   MFEM_ABORT("BlockBilinearForm::ComputeElementMatrices:not implemented yet")
-}
-
-void BlockBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (element_matrices)
-   {
-      elmat.SetSize(element_matrices->SizeI(), element_matrices->SizeJ());
-      elmat = element_matrices->GetData(i);
-      return;
-   }
-
-   int nblocks = fespaces.Size();
-   Array<const FiniteElement *> fe(nblocks);
-   ElementTransformation *eltrans;
-
-   elmat.SetSize(0);
-   if (domain_integs.Size())
-   {
-      for (int j = 0; j<nblocks; j++)
-      {
-         fe[j] = fespaces[j]->GetFE(i);
-      }
-      eltrans = fespaces[0]->GetElementTransformation(i);
-      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
-      for (int k = 1; k < domain_integs.Size(); k++)
-      {
-         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      int matsize = 0;
-      for (int j = 0; j<nblocks; j++)
-      {
-         matsize += fespaces[j]->GetFE(i)->GetDof();
-      }
-      elmat.SetSize(matsize);
-      elmat = 0.0;
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                             const Vector &sol, Vector &rhs,
-                                             DiagonalPolicy dpolicy)
-{
-   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
-   // Array<int> ess_dofs, conf_ess_dofs;
-   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
-
-   // if (fes->GetVSize() == height)
-   // {
-   //    EliminateEssentialBCFromDofs(ess_dofs, sol, rhs, dpolicy);
-   // }
-   // else
-   // {
-   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
-   //    EliminateEssentialBCFromDofs(conf_ess_dofs, sol, rhs, dpolicy);
-   // }
-}
-
-void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                             DiagonalPolicy dpolicy)
-{
-   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
-   // Array<int> ess_dofs, conf_ess_dofs;
-   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
-
-   // if (fes->GetVSize() == height)
-   // {
-   //    EliminateEssentialBCFromDofs(ess_dofs, dpolicy);
-   // }
-   // else
-   // {
-   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
-   //    EliminateEssentialBCFromDofs(conf_ess_dofs, dpolicy);
-   // }
-}
-
-void BlockBilinearForm::EliminateEssentialBCDiag (const Array<int>
-                                                  &bdr_attr_is_ess,
-                                                  double value)
-{
-   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBCDiag: not implemented yet");
-   // Array<int> ess_dofs, conf_ess_dofs;
-   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
-
-   // if (fes->GetVSize() == height)
-   // {
-   //    EliminateEssentialBCFromDofsDiag(ess_dofs, value);
-   // }
-   // else
-   // {
-   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
-   //    EliminateEssentialBCFromDofsDiag(conf_ess_dofs, value);
-   // }
-}
-
-void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
-                                       const Vector &sol, Vector &rhs,
-                                       DiagonalPolicy dpolicy)
-{
-   vdofs.HostRead();
-   for (int i = 0; i < vdofs.Size(); i++)
-   {
-      int vdof = vdofs[i];
-      if ( vdof >= 0 )
-      {
-         mat -> EliminateRowCol (vdof, sol(vdof), rhs, dpolicy);
-      }
-      else
-      {
-         mat -> EliminateRowCol (-1-vdof, sol(-1-vdof), rhs, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
-                                       DiagonalPolicy dpolicy)
-{
-   if (mat_e == NULL)
-   {
-      mat_e = new SparseMatrix(height);
-   }
-
-   // mat -> EliminateCols(vdofs, *mat_e,)
-
-   for (int i = 0; i < vdofs.Size(); i++)
-   {
-      int vdof = vdofs[i];
-      if ( vdof >= 0 )
-      {
-         mat -> EliminateRowCol (vdof, *mat_e, dpolicy);
-      }
-      else
-      {
-         mat -> EliminateRowCol (-1-vdof, *mat_e, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBCFromDofs(
-   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
-   DiagonalPolicy dpolicy)
-{
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
-   MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
-   MFEM_ASSERT(rhs.Size() == height, "incorrect rhs Vector size");
-
-   for (int i = 0; i < ess_dofs.Size(); i++)
-   {
-      if (ess_dofs[i] < 0)
-      {
-         mat -> EliminateRowCol (i, sol(i), rhs, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBCFromDofs (const Array<int>
-                                                      &ess_dofs,
-                                                      DiagonalPolicy dpolicy)
-{
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
-
-   for (int i = 0; i < ess_dofs.Size(); i++)
-   {
-      if (ess_dofs[i] < 0)
-      {
-         mat -> EliminateRowCol (i, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBCFromDofsDiag (
-   const Array<int> &ess_dofs,
-   double value)
-{
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
-
-   for (int i = 0; i < ess_dofs.Size(); i++)
-   {
-      if (ess_dofs[i] < 0)
-      {
-         mat -> EliminateRowColDiag (i, value);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateVDofsInRHS(
-   const Array<int> &vdofs, const Vector &x, Vector &b)
-{
-   mat_e->AddMult(x, b, -1.);
-   mat->PartMult(vdofs, x, b);
-}
-
-
-
-BlockBilinearForm::~BlockBilinearForm()
-{
-   delete mat_e;
-   delete mat;
-   delete element_matrices;
-
-   for (int k=0; k < domain_integs.Size(); k++)
-   {
-      delete domain_integs[k];
-   }
-   for (int k=0; k < trace_integs.Size(); k++)
-   {
-      delete trace_integs[k];
-   }
-   delete P;
-   delete R;
-}
-
-
-
-} // namespace mfem
@@ -1,288 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCKBILINEARFORM
-#define MFEM_BLOCKBILINEARFORM
-
-#include "../config/config.hpp"
-#include "../linalg/linalg.hpp"
-
-namespace mfem
-{
-
-/** @brief A "square matrix" operator for the associated FE space and
-    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
-    M.  */
-class BlockBilinearForm : public Matrix
-{
-
-protected:
-
-   int nblocks;
-   Array<int> dof_offsets;
-   Array<int> tdof_offsets;
-
-   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
-   SparseMatrix *mat;
-
-   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
-        from the b.c.  Owned.
-       \f$ M + M_e = M_{original} \f$ */
-   SparseMatrix *mat_e;
-
-   /// FE spaces on which the block form lives. Not owned.
-   Array<FiniteElementSpace * > fespaces;
-
-   /** @brief Indicates the Mesh::sequence corresponding to the current state of
-      the BilinearForm. */
-   long sequence;
-
-   /** @brief Indicates the BlockBilinearFormIntegrator%s stored in #domain_integs,
-     are owned by another BlockBilinearForm. */
-   int extern_bfs;
-
-   /// Set of Domain Integrators to be applied.
-   Array<BlockBilinearFormIntegrator * > domain_integs;
-
-   /// Trace integrators.
-   Array<BlockBilinearFormIntegrator * > trace_integs;
-
-   DenseMatrix elemmat;
-   Array<int>  vdofs;
-
-   DenseTensor *element_matrices; ///< Owned.
-
-   BlockMatrix * P = nullptr; // Block Prolongation
-   BlockMatrix * R = nullptr; // Block Restriction
-
-   /** This data member allows one to specify what should be done to the
-    diagonal matrix entries and corresponding RHS values upon elimination of
-    the constrained DoFs. */
-   DiagonalPolicy diag_policy;
-
-   // Allocate appropriate SparseMatrix and assign it to mat
-   void AllocMat();
-
-   void ConformingAssemble();
-
-   void BuildProlongation();
-
-
-private:
-
-public:
-
-   /// Creates bilinear form associated with FE spaces @a *fespaces.
-   BlockBilinearForm(Array<FiniteElementSpace * > & fespaces_);
-
-   /// Get the size of the BilinearForm as a square matrix.
-   int Size() const { return height; }
-
-
-   /// Pre-allocate the internal SparseMatrix before assembly.
-   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
-
-   /// Returns a reference to: \f$ M_{ij} \f$
-   const double &operator()(int i, int j) { return (*mat)(i,j); }
-
-
-   /// Matrix vector multiplication:  \f$ y = M x \f$
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   /** @brief Matrix vector multiplication with the original uneliminated
-       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
-       \f$ y = M x + M_e x \f$ */
-   void FullMult(const Vector &x, Vector &y) const
-   { mat->Mult(x, y); mat_e->AddMult(x, y); }
-
-   virtual double &Elem(int i, int j);
-   virtual const double &Elem(int i, int j) const;
-   virtual MatrixInverse *Inverse() const;
-
-   /// Finalizes the matrix initialization.
-   virtual void Finalize(int skip_zeros = 1);
-
-   /// Returns a const reference to the sparse matrix.
-   const SparseMatrix &SpMat() const
-   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
-   }
-
-   /// Returns a reference to the sparse matrix:  \f$ M \f$
-   SparseMatrix &SpMat()
-   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
-   }
-
-   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
-   const SparseMatrix &SpMatElim() const
-   {
-      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
-      return *mat_e;
-   }
-
-   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
-   SparseMatrix &SpMatElim()
-   {
-      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
-      return *mat_e;
-   }
-
-   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
-   void AddDomainIntegrator(BlockBilinearFormIntegrator *bfi);
-
-   /// Adds new Trace Integrator. Assumes ownership of @a bfi.
-   void AddTraceIntegrator(BlockBilinearFormIntegrator *bfi);
-
-   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
-   void operator=(const double a)
-   {
-      if (mat != NULL) { *mat = a; }
-      if (mat_e != NULL) { *mat_e = a; }
-   }
-
-   /// Assembles the form i.e. sums over all domain integrators.
-   void Assemble(int skip_zeros = 1);
-
-
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
-                                 Vector &b, OperatorHandle &A, Vector &X,
-                                 Vector &B, int copy_interior = 0);
-
-   /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.). */
-   /** Version of the method FormLinearSystem() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
-   template <typename OpType>
-   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OpType &A, Vector &X, Vector &B,
-                         int copy_interior = 0)
-   {
-      OperatorHandle Ah;
-      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A);
-
-   /// Form the linear system matrix A, see FormLinearSystem() for details.
-   /** Version of the method FormSystemMatrix() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
-   template <typename OpType>
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
-   {
-      OperatorHandle Ah;
-      FormSystemMatrix(ess_tdof_list, Ah);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
-
-
-   void ComputeElementMatrices();
-
-   /// Free the memory used by the element matrices.
-   void FreeElementMatrices()
-   { delete element_matrices; element_matrices = NULL; }
-
-   /// Compute the element matrix of the given element
-   /** The element matrix is computed by calling the domain integrators
-       or the one stored internally by a prior call of ComputeElementMatrices()
-       is returned when available.
-   */
-   void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Eliminate essential boundary DOFs from the system.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. By default, the diagonal at the
-       essential DOFs is set to 1.0. This behavior is controlled by the argument
-       @a dpolicy. */
-   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                             const Vector &sol, Vector &rhs,
-                             DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /// Eliminate essential boundary DOFs from the system matrix.
-   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                             DiagonalPolicy dpolicy = DIAG_ONE);
-   /// Perform elimination and set the diagonal entry to the given value
-   void EliminateEssentialBCDiag(const Array<int> &bdr_attr_is_ess,
-                                 double value);
-
-   /// Eliminate the given @a vdofs.
-   /** NOTE: here, @a vdofs is a list of DOFs from all the fespaces
-       In this case the eliminations are applied to the internal \f$ M \f$
-       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
-   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
-                       DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /// Eliminate the given @a vdofs (all the fespaces), storing the eliminated part internally in \f$ M_e \f$.
-   /** This method works in conjunction with EliminateVDofsInRHS() and allows
-       elimination of boundary conditions in multiple right-hand sides. In this
-       method, @a vdofs is a list of DOFs. */
-   void EliminateVDofs(const Array<int> &vdofs,
-                       DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /** @brief Similar to
-       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
-       but here @a ess_dofs is a marker (boolean) array on all vector-dofs
-       (@a ess_dofs[i] < 0 is true). */
-   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
-                                     Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
-       here @a ess_dofs is a marker (boolean) array on all vector-dofs
-       (@a ess_dofs[i] < 0 is true). */
-   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs,
-                                     DiagonalPolicy dpolicy = DIAG_ONE);
-   /// Perform elimination and set the diagonal entry to the given value
-   void EliminateEssentialBCFromDofsDiag(const Array<int> &ess_dofs,
-                                         double value);
-
-   /** @brief Use the stored eliminated part of the matrix (see
-       EliminateVDofs(const Array<int> &, DiagonalPolicy)) to modify the r.h.s.
-       @a b; @a vdofs is a list of DOFs (non-directional, i.e. >= 0). */
-   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
-                            Vector &b);
-
-
-   /// Sets diagonal policy used upon construction of the linear system.
-   /** Policies include:
-
-       - DIAG_ZERO (Set the diagonal values to zero)
-       - DIAG_ONE  (Set the diagonal values to one)
-       - DIAG_KEEP (Keep the diagonal values)
-   */
-   void SetDiagonalPolicy(DiagonalPolicy policy)
-   {
-      diag_policy = policy;
-   }
-
-   /// Destroys bilinear form.
-   virtual ~BlockBilinearForm();
-
-};
-
-} // namespace mfem
-
-#endif
@@ -1,136 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-void BlockBilinearFormIntegrator::AssembleElementMatrix(
-   const Array<const FiniteElement *> &el,
-   ElementTransformation &Trans,
-   DenseMatrix &elmat)
-{
-   mfem_error ("BlockBilinearFormIntegrator::AssembleElementMatrix\n"
-               "   is not implemented for this class.");
-}
-
-void BlockLinearFormIntegrator::AssembleRHSElementVect(
-   const Array<const FiniteElement *> &el,
-   ElementTransformation &Trans,
-   Vector &elvect)
-{
-   mfem_error ("BlockLinearFormIntegrator::AssembleElementVector\n"
-               "   is not implemented for this class.");
-}
-
-/** Given a particular Finite Element computes the element vector */
-void TestBlockBilinearFormIntegrator::AssembleElementMatrix
-(const Array<const FiniteElement *> &el,
- ElementTransformation &Trans,
- DenseMatrix &elmat)
-{
-   int nd = 0;
-   int nblocks = el.Size();
-   Array<int> offsets(nblocks+1);
-   offsets[0] = 0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      nd += el[i]->GetDof();
-      offsets[i+1] = el[i]->GetDof();
-   }
-   offsets.PartialSum();
-   elmat.SetSize(nd);
-   elmat = 0.0;
-   DenseMatrix dmat;
-
-   if (blfis.NumRows())
-   {
-      // Get the matrices directly from the existing BilinearFormIntegrators
-      for (int i = 0; i<nblocks; i++)
-      {
-         // mfem::out << "i = " << i << std::endl;
-         int offset_i = offsets[i];
-         const FiniteElement * fe_i = el[i];
-         for (int j = 0; j<nblocks; j++)
-         {
-            // mfem::out << "j = " << j << std::endl;
-            BilinearFormIntegrator * blfi = blfis(i,j);
-            if (!blfi) { continue; }
-            if (j == i)
-            {
-               blfi->AssembleElementMatrix(*fe_i,Trans,dmat);
-               // mfem::out << "j 1 = " << j << std::endl;
-               elmat.SetSubMatrix(offset_i,dmat);
-            }
-            else
-            {
-               const FiniteElement * fe_j = el[j];
-               blfi->AssembleElementMatrix2(*fe_j,*fe_i,Trans,dmat);
-               // mfem::out << "j 2 = " << j << std::endl;
-               int offset_j = offsets[j];
-               elmat.SetSubMatrix(offset_i,offset_j,dmat);
-            }
-         }
-      }
-      return;
-   }
-
-   // else compute the matrices
-   elmat = 25.0;
-   // TODO
-
-}
-
-/** Given a particular Finite Element computes the element vector */
-void TestBlockLinearFormIntegrator::AssembleRHSElementVect
-(const Array<const FiniteElement *> &el,
- ElementTransformation &Trans,
- Vector &elvector)
-{
-   int nd = 0;
-   int nblocks = el.Size();
-   Array<int> offsets(nblocks+1);
-   offsets[0] = 0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      nd += el[i]->GetDof();
-      offsets[i+1] = el[i]->GetDof();
-   }
-   offsets.PartialSum();
-   elvector.SetSize(nd);
-   elvector = 0.0;
-   Vector subvector;
-
-   if (lfis.Size())
-   {
-      // Get the matrices directly from the existing BilinearFormIntegrators
-      for (int i = 0; i<nblocks; i++)
-      {
-         int offset = offsets[i];
-         const FiniteElement * fe_i = el[i];
-         LinearFormIntegrator * lfi = lfis[i];
-         if (!lfi)
-         {
-            continue;
-         }
-         lfi->AssembleRHSElementVect(*fe_i,Trans,subvector);
-         elvector.SetVector(subvector,offset);
-      }
-      return;
-   }
-
-   // else, compute the block linear form integrator
-   // elvector = 1.0;
-   // TODO
-}
-
-} // namespace mfem
@@ -1,147 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCKINTEG
-#define MFEM_BLOCKINTEG
-
-#include "../config/config.hpp"
-#include "fe.hpp"
-#include "coefficient.hpp"
-#include "fespace.hpp"
-
-namespace mfem
-{
-/** The abstract base class BlockBilinearFormIntegrator is
- a generalization of the BilinearFormIntegrator class suitable
- for block formulations. */
-class BlockBilinearFormIntegrator
-{
-protected:
-   const IntegrationRule *IntRule;
-   BlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : IntRule(ir) { }
-public:
-
-
-   /// Given a particular Finite Element computes the element matrix elmat.
-   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
-                                      ElementTransformation &Trans,
-                                      DenseMatrix &elmat);
-
-   virtual ~BlockBilinearFormIntegrator() { }
-};
-
-/** The abstract base class BlockBilinearFormIntegrator is
- a generalization of the BilinearFormIntegrator class suitable
- for block formulations. */
-class BlockLinearFormIntegrator
-{
-protected:
-   const IntegrationRule *IntRule;
-   BlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : IntRule(ir) { }
-
-public:
-   /// Given a particular Finite Element computes the element matrix elmat.
-   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
-                                       ElementTransformation &Trans,
-                                       Vector &elvect);
-
-   virtual ~BlockLinearFormIntegrator() { }
-};
-
-
-
-
-class TestBlockBilinearFormIntegrator: public BlockBilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-   Array<const FiniteElementSpace * > fespaces;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
-
-   Array2D<BilinearFormIntegrator *> blfis;
-
-
-public:
-
-   TestBlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : BlockBilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
-
-   /// Construct a mass integrator with coefficient q
-   TestBlockBilinearFormIntegrator(Coefficient &q,
-                                   const IntegrationRule *ir = NULL)
-      : BlockBilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL)   { }
-
-
-   TestBlockBilinearFormIntegrator(Array2D<BilinearFormIntegrator *> blfis_)
-      : BlockBilinearFormIntegrator(NULL), blfis(blfis_) { }
-
-   void SetIntegrators(Array2D<BilinearFormIntegrator *> blfis_)
-   {
-      blfis = blfis_;
-   }
-
-
-   /** Given a particular Finite Element computes the element matrix
-       elmat. */
-   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
-                                      ElementTransformation &Trans,
-                                      DenseMatrix &elmat);
-
-   virtual ~TestBlockBilinearFormIntegrator() { }
-
-};
-
-/** Class for local vector assembly */
-class TestBlockLinearFormIntegrator: public BlockLinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-   Array<const FiniteElementSpace * > fespaces;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
-   Array<LinearFormIntegrator *> lfis;
-
-public:
-
-   TestBlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : BlockLinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
-
-   /// Construct a test linear integrator with coefficient q
-   TestBlockLinearFormIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BlockLinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL) { }
-
-
-   TestBlockLinearFormIntegrator(Array<LinearFormIntegrator *> lfis_)
-      : BlockLinearFormIntegrator(NULL), lfis(lfis_) { }
-
-   void SetIntegrators(Array<LinearFormIntegrator *> lfis_)
-   {
-      lfis = lfis_;
-   }
-
-   /** Given a particular Finite Element computes the element vector */
-   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
-                                       ElementTransformation &Trans,
-                                       Vector &elvector);
-
-
-};
-
-
-} // namespace mfem
-
-
-#endif
@@ -1,123 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-BlockLinearForm::BlockLinearForm(Array<FiniteElementSpace * > & fespaces_) :
-   Vector(0), fespaces(fespaces_)
-{
-   int s = 0;
-   int nblocks = fespaces.Size();
-   for (int i =0; i<nblocks; i++)
-   {
-      s += fespaces[i]->GetVSize();
-   }
-   // mfem::out << "size = " << size << std::endl;
-
-   SetSize(s);
-
-}
-
-
-void BlockLinearForm::AddDomainIntegrator(BlockLinearFormIntegrator *lfi)
-{
-   domain_integs.Append(lfi);
-}
-
-void BlockLinearForm::Assemble()
-{
-   ElementTransformation *eltrans;
-   DofTransformation *doftrans;
-   Mesh *mesh = fespaces[0] -> GetMesh();
-   Vector subvect,elvect, *elvect_p;
-
-   int nblocks = fespaces.Size();
-   Array<const FiniteElement *> fe(nblocks);
-   Array<int> offsetvdofs;
-   Array<int> elementblockoffsets(nblocks+1);
-   elementblockoffsets[0] = 0;
-   Array<int> blockoffsets(nblocks+1);
-   blockoffsets[0] = 0;
-   for (int i =0; i<nblocks; i++)
-   {
-      blockoffsets[i+1] = fespaces[i]->GetVSize();
-   }
-   blockoffsets.PartialSum();
-
-   Vector::operator=(0.0);
-
-   if (domain_integs.Size())
-   {
-      // loop through elements
-      for (int i = 0; i < mesh -> GetNE(); i++)
-      {
-         elvect.SetSize(0);
-         for (int k = 0; k < domain_integs.Size(); k++)
-         {
-            for (int j = 0; j<nblocks; j++)
-            {
-               fe[j] = fespaces[j]->GetFE(i);
-               elementblockoffsets[j+1] = fe[j]->GetDof();
-            }
-            elementblockoffsets.PartialSum();
-            eltrans = mesh->GetElementTransformation(i);
-
-            domain_integs[k]->AssembleRHSElementVect(fe, *eltrans, elemvect);
-            if (elvect.Size() == 0)
-            {
-               elvect = elemvect;
-            }
-            else
-            {
-               elvect += elemvect;
-            }
-         }
-         if (elvect.Size() == 0)
-         {
-            continue;
-         }
-         else
-         {
-            elvect_p = &elvect;
-         }
-
-         double *data = elvect_p->GetData();
-
-         for (int j = 0; j<nblocks; j++)
-         {
-            doftrans = fespaces[j]->GetElementVDofs(i, vdofs);
-            int offset = blockoffsets[j];
-            offsetvdofs.SetSize(vdofs.Size());
-            for (int l = 0; l<vdofs.Size(); l++)
-            {
-               offsetvdofs[l] = vdofs[l]<0 ? -offset + vdofs[l]
-                                :  offset + vdofs[l];
-            }
-            int jbeg = elementblockoffsets[j];
-            int jend = elementblockoffsets[j+1]-1;
-            subvect.SetSize(jend-jbeg+1);
-            subvect.SetData(&data[jbeg]);
-
-            if (doftrans)
-            {
-               doftrans->TransformDual(subvect);
-            }
-            AddElementVector(offsetvdofs,subvect);
-         }
-      }
-   }
-}
-
-
-} // name space mfem
@@ -1,49 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCKLINEARFORM
-#define MFEM_BLOCKLINEARFORM
-
-#include "../config/config.hpp"
-#include "../linalg/linalg.hpp"
-
-namespace mfem
-{
-
-
-class BlockLinearForm : public Vector
-{
-protected:
-   /// FE spaces on which the LinearForm lives. Not owned.
-   Array<FiniteElementSpace * > fespaces;
-
-   /// Set of Domain Integrators to be applied.
-   Array<BlockLinearFormIntegrator*> domain_integs;
-
-   Vector elemvect;
-   Array<int> vdofs;
-
-public:
-   BlockLinearForm(Array<FiniteElementSpace * > & fespaces_);
-
-   /// Adds new Domain Integrator. Assumes ownership of @a lfi.
-   void AddDomainIntegrator(BlockLinearFormIntegrator *lfi);
-
-   /// Assembles the block linear form i.e. sums over all domain integrators.
-   void Assemble();
-
-
-};
-
-} // namespace mfem
-
-
-#endif
@@ -1,969 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "blockstaticcond.hpp"
-
-namespace mfem
-{
-
-
-BlockStaticCondensation::BlockStaticCondensation(Array<FiniteElementSpace *> &
-                                                 fes_)
-{
-   SetSpaces(fes_);
-
-   Array<int> rvdofs;
-   Array<int> vdofs;
-   Array<int> rdof_edof0;
-   for (int k = 0; k<nblocks; k++)
-   {
-      if (!tr_fes[k]) { continue; }
-      rdof_edof0.SetSize(tr_fes[k]->GetVSize());
-      for (int i = 0; i < mesh->GetNE(); i++)
-      {
-         fes[k]->GetElementVDofs(i, vdofs);
-         tr_fes[k]->GetElementVDofs(i, rvdofs);
-         const int vdim = fes[k]->GetVDim();
-         const int nsd = vdofs.Size()/vdim;
-         const int nsrd = rvdofs.Size()/vdim;
-         for (int vd = 0; vd < vdim; vd++)
-         {
-            for (int j = 0; j < nsrd; j++)
-            {
-               int rvdof = rvdofs[j+nsrd*vd];
-               int vdof = vdofs[j+nsd*vd];
-               if (rvdof < 0)
-               {
-                  rvdof = -1-rvdof;
-                  vdof = -1-vdof;
-               }
-               MFEM_ASSERT(vdof >= 0, "incompatible volume and trace FE spaces");
-               rdof_edof0[rvdof] = vdof + dof_offsets[k];
-            }
-         }
-      }
-      rdof_edof.Append(rdof_edof0);
-   }
-}
-
-void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
-{
-#ifdef MFEM_USE_MPI
-   ParMesh *pmesh = nullptr;
-   parallel = false;
-   if (dynamic_cast<ParFiniteElementSpace *>(fes_[0]))
-   {
-      parallel = true;
-   }
-#else
-   parallel = false;
-#endif
-   fes=fes_;
-   nblocks = fes.Size();
-   rblocks = 0;
-   tr_fes.SetSize(nblocks);
-   mesh = fes[0]->GetMesh();
-
-   IsTraceSpace.SetSize(nblocks);
-   const FiniteElementCollection * fec;
-   for (int i = 0; i < nblocks; i++)
-   {
-      fec = fes[i]->FEColl();
-      IsTraceSpace[i] =
-         (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
-          dynamic_cast<const ND_Trace_FECollection*>(fec) ||
-          dynamic_cast<const RT_Trace_FECollection*>(fec));
-#ifdef MFEM_USE_MPI
-      if (parallel)
-      {
-         pmesh = dynamic_cast<ParMesh *>(mesh);
-         tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
-                     nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
-                                               fes[i]->GetOrdering());
-      }
-      else
-      {
-         tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
-                     nullptr : (IsTraceSpace[i]) ? fes[i] :
-                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
-                                            fes[i]->GetOrdering());
-      }
-#else
-      // skip if it's an L2 space (no trace space to construct)
-      tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
-                  nullptr : (IsTraceSpace[i]) ? fes[i] :
-                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
-                                         fes[i]->GetOrdering());
-#endif
-      if (tr_fes[i]) { rblocks++; }
-   }
-   if (parallel)
-   {
-      ess_tdofs.SetSize(rblocks);
-      for (int i = 0; i<rblocks; i++)
-      {
-         ess_tdofs[i] = new Array<int>();
-      }
-   }
-   Init();
-}
-
-void BlockStaticCondensation::ComputeOffsets()
-{
-   dof_offsets.SetSize(nblocks+1);
-   tdof_offsets.SetSize(nblocks+1);
-   dof_offsets[0] = 0;
-   tdof_offsets[0] = 0;
-
-   rdof_offsets.SetSize(rblocks+1);
-   rtdof_offsets.SetSize(rblocks+1);
-   rdof_offsets[0] = 0;
-   rtdof_offsets[0] = 0;
-
-   int j=0;
-   for (int i =0; i<nblocks; i++)
-   {
-      dof_offsets[i+1] = fes[i]->GetVSize();
-      tdof_offsets[i+1] = fes[i]->GetTrueVSize();
-      if (tr_fes[i])
-      {
-         rdof_offsets[j+1] = tr_fes[i]->GetVSize();
-         rtdof_offsets[j+1] = tr_fes[i]->GetTrueVSize();
-         j++;
-      }
-   }
-   rdof_offsets.PartialSum();
-   rtdof_offsets.PartialSum();
-   dof_offsets.PartialSum();
-   tdof_offsets.PartialSum();
-}
-
-
-void BlockStaticCondensation::Init()
-{
-   lmat.SetSize(mesh->GetNE());
-   lvec.SetSize(mesh->GetNE());
-   for (int i = 0; i < mesh->GetNE(); i++)
-   {
-      lmat[i] = nullptr;
-      lvec[i] = nullptr;
-   }
-
-   ComputeOffsets();
-
-   S = new BlockMatrix(rdof_offsets);
-   S->owns_blocks = 1;
-
-   for (int i = 0; i<S->NumRowBlocks(); i++)
-   {
-      int h = rdof_offsets[i+1] - rdof_offsets[i];
-      for (int j = 0; j<S->NumColBlocks(); j++)
-      {
-         int w = rdof_offsets[j+1] - rdof_offsets[j];
-         S->SetBlock(i,j,new SparseMatrix(h, w));
-      }
-   }
-   y = new BlockVector(rdof_offsets);
-   *y = 0.;
-}
-
-void BlockStaticCondensation::GetReduceElementIndicesAndOffsets(int el,
-                                                                Array<int> & trace_ldofs,
-                                                                Array<int> & interior_ldofs,
-                                                                Array<int> & offsets) const
-{
-   int dim = mesh->Dimension();
-   offsets.SetSize(tr_fes.Size()+1); offsets = 0;
-   Array<int> dofs;
-   Array<int> faces, ori;
-   if (dim == 1)
-   {
-      mesh->GetElementVertices(el, faces);
-   }
-   if (dim == 2)
-   {
-      mesh->GetElementEdges(el, faces, ori);
-   }
-   else //dim = 3
-   {
-      mesh->GetElementFaces(el,faces,ori);
-   }
-   int numfaces = faces.Size();
-
-   trace_ldofs.SetSize(0);
-   interior_ldofs.SetSize(0);
-   // construct Array of bubble dofs to be extracted
-   int skip=0;
-   Array<int> tr_dofs;
-   Array<int> int_dofs;
-   for (int i = 0; i<tr_fes.Size(); i++)
-   {
-      int td = 0;
-      int ndof;
-      // if it's an L2 space (bubbles)
-      if (!tr_fes[i])
-      {
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
-         td = 0;
-      }
-      else if (IsTraceSpace[i])
-      {
-         for (int iface = 0; iface < numfaces; iface++)
-         {
-            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
-         }
-         ndof = td;
-      }
-      else
-      {
-         Array<int> trace_dofs;
-         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
-         tr_fes[i]->GetElementVDofs(el, trace_dofs);
-         td = trace_dofs.Size(); // number of trace dofs
-      }
-      offsets[i+1] = td;
-      tr_dofs.SetSize(td);
-      int_dofs.SetSize(ndof - td);
-      for (int j = 0; j<td; j++)
-      {
-         tr_dofs[j] = skip + j;
-      }
-      for (int j = 0; j<ndof-td; j++)
-      {
-         int_dofs[j] = skip + td + j;
-      }
-      skip+=ndof;
-
-      trace_ldofs.Append(tr_dofs);
-      interior_ldofs.Append(int_dofs);
-   }
-   offsets.PartialSum();
-}
-
-
-void BlockStaticCondensation::GetReduceElementVDofs(int el,
-                                                    Array<int> & rdofs) const
-{
-   Array<int> faces, ori;
-   int dim = mesh->Dimension();
-   if (dim == 1)
-   {
-      mesh->GetElementVertices(el, faces);
-   }
-   if (dim == 2)
-   {
-      mesh->GetElementEdges(el, faces, ori);
-   }
-   else //dim = 3
-   {
-      mesh->GetElementFaces(el,faces,ori);
-   }
-   int numfaces = faces.Size();
-   rdofs.SetSize(0);
-   int skip = 0;
-   for (int i = 0; i<tr_fes.Size(); i++)
-   {
-      if (!tr_fes[i]) { continue; }
-      Array<int> vdofs;
-      if (IsTraceSpace[i])
-      {
-         Array<int> face_vdofs;
-         for (int k = 0; k < numfaces; k++)
-         {
-            int iface = faces[k];
-            tr_fes[i]->GetFaceVDofs(iface, face_vdofs);
-            vdofs.Append(face_vdofs);
-         }
-      }
-      else
-      {
-         tr_fes[i]->GetElementVDofs(el, vdofs);
-      }
-      for (int j=0; j<vdofs.Size(); j++)
-      {
-         vdofs[j] = (vdofs[j]>=0) ? vdofs[j]+rdof_offsets[skip] :
-                    vdofs[j]-rdof_offsets[skip];
-      }
-      skip++;
-      rdofs.Append(vdofs);
-   }
-}
-void BlockStaticCondensation::GetElementVDofs(int el, Array<int> & vdofs) const
-{
-   Array<int> faces, ori;
-   int dim = mesh->Dimension();
-   if (dim == 1)
-   {
-      mesh->GetElementVertices(el, faces);
-   }
-   if (dim == 2)
-   {
-      mesh->GetElementEdges(el, faces, ori);
-   }
-   else //dim = 3
-   {
-      mesh->GetElementFaces(el,faces,ori);
-   }
-   int numfaces = faces.Size();
-   vdofs.SetSize(0);
-   for (int i = 0; i<tr_fes.Size(); i++)
-   {
-      Array<int> dofs;
-      if (IsTraceSpace[i])
-      {
-         Array<int> face_vdofs;
-         for (int k = 0; k < numfaces; k++)
-         {
-            int iface = faces[k];
-            fes[i]->GetFaceVDofs(iface, face_vdofs);
-            dofs.Append(face_vdofs);
-         }
-      }
-      else
-      {
-         fes[i]->GetElementVDofs(el, dofs);
-      }
-      for (int j=0; j<dofs.Size(); j++)
-      {
-         dofs[j] = (dofs[j]>=0) ? dofs[j]+dof_offsets[i] :
-                   dofs[j]-dof_offsets[i];
-      }
-      vdofs.Append(dofs);
-   }
-}
-
-
-void BlockStaticCondensation::GetLocalShurComplement(int el,
-                                                     const Array<int> & tr_idx, const Array<int> & int_idx,
-                                                     const DenseMatrix & elmat, const Vector & elvect,
-                                                     DenseMatrix & rmat, Vector & rvect)
-{
-   int rdofs = tr_idx.Size();
-   int idofs = int_idx.Size();
-   MFEM_VERIFY(idofs != 0, "Number of interior dofs is zero");
-   MFEM_VERIFY(rdofs != 0, "Number of interface dofs is zero");
-
-   rmat.SetSize(rdofs);
-   rvect.SetSize(rdofs);
-
-   DenseMatrix A_tt, A_ti, A_it, A_ii;
-   Vector y_t, y_i;
-
-   elmat.GetSubMatrix(tr_idx,A_tt);
-   elmat.GetSubMatrix(tr_idx,int_idx, A_ti);
-   elmat.GetSubMatrix(int_idx, tr_idx, A_it);
-   elmat.GetSubMatrix(int_idx, A_ii);
-
-   elvect.GetSubVector(tr_idx, y_t);
-   elvect.GetSubVector(int_idx, y_i);
-
-   DenseMatrixInverse lu(A_ii);
-   lu.Factor();
-   lmat[el] = new DenseMatrix(idofs,rdofs);
-   lvec[el] = new Vector(idofs);
-
-   lu.Mult(A_it,*lmat[el]);
-   lu.Mult(y_i,*lvec[el]);
-
-   // LHS
-   mfem::Mult(A_ti,*lmat[el],rmat);
-
-   rmat.Neg();
-   rmat.Add(1., A_tt);
-
-   // RHS
-   A_ti.Mult(*lvec[el], rvect);
-   rvect.Neg();
-   rvect.Add(1., y_t);
-}
-
-
-void BlockStaticCondensation::AssembleReducedSystem(int el,
-                                                    DenseMatrix &elmat,
-                                                    Vector & elvect)
-{
-   // Get Shur Complement
-   Array<int> tr_idx, int_idx;
-   Array<int> offsets;
-   // Get local element idx and offsets for global assembly
-   GetReduceElementIndicesAndOffsets(el, tr_idx,int_idx, offsets);
-
-   DenseMatrix rmat, *rmatptr;
-   Vector rvec, *rvecptr;
-   // Extract the reduced matrices based on tr_idx and int_idx
-   if (int_idx.Size()!=0)
-   {
-      GetLocalShurComplement(el,tr_idx,int_idx, elmat, elvect, rmat, rvec);
-      rmatptr = &rmat;
-      rvecptr = &rvec;
-   }
-   else
-   {
-      rmatptr = &elmat;
-      rvecptr = &elvect;
-   }
-
-   // Assemble global mat and rhs
-   DofTransformation * doftrans_i, *doftrans_j;
-
-
-   Array<int> faces, ori;
-   int dim = mesh->Dimension();
-   if (dim == 1)
-   {
-      mesh->GetElementVertices(el, faces);
-   }
-   if (dim == 2)
-   {
-      mesh->GetElementEdges(el, faces, ori);
-   }
-   else //dim = 3
-   {
-      mesh->GetElementFaces(el,faces,ori);
-   }
-   int numfaces = faces.Size();
-
-   int skip_i=0;
-   for (int i = 0; i<tr_fes.Size(); i++)
-   {
-      if (!tr_fes[i]) { continue; }
-      Array<int> vdofs_i;
-      doftrans_i = nullptr;
-      if (IsTraceSpace[i])
-      {
-         Array<int> face_vdofs;
-         for (int k = 0; k < numfaces; k++)
-         {
-            int iface = faces[k];
-            tr_fes[i]->GetFaceVDofs(iface, face_vdofs);
-            vdofs_i.Append(face_vdofs);
-         }
-      }
-      else
-      {
-         doftrans_i = tr_fes[i]->GetElementVDofs(el, vdofs_i);
-      }
-      int skip_j=0;
-      for (int j = 0; j<tr_fes.Size(); j++)
-      {
-         if (!tr_fes[j]) { continue; }
-         Array<int> vdofs_j;
-         doftrans_j = nullptr;
-
-         if (IsTraceSpace[j])
-         {
-            Array<int> face_vdofs;
-            for (int k = 0; k < numfaces; k++)
-            {
-               int iface = faces[k];
-               tr_fes[j]->GetFaceVDofs(iface, face_vdofs);
-               vdofs_j.Append(face_vdofs);
-            }
-         }
-         else
-         {
-            doftrans_j = tr_fes[j]->GetElementVDofs(el, vdofs_j);
-         }
-
-         DenseMatrix Ae;
-         rmatptr->GetSubMatrix(offsets[i],offsets[i+1],
-                               offsets[j],offsets[j+1], Ae);
-         if (doftrans_i || doftrans_j)
-         {
-            TransformDual(doftrans_i, doftrans_j, Ae);
-         }
-         S->GetBlock(skip_i,skip_j).AddSubMatrix(vdofs_i,vdofs_j, Ae);
-         skip_j++;
-      }
-
-      // assemble rhs
-      double * data = rvecptr->GetData();
-      Vector vec1;
-      // ref subvector
-      vec1.SetDataAndSize(&data[offsets[i]],
-                          offsets[i+1]-offsets[i]);
-      if (doftrans_i)
-      {
-         doftrans_i->TransformDual(vec1);
-      }
-      y->GetBlock(skip_i).AddElementVector(vdofs_i,vec1);
-      skip_i++;
-   }
-}
-
-void BlockStaticCondensation::BuildProlongation()
-{
-   P = new BlockMatrix(rdof_offsets, rtdof_offsets);
-   R = new BlockMatrix(rtdof_offsets, rdof_offsets);
-   P->owns_blocks = 0;
-   R->owns_blocks = 0;
-   int skip = 0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      if (!tr_fes[i]) { continue; }
-      const SparseMatrix *P_ = tr_fes[i]->GetConformingProlongation();
-      if (P_)
-      {
-         const SparseMatrix *R_ = tr_fes[i]->GetRestrictionMatrix();
-         P->SetBlock(skip,skip,const_cast<SparseMatrix*>(P_));
-         R->SetBlock(skip,skip,const_cast<SparseMatrix*>(R_));
-      }
-      skip++;
-   }
-}
-
-#ifdef MFEM_USE_MPI
-void BlockStaticCondensation::BuildParallelProlongation()
-{
-   MFEM_VERIFY(parallel, "BuildParallelProlongation: wrong code path");
-   pP = new BlockOperator(rdof_offsets, rtdof_offsets);
-   R = new BlockMatrix(rtdof_offsets, rdof_offsets);
-   pP->owns_blocks = 0;
-   R->owns_blocks = 0;
-   int skip = 0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      if (!tr_fes[i]) { continue; }
-      const HypreParMatrix *P_ =
-         dynamic_cast<ParFiniteElementSpace *>(tr_fes[i])->Dof_TrueDof_Matrix();
-      if (P_)
-      {
-         const SparseMatrix *R_ = tr_fes[i]->GetRestrictionMatrix();
-         pP->SetBlock(skip,skip,const_cast<HypreParMatrix*>(P_));
-         R->SetBlock(skip,skip,const_cast<SparseMatrix*>(R_));
-      }
-      skip++;
-   }
-}
-
-void BlockStaticCondensation::ParallelAssemble(BlockMatrix *m)
-{
-   if (!pP) { BuildParallelProlongation(); }
-
-   pS = new BlockOperator(rtdof_offsets);
-   pS_e = new BlockOperator(rtdof_offsets);
-   pS->owns_blocks = 1;
-   pS_e->owns_blocks = 1;
-   HypreParMatrix * A = nullptr;
-   HypreParMatrix * PtAP = nullptr;
-   int skip_i=0;
-   ParFiniteElementSpace * pfes_i = nullptr;
-   ParFiniteElementSpace * pfes_j = nullptr;
-   for (int i = 0; i<nblocks; i++)
-   {
-      if (!tr_fes[i]) { continue; }
-      pfes_i = dynamic_cast<ParFiniteElementSpace*>(fes[i]);
-      HypreParMatrix * Pi = (HypreParMatrix*)(&pP->GetBlock(skip_i,skip_i));
-      int skip_j=0;
-      for (int j = 0; j<nblocks; j++)
-      {
-         if (!tr_fes[j]) { continue; }
-         if (m->IsZeroBlock(skip_i,skip_j)) { continue; }
-         if (skip_i == skip_j)
-         {
-            // Make block diagonal square hypre matrix
-            A = new HypreParMatrix(pfes_i->GetComm(), pfes_i->GlobalVSize(),
-                                   pfes_i->GetDofOffsets(),&m->GetBlock(skip_i,skip_i));
-            PtAP = RAP(A,Pi);
-            delete A;
-            pS_e->SetBlock(skip_i,skip_i,PtAP->EliminateRowsCols(*ess_tdofs[skip_i]));
-         }
-         else
-         {
-            pfes_j = dynamic_cast<ParFiniteElementSpace*>(fes[j]);
-            HypreParMatrix * Pj = (HypreParMatrix*)(&pP->GetBlock(skip_j,skip_j));
-            A = new HypreParMatrix(pfes_i->GetComm(), pfes_i->GlobalVSize(),
-                                   pfes_j->GlobalVSize(), pfes_i->GetDofOffsets(),
-                                   pfes_j->GetDofOffsets(), &m->GetBlock(skip_i,skip_j));
-            PtAP = RAP(Pi,A,Pj);
-            delete A;
-            pS_e->SetBlock(skip_i,skip_j,PtAP->EliminateCols(*ess_tdofs[skip_j]));
-            PtAP->EliminateRows(*ess_tdofs[skip_i]);
-         }
-         pS->SetBlock(skip_i,skip_j,PtAP);
-         skip_j++;
-      }
-      skip_i++;
-   }
-}
-
-#endif
-
-
-void BlockStaticCondensation::ConformingAssemble(int skip_zeros)
-{
-   Finalize(0);
-   if (!P) { BuildProlongation(); }
-
-   BlockMatrix * Pt = Transpose(*P);
-   BlockMatrix * PtA = mfem::Mult(*Pt, *S);
-   delete S;
-   if (S_e)
-   {
-      BlockMatrix *PtAe = mfem::Mult(*Pt, *S_e);
-      delete S_e;
-      S_e = PtAe;
-   }
-   delete Pt;
-   S = mfem::Mult(*PtA, *P);
-   delete PtA;
-
-   if (S_e)
-   {
-      BlockMatrix *PtAeP = mfem::Mult(*S_e, *P);
-      S_e = PtAeP;
-   }
-   height = S->Height();
-   width = S->Width();
-}
-
-void BlockStaticCondensation::Finalize(int skip_zeros)
-{
-   if (S) { S->Finalize(skip_zeros); }
-   if (S_e) { S_e->Finalize(skip_zeros); }
-}
-
-void BlockStaticCondensation::FormSystemMatrix(Operator::DiagonalPolicy
-                                               diag_policy)
-{
-   if (parallel)
-   {
-      FillEssTdofLists(ess_rtdof_list);
-      if (S)
-      {
-         const int remove_zeros = 0;
-         Finalize(remove_zeros);
-         ParallelAssemble(S);
-         delete S;
-         S=nullptr;
-         delete S_e;
-         S_e = nullptr;
-      }
-   }
-   else
-   {
-      if (!S_e)
-      {
-         bool conforming = true;
-         for (int i = 0; i<nblocks; i++)
-         {
-            if (!tr_fes[i]) { continue; }
-            const SparseMatrix *P_ = tr_fes[i]->GetConformingProlongation();
-            if (P_)
-            {
-               conforming = false;
-               break;
-            }
-         }
-         if (!conforming) { ConformingAssemble(0); }
-         const int remove_zeros = 0;
-         EliminateReducedTrueDofs(ess_rtdof_list, diag_policy);
-         Finalize(remove_zeros);
-      }
-   }
-}
-
-
-void BlockStaticCondensation::ConvertMarkerToReducedTrueDofs(
-   Array<int> & tdof_marker,
-   Array<int> & rtdof_marker)
-{
-   // convert tdof_marker to dof_marker
-   rtdof_marker.SetSize(0);
-   Array<int> tdof_marker0;
-   Array<int> dof_marker0;
-   Array<int> dof_marker;
-   int * data = tdof_marker.GetData();
-   for (int i = 0; i<nblocks; i++)
-   {
-      tdof_marker0.MakeRef(&data[tdof_offsets[i]],tdof_offsets[i+1]-tdof_offsets[i]);
-      const SparseMatrix * R = fes[i]->GetRestrictionMatrix();
-      if (!R)
-      {
-         dof_marker0.MakeRef(tdof_marker0);
-      }
-      else
-      {
-         dof_marker0.SetSize(fes[i]->GetVSize());
-         R->BooleanMultTranspose(tdof_marker0, dof_marker0);
-      }
-      dof_marker.Append(dof_marker0);
-   }
-
-   int rdofs = rdof_edof.Size();
-   Array<int> rdof_marker(rdofs);
-
-   for (int i = 0; i < rdofs; i++)
-   {
-      rdof_marker[i] = dof_marker[rdof_edof[i]];
-   }
-
-   // convert rdof_marker to rtdof_marker
-   Array<int> rtdof_marker0;
-   Array<int> rdof_marker0;
-   int * rdata = rdof_marker.GetData();
-   int k=0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      if (!tr_fes[i]) { continue; }
-      rdof_marker0.MakeRef(&rdata[rdof_offsets[k]],rdof_offsets[k+1]-rdof_offsets[k]);
-      const SparseMatrix *tr_R = tr_fes[i]->GetRestrictionMatrix();
-      if (!tr_R)
-      {
-         rtdof_marker0.MakeRef(rdof_marker0);
-      }
-      else
-      {
-         rtdof_marker0.SetSize(tr_fes[i]->GetTrueVSize());
-         tr_R->BooleanMult(rdof_marker0, rtdof_marker0);
-      }
-      rtdof_marker.Append(rtdof_marker0);
-      k++;
-   }
-}
-
-void BlockStaticCondensation::FillEssTdofLists(const Array<int> & ess_tdof_list)
-{
-   int j;
-   for (int i = 0; i<ess_tdof_list.Size(); i++)
-   {
-      int tdof = ess_tdof_list[i];
-      for (j = 0; j < rblocks; j++)
-      {
-         if (rtdof_offsets[j+1] > tdof) { break; }
-      }
-      ess_tdofs[j]->Append(tdof-rtdof_offsets[j]);
-   }
-}
-
-void BlockStaticCondensation::SetEssentialTrueDofs(const Array<int>
-                                                   &ess_tdof_list)
-{
-   Array<int> tdof_marker;
-   Array<int> rtdof_marker;
-   FiniteElementSpace::ListToMarker(ess_tdof_list,tdof_offsets.Last(),tdof_marker);
-   ConvertMarkerToReducedTrueDofs(tdof_marker, rtdof_marker);
-   FiniteElementSpace::MarkerToList(rtdof_marker,ess_rtdof_list);
-}
-
-void BlockStaticCondensation::EliminateReducedTrueDofs(const Array<int>
-                                                       &ess_rtdof_list,
-                                                       Matrix::DiagonalPolicy dpolicy)
-{
-
-   MFEM_VERIFY(!parallel, "EliminateReducedTrueDofs::Wrong Code path");
-
-   if (S_e == NULL)
-   {
-      Array<int> offsets;
-
-      offsets.MakeRef( (P) ? rtdof_offsets : rdof_offsets);
-
-      S_e = new BlockMatrix(offsets);
-      S_e->owns_blocks = 1;
-      for (int i = 0; i<S_e->NumRowBlocks(); i++)
-      {
-         int h = offsets[i+1] - offsets[i];
-         for (int j = 0; j<S_e->NumColBlocks(); j++)
-         {
-            int w = offsets[j+1] - offsets[j];
-            S_e->SetBlock(i,j,new SparseMatrix(h, w));
-         }
-      }
-   }
-   S->EliminateRowCols(ess_rtdof_list,S_e,dpolicy);
-}
-
-void BlockStaticCondensation::EliminateReducedTrueDofs(Matrix::DiagonalPolicy
-                                                       dpolicy)
-{
-   EliminateReducedTrueDofs(ess_rtdof_list, dpolicy);
-}
-
-void BlockStaticCondensation::ReduceSolution(const Vector &sol,
-                                             Vector &sc_sol) const
-{
-   MFEM_ASSERT(sol.Size() == dof_offsets.Last(), "'sol' has incorrect size");
-   const int nrdofs = rdof_offsets.Last();
-   Vector sol_r;
-   if (!R)
-   {
-      sc_sol.SetSize(nrdofs);
-      sol_r.SetDataAndSize(sc_sol.GetData(), sc_sol.Size());
-   }
-   else
-   {
-      sol_r.SetSize(nrdofs);
-   }
-   for (int i = 0; i < nrdofs; i++)
-   {
-      sol_r(i) = sol(rdof_edof[i]);
-   }
-   if (R)
-   {
-      // wrap vector into a block vector
-      BlockVector blsol_r(sol_r,rdof_offsets);
-      sc_sol.SetSize(R->Height());
-      R->Mult(blsol_r, sc_sol);
-   }
-}
-
-void BlockStaticCondensation::ReduceSystem(Vector &x, Vector &X,
-                                           Vector &B,
-                                           int copy_interior) const
-{
-   ReduceSolution(x, X);
-   if (parallel)
-   {
-      B.SetSize(pP->Width());
-      pP->MultTranspose(*y,B);
-
-      Vector tmp(B.Size());
-      pS_e->Mult(X,tmp);
-      B-=tmp;
-      for (int j = 0; j<rblocks; j++)
-      {
-         if (!ess_tdofs[j]->Size()) { continue; }
-         HypreParMatrix *Ah = (HypreParMatrix *)(&pS->GetBlock(j,j));
-         Vector diag;
-         Ah->GetDiag(diag);
-         for (int i = 0; i < ess_tdofs[j]->Size(); i++)
-         {
-            int tdof = (*ess_tdofs[j])[i];
-            int gdof = tdof + rtdof_offsets[j];
-            B(gdof) = diag(tdof)*X(gdof);
-         }
-      }
-   }
-   else
-   {
-      if (!P)
-      {
-         S_e->AddMult(X,*y,-1.);
-         S->PartMult(ess_rtdof_list,X,*y);
-         B.MakeRef(*y, 0, y->Size());
-      }
-      else
-      {
-         B.SetSize(P->Width());
-         P->MultTranspose(*y, B);
-         S_e->AddMult(X,B,-1.);
-         S->PartMult(ess_rtdof_list,X,B);
-      }
-   }
-   if (!copy_interior) { X.SetSubVectorComplement(ess_rtdof_list, 0.0); }
-}
-
-
-void BlockStaticCondensation::ComputeSolution(const Vector &sc_sol,
-                                              Vector &sol) const
-{
-
-   const int nrdofs = rdof_offsets.Last();
-   const int nrtdofs = rtdof_offsets.Last();
-   MFEM_VERIFY(sc_sol.Size() == nrtdofs, "'sc_sol' has incorrect size");
-
-   Vector sol_r;
-   if (parallel)
-   {
-      sol_r.SetSize(nrdofs);
-      pP->Mult(sc_sol, sol_r);
-   }
-   else
-   {
-      if (!P)
-      {
-         sol_r.SetDataAndSize(sc_sol.GetData(), sc_sol.Size());
-      }
-      else
-      {
-         sol_r.SetSize(nrdofs);
-         P->Mult(sc_sol, sol_r);
-      }
-   }
-
-
-   if (rdof_offsets.Last() == dof_offsets.Last())
-   {
-      sol = sol_r;
-      return;
-   }
-   else
-   {
-      sol.SetSize(dof_offsets.Last());
-   }
-
-   Vector lsr; // element (local) sc solution vector
-   Vector lsi; // element (local) interior solution vector
-   const int NE = mesh->GetNE();
-
-   Array<int> trace_vdofs;
-   Array<int> vdofs;
-   Array<int> tr_offsets;
-   Vector lsol;
-   for (int iel = 0; iel < NE; iel++)
-   {
-      lsol.SetSize(lmat[iel]->Width() + lmat[iel]->Height());
-      // GetReduceElementIndicesAndOffsets(iel, trace_ldofs, interior_ldofs, tr_offsets);
-      GetReduceElementVDofs(iel, trace_vdofs);
-
-      lsr.SetSize(trace_vdofs.Size());
-      sol_r.GetSubVector(trace_vdofs, lsr);
-      // complete the interior dofs
-
-      lsi.SetSize(lmat[iel]->Height());
-      lmat[iel]->Mult(lsr,lsi);
-      lsi.Neg();
-      lsi+=*lvec[iel];
-
-      Array<int> tr_idx,int_idx,idx_offs;
-      GetReduceElementIndicesAndOffsets(iel,tr_idx, int_idx, idx_offs);
-      lsol.SetSubVector(tr_idx,lsr);
-
-      lsol.SetSubVector(int_idx,lsi);
-
-      GetElementVDofs(iel, vdofs);
-      sol.SetSubVector(vdofs,lsol);
-
-   }
-
-}
-
-BlockStaticCondensation::~BlockStaticCondensation()
-{
-   delete S_e; S_e = nullptr;
-   delete S; S=nullptr;
-   delete y; y=nullptr;
-
-   if (P) { delete P; } P=nullptr;
-   if (R) { delete R; } R=nullptr;
-
-   if (parallel)
-   {
-      delete pS; pS=nullptr;
-      delete pS_e; pS_e=nullptr;
-      for (int i = 0; i<rblocks; i++)
-      {
-         delete ess_tdofs[i];
-      }
-      delete pP; pP=nullptr;
-   }
-
-   for (int i=0; i<lmat.Size(); i++)
-   {
-      delete lmat[i]; lmat[i] = nullptr;
-      delete lvec[i]; lvec[i] = nullptr;
-   }
-}
-
-}
-
@@ -1,191 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCK_STATIC_CONDENSATION
-#define MFEM_BLOCK_STATIC_CONDENSATION
-
-#include "../config/config.hpp"
-#include "fespace.hpp"
-
-#ifdef MFEM_USE_MPI
-#include "pfespace.hpp"
-#endif
-
-namespace mfem
-{
-
-
-class BlockStaticCondensation
-{
-   int height, width;
-   int nblocks; // original number of blocks
-   int rblocks; // reduces number of blocks
-   Mesh * mesh = nullptr;
-   bool parallel = false;
-   // original set of Finite Element Spaces
-   Array<FiniteElementSpace *> fes;
-   // indicates if the original space is already a trace space
-   Array<bool> IsTraceSpace;
-
-   // New set of "reduced" Finite Element Spaces
-   // (after static condensation)
-   Array<FiniteElementSpace *> tr_fes;
-
-   Array<int> dof_offsets;
-   Array<int> tdof_offsets;
-
-   Array<int> rdof_offsets;
-   Array<int> rtdof_offsets;
-
-   // Schur complement matrix
-   // S = A_ii - A_ib (A_bb)^{-1} A_bi.
-   BlockMatrix * S = nullptr;
-   BlockMatrix * S_e = nullptr;
-
-   BlockVector * y = nullptr;
-
-   Array<DenseMatrix * > lmat;
-   Array<Vector * > lvec;
-
-   Array<int> rdof_edof;      // Map from reduced dofs to exposed dofs
-   Array<int> ess_rtdof_list;
-
-   BlockMatrix * P = nullptr; // Block Prolongation
-   BlockMatrix * R = nullptr; // Block Restriction
-
-#ifdef MFEM_USE_MPI
-   BlockOperator * pS = nullptr;
-   BlockOperator * pS_e = nullptr;
-   // Block HypreParMatrix for Prolongation
-   BlockOperator * pP = nullptr;
-#endif
-
-   bool Parallel() const { return parallel; }
-
-
-   // tr_idx (trace dofs indices)
-   // int_idx (interior dof indices)
-   void GetReduceElementIndicesAndOffsets(int el, Array<int> & tr_idx,
-                                          Array<int> & int_idx,
-                                          Array<int> & offsets) const;
-
-   void GetReduceElementVDofs(int el, Array<int> & rdofs) const;
-   void GetElementVDofs(int el, Array<int> & vdofs) const;
-
-
-   //  S = A_ii - A_ib (A_bb)^{-1} A_bi.
-   //  y = y_i - A_ib (A_bb)^{-1} y_b
-   void GetLocalShurComplement(int el, const Array<int> & tr_idx,
-                               const Array<int> & int_idx,
-                               const DenseMatrix & elmat, const Vector & elvect,
-                               DenseMatrix & rmat, Vector & rvect);
-
-   void ComputeOffsets();
-
-   void BuildProlongation();
-#ifdef MFEM_USE_MPI
-   void BuildParallelProlongation();
-#endif
-
-   // ess_tdof list for each space
-   Array<Array<int> *> ess_tdofs;
-   void FillEssTdofLists(const Array<int> & ess_tdof_list);
-
-   void ConformingAssemble(int skip_zeros);
-
-   /** Restrict a marker Array on the true FE spaces dofs to a marker Array on
-    the reduced/trace true FE spaces dofs. */
-   void ConvertMarkerToReducedTrueDofs(Array<int> & tdof_marker,
-                                       Array<int> & rtdof_marker);
-public:
-
-   BlockStaticCondensation(Array<FiniteElementSpace *> & fes_);
-
-   ~BlockStaticCondensation();
-
-   void SetSpaces(Array<FiniteElementSpace*> & fes_);
-
-   void Init();
-
-   /** Assemble the contribution to the Schur complement from the given
-       element matrix 'elmat'; save the other blocks internally: A_bb_inv, A_bi,
-       and A_bi. */
-
-   void AssembleReducedSystem(int el, DenseMatrix &elmat,
-                              Vector & elvect);
-
-   /// Finalize the construction of the Schur complement matrix.
-   void Finalize(int skip_zeros = 0);
-
-   /// Determine and save internally essential reduced true dofs.
-   void SetEssentialTrueDofs(const Array<int> &ess_tdof_list);
-
-   /// Eliminate the given reduced true dofs from the Schur complement matrix S.
-   void EliminateReducedTrueDofs(const Array<int> &ess_rtdof_list,
-                                 Matrix::DiagonalPolicy dpolicy);
-
-   void EliminateReducedTrueDofs(Matrix::DiagonalPolicy dpolicy);
-
-   bool HasEliminatedBC() const
-   {
-#ifndef MFEM_USE_MPI
-      return S_e;
-#else
-      return S_e || pS_e;
-#endif
-
-   }
-
-   /// Return the serial Schur complement matrix.
-   BlockMatrix &GetMatrix() { return *S; }
-
-   /// Return the eliminated part of the serial Schur complement matrix.
-   BlockMatrix &GetMatrixElim() { return *S_e; }
-
-#ifdef MFEM_USE_MPI
-   /// Return the parallel Schur complement matrix.
-   BlockOperator &GetParallelMatrix() { return *pS; }
-
-   /// Return the eliminated part of the parallel Schur complement matrix.
-   BlockOperator &GetParallelMatrixElim() { return *pS_e; }
-
-   void ParallelAssemble(BlockMatrix *m);
-#endif
-
-   void FormSystemMatrix(Operator::DiagonalPolicy diag_policy);
-
-   /** Restrict a solution vector on the full FE space dofs to a vector on the
-       reduced/trace true FE space dofs. */
-   void ReduceSolution(const Vector &sol, Vector &sc_sol) const;
-
-   /** @brief Set the reduced solution `X` and r.h.s `B` vectors from the full
-    linear system solution `x` and r.h.s. `b` vectors.
-
-      This method should be called after the internal reduced essential dofs
-      have been set using SetEssentialTrueDofs() and both the Schur complement
-      and its eliminated part have been finalized. */
-   void ReduceSystem(Vector &x, Vector &X, Vector &B,
-                     int copy_interior = 0) const;
-
-   /** Restrict a list of true FE space dofs to a list of reduced/trace true FE
-    space dofs. */
-   void ConvertListToReducedTrueDofs(const Array<int> &ess_tdof_list,
-                                     Array<int> &ess_rtdof_list) const;
-
-   /** Given a solution of the reduced system 'sc_sol' and the RHS 'b' for the
-       full linear system, compute the solution of the full system 'sol'. */
-   void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
-
-};
-
-}
-
-#endif
@@ -112,7 +112,7 @@ static void InitBasisImpl(const FiniteElementSpace &fes,
   const bool tensor = dynamic_cast<const mfem::TensorBasisElement *>
                       (&fe) != nullptr;

-   // Init or retreive key values
+   // Init or retrieve key values
   if (basis_itr == mfem::internal::ceed_basis_map.end())
   {
      if ( tensor )
--- a/Show More
+++ b/Show More