fix path issue

Merge branch 'dpg-complex-dev' of github.com:mfem/mfem into dpg-complex-dev
small print edits
2022-08-16 14:30:09 -07:00 · 2022-08-15 17:12:29 -07:00 · 2022-08-15 17:12:11 -07:00 · 2022-08-15 16:27:04 -07:00 · 2022-07-27 06:51:45 -07:00 · 2022-07-27 01:45:43 -07:00
297 changed files with 21611 additions and 14884 deletions
@@ -23,8 +23,9 @@ install:
 - set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
 - set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include

-# Install METIS, use MFEM's mirror because the original source server is often
-# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
+# Install METIS, use a mirror because the original source server is not always
+# up. Original url:
+#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
@@ -1,4 +1,4 @@
-name: "Docker"
+name: Build Deploy Container

 on:

@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.

 # In this CI section, we build different variants of mfem and run test on them.
-name: "Tests"
+name: builds-and-tests

 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -47,17 +47,17 @@ jobs:
  builds-and-tests:
    strategy:
      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-20.04, macos-10.15, windows-2022]
        target: [dbg, opt]
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
        exclude:
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
            build-system: cmake
-          - os: macos-latest
+          - os: macos-10.15
            build-system: cmake
-          - os: windows-latest
+          - os: windows-2022
            build-system: make
        # 'include' allows us to:
        # - Add a variable to all jobs without creating a new matrix dimension.
@@ -72,15 +72,15 @@ jobs:
            codecov: NO
          - target: opt
            codecov: YES
-          - os: windows-latest
+          - os: windows-2022
            codecov: NO
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
            target: opt
            codecov: NO
            mpi: par
@@ -112,35 +112,35 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
      run: |
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install lcov

    - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
      uses: mpi4py/setup-mpi@v1.0.3

    # Get Hypre through cache, or build it.
@@ -154,7 +154,7 @@ jobs:
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
@@ -163,7 +163,7 @@ jobs:
        build-system: make

    - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
@@ -175,14 +175,14 @@ jobs:
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
@@ -196,16 +196,20 @@ jobs:
        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1

    - name: prepare binary cache location
-      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
      run: |
        mkdir -p vcpkg_cache

    - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
      env:
        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
      run: |
-        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
+        $PortFile = 'C:\vcpkg\ports\metis\portfile.cmake'
+        $OriginalURL = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-${METIS_VERSION}.tar.gz'
+        $NewURL = 'https://github.com/mfem/tpls/raw/gh-pages/metis-5.1.0.tar.gz'
+        (Get-Content $PortFile).replace($OriginalURL, $NewURL) | Set-Content $PortFile
+        vcpkg install metis --triplet=x64-windows-static

    # MFEM build and test
    - name: build
@@ -248,7 +252,7 @@ jobs:
      shell: bash

    - name: cmake unit tests (Ubuntu 20.04)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
      run: |
        CTEST_CONFIG="Release"
        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
@@ -256,7 +260,7 @@ jobs:
      shell: bash

    - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
      run: |
        CTEST_CONFIG="Release"
        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
@@ -1,70 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "Static Analysis"
-
-on:
-  push:
-    branches: [ "master", "next"]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "master" ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: "Build Analysis"
+name: build-analysis

 permissions:
  actions: write
@@ -31,7 +31,7 @@ env:

 jobs:
  gitignore:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04

    steps:
    - name: Cancel Previous Runs
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: "Checks"
+name: repo-check

 permissions:
  actions: write
@@ -28,7 +28,7 @@ on:

 jobs:
  file-headers-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -78,7 +78,7 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -88,14 +88,14 @@ jobs:

    - name: get astyle
      run: |
-        sudo apt-get install astyle
+        sudo apt-get install astyle=3.1-1ubuntu2

    - name: style check
      run: |
        ./config/githooks/pre-push --style

  documentation:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -106,8 +106,6 @@ jobs:
    - name: get doxygen and graphviz
      run: |
        sudo apt-get install doxygen graphviz
-        cd doc
-        doxygen -u CodeDocumentation.conf.in

    - name: build documentation
      run: |
@@ -120,7 +118,7 @@ jobs:
      github.ref != 'refs/heads/master' &&
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -18,11 +18,6 @@ CMakeFiles/
 # Backup files
 *~

-*.sqlite
-*.nsys-rep
-*.qdstrm
-*.csv
-
 # Default install location
 /mfem/

@@ -280,7 +275,6 @@ miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric

 miniapps/toys/automata
 miniapps/toys/life
@@ -313,7 +307,6 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh

-miniapps/multidomain/multidomain
 miniapps/hooke/hooke

 # Unit test binary and outputs
@@ -334,7 +327,6 @@ tests/benchmarks/bench_ceed
 tests/benchmarks/bench_tmop
 tests/benchmarks/bench_vector
 tests/benchmarks/bench_virtuals
-tests/benchmarks/bench_lor

 # Test script output
 tests/scripts/*.err
@@ -8,48 +8,31 @@
                               https://mfem.org


-Version 4.5, released on October 22, 2022
-=========================================
+Version 4.4.1 (development)
+===========================

 Meshing improvements
 --------------------
- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
-  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
-  and work with all existing MFEM interfaces like finite element spaces etc.
-
- Added a method, ParMesh::GetSerialMesh(), that reconstructs a partitioned
-  parallel mesh on a given single rank. Also, added ParMesh::PrintAsSerial(),
-  which saves the reconstructed serial mesh to a C++ stream on rank 0.
-
- Added more 3D TMOP metrics, as well as specialized metrics for mesh
-  untangling and worst-case quality improvement.
-
- Added a new method, Mesh::NodesUpdated, which should be called after the mesh
-  node coordinates have changed, e.g. after the mesh has moved. This is
-  necessary, for example, with device assembly of linear and bilinear forms.
-
 - Added support for mixed meshes and pyramids in GSLIB-FindPoints.

 Discretization improvements
 ---------------------------
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration.
+
+- Added support for partial assembly and fully matrix-free operators on mixed
+  meshes (different element types and p-adaptivity) through libCEED, including
+  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
+  currently limited by MFEM capabilities, i.e. 2D serial meshes. All mixed
+  element topologies are supported in serial and parallel: segment, triangle,
+  square, tetrahedron, cube, prism, and pyramid.
+
 - Added full assembly and device support for several LinearForm integrators:
  * DomainLF: (f, v)
  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
  * DomainLFGrad: (f, grad(v))
  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
-  The device assembly of linear forms has to be explicitly enabled by calling
-  LinearForm::UseFastAssembly(true), otherwise the legacy linear form assembly
-  is used by default.
-
- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration with arbitrary user-supplied coefficients.
-
- Added a new class FaceQuadratureSpace that allows for the construction of
-  QuadratureFunctions on the interior or boundary faces of a mesh.
-
- Added a class CoefficientVector for efficient access of variable coefficient
-  values at quadrature points (in particular for GPU/device kernels).

 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
  spatial Gaussian white noise.
@@ -57,25 +40,8 @@ Discretization improvements
 - Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
  See fem/estimators.hpp.

- Various fixes and improvements in LinearFormExtension.
-
 Linear and nonlinear solvers
 ----------------------------
- Added a new class DGMassInverse that performs a local element-wise CG
-  iteration to solve systems involving the discontinuous Galerkin mass matrix,
-  including support for device/GPU acceleration.
-
- Added more flexibility to the constrained solver classes:
-  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
-    (necessary for penalty contact)
-  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
-  * All constraint solver classes can take a user-defined preconditioner
-
- - Added functions to toggle additional options for the SuperLU_Dist and Hypre
-   preconditioners (ParaSails, Euclid, ILU).
-
- Added boundary elimination with device support for `SparseMatrix` and
-  `HypreParMatrix`.

 New and updated examples and miniapps
 -------------------------------------
@@ -85,28 +51,12 @@ New and updated examples and miniapps
  automatic differentiation tools like a native dual number implementation or a
  third party library such as Enzyme. See miniapps/elasticity for more details.

- Added example for body-fitted volumetric and shape integration using the
-  Algoim library in miniapps/shifted.
-
 - Add a new example code, Example 33/33p, to demonstrate the solution of
  spectral fractional PDEs with MFEM.

 Integrations, testing and documentation
 ---------------------------------------
 - Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
-  More sophisticated developer containers are available in the new repo
-  https://github.com/mfem/containers.
-
- Added support for the LLVM-based automatic differentiation tool Enzyme, see
-  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
-  header are provided. The functionality and interaction are demonstrated in a
-  new miniapp in miniapps/elasticity.
-
- Added support for partial assembly and fully matrix-free operators on mixed
-  meshes (different element types and p-adaptivity) through libCEED, including
-  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
-  currently limited to 2D serial meshes. All mixed element topologies are
-  supported in both serial and parallel.

 - Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
  which provides parallel non-conforming, non-matching, variational, volumetric
@@ -114,40 +64,29 @@ Integrations, testing and documentation
  between arbitrarily distributed and unrelated finite element meshes in a
  variationally consistent way.

- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
-  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
-  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.
+- Added support for the LLVM-based automatic differentiation tool Enzyme, see
+  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
+  header are provided. The functionality and interaction are demonstrated in a
+  new miniapp in miniapps/elasticity.

- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+- Added example for body-fitted volumetric and shape integration using the
+  Algoim library.

 - Added Windows 2022 CI testing with GitHub actions.

 Miscellaneous
 -------------
- The method SparseMatrix::EnsureMultTranspose() is now automatically called
-  by the methods AddMultTranspose(), MultTranspose(), and AbsMultTranspose().
-  Added a method with the same name to class HypreParMatrix which is also called
-  automatically by the HypreParMatrix::MultTranspose() methods.
+- Various other simplifications, extensions, and bugfixes in the code.

- Updated various MemoryUsage methods to return 'std::size_t' instead of 'long'
-  since the latter is 32-bit in Win64 builds.

+- Added boundary elimination with device support for `SparseMatrix` and
+  `HypreParMatrix`.
+  
 - When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).

- In various places in the library, replace the use of 'long' with 'long long'
-  to better support Win64 builds where 'long' is 32-bit and 'long long' is
-  64-bit. On Linux and MacOS, both types are typically 64-bit.
-
- The behavior of GridFunction::GetTrueVector() has been changed to not return
-  an empty true vector.
-
- Added support for ordering search points byVDIM in FindPointsGSLIB.
-
- Various other simplifications, extensions, and bugfixes in the code.
-
+- Added TMOP metrics for mesh untangling and worst-case quality improvement.

 Version 4.4, released on March 21, 2022
 =======================================
@@ -180,6 +119,11 @@ Meshing improvements
 - Added a simpler interface to access mesh face information, see FaceInformation
  and GetFaceInformation in the Mesh class.

+- Added the method ParMesh::GetSerialMesh() that reconstructs a partitioned
+  parallel mesh on a given single rank. Also, added the method
+  ParMesh::PrintAsSerial() that saves the reconstructed serial mesh to a C++
+  stream on rank 0.
+
 - Gmsh meshes where all elements have zero physical tag (the default Gmsh output
  format if no physical groups are defined) are now successfully loaded, and
  elements are reassigned attribute number 1.
@@ -269,6 +213,9 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
  formatting. See the "make style" target.

+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -279,6 +226,13 @@ Miscellaneous

 - Fixed several MinGW build issues on Windows.

+- In various places in the library, replace the use of 'long' with 'long long'
+  to better support Win64 builds where 'long' is 32-bit and 'long long' is
+  64-bit. On Linux and MacOS, both types are typically 64-bit.
+
+- Update various "MemoryUsage" methods to return 'std::size_t' instead of 'long'
+  since the latter is 32-bit in Win64 builds.
+
 - Added 'double' atomicAdd implementation for previous versions of CUDA.

 - HypreParVector and Vector now support C++ move semantics, and the copy
@@ -51,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.0)
+set(${PROJECT_NAME}_VERSION 4.4.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -81,10 +81,6 @@ if (MFEM_USE_STRUMPACK)
  # Just needed to find the MPI_Fortran libraries to link with
  set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS >= 6.4.0 requires C++14:
-if (MFEM_USE_SUNDIALS AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
-  set(CMAKE_CXX_STANDARD 14)
-endif()
 if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
  set(CMAKE_CXX_STANDARD 14)
 endif()
@@ -141,7 +137,7 @@ if (MFEM_USE_CUDA)
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
   set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
+   set(CUSBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -204,10 +200,10 @@ if (MFEM_USE_MPI)
  find_package(MPI REQUIRED)
  set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
  if (MFEM_MPIEXEC)
-    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
+    set(MPIEXEC ${MFEM_MPIEXEC})
  endif()
  if (MFEM_MPIEXEC_NP)
-    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
+    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
  endif()
  # Parallel MFEM depends on hypre
  find_package(HYPRE REQUIRED)
@@ -481,21 +477,12 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
  endif()
 endif()

-# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
-# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
-#   Target "mfem" links to target "Threads::Threads" but the target was not
-#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
-#   an ALIAS target is missing?
-# Call Stack (most recent call first):
-#   CMakeLists.txt:474 (mfem_add_library)
-find_package(Threads REQUIRED)
-
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist METIS SuiteSparse SUNDIALS
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
@@ -102,9 +102,7 @@ The MFEM source code has the following structure:
  .
  ├── config
  │   ├── cmake
-  │   ├── docker
-  │   ├── githooks
-  │   └── vcpkg
+  │   └── githooks
  ├── data
  ├── doc
  ├── examples
@@ -113,7 +111,6 @@ The MFEM source code has the following structure:
  │   ├── ginkgo
  │   ├── hiop
  │   ├── jupyter
-  │   ├── moonolith
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
@@ -121,15 +118,13 @@ The MFEM source code has the following structure:
  ├── fem
  │   ├── ceed
  │   ├── fe
-  │   ├── lor
-  │   ├── moonolith
  │   ├── qinterp
+  │   ├── moonolith
  │   └── tmop
  ├── general
  ├── linalg
  │   └── simd
  ├── mesh
-  │   └── submesh
  ├── miniapps
  │   ├── adjoint
  │   ├── autodiff
@@ -139,7 +134,6 @@ The MFEM source code has the following structure:
  │   ├── hooke
  │   ├── meshing
  │   ├── mtop
-  │   ├── multidomain
  │   ├── navier
  │   ├── nurbs
  │   ├── parelag
@@ -7,10 +7,6 @@

                               https://mfem.org

-This file provides a detailed description of how to build and install the MFEM
-library. For a simple build, see the step-by-step instructions on the website
-at https://mfem.org/building.
-
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
 of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
@@ -20,11 +16,7 @@ requires an MPI C++ compiler, as well as the following external libraries:
  https://github.com/hypre-space/hypre

 - METIS (a family of multilevel partitioning algorithms)
-  https://github.com/mfem/tpls
-
-  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
-  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
-  and we don't support yet the new repo https://github.com/KarypisLab/METIS.
+  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview

 The hypre dependency can be downloaded as a tarball from GitHub or from the
 project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
@@ -480,10 +472,10 @@ MFEM_USE_CODIPACK = YES/NO
  www.scicomp.uni-kl.de/codi/

 MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical
-  methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM
-  provides interface to Algoim v1. Thus, to check out the specific state use:
+  Enable the usage of Algoim - a collection of high-order accurate numerical 
+  methods and C++ algorithms for working with implicitly-defined geometry and 
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
+  provides interface to Algoim v1. Thus, to check out the specific state use: 
  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
  https://algoim.github.io

@@ -558,7 +550,7 @@ MFEM_USE_FMS = YES/NO
   Enables support for the FMS library which consists of the DataCollection
   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   conversion routines between FMS's FmsDataCollection structure and MFEM's
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
   DataCollection class, see the header file fem/fmsconvert.hpp.

 MFEM_USE_PARELAG = YES/NO
@@ -605,7 +597,7 @@ The specific libraries and their options are:

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
+  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.

@@ -762,12 +754,12 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.7.

- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
+- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
  headers so it just needs to be downloaded at the same level as MFEM. Download
  the specific version we use as:
  "git clone https://github.com/algoim/algoim.git;
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
+  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
  "ln -s blitz-1.0.2 blitz".
  Build Blitz using CMake as:
@@ -19,7 +19,7 @@ if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
  # Set ENZYME_FOUND
  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)

-  # Set CXX flags to accommodate the Enzyme Clang plugin
+  # Set CXX flags to accomodate the Enzyme Clang plugin
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
  set(MFEM_USE_ENZYME YES)
 else()
@@ -1,57 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - HDF5_FOUND - If HDF5 was found
-#   - HDF5_LIBRARIES - The HDF5 libraries
-#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
-
-# First Check for HDF5_DIR
-if(NOT HDF5_DIR)
-    MESSAGE(FATAL_ERROR "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
-endif()
-
-# Find includes
-find_path( HDF5_INCLUDE_DIRS hdf5.h
-           PATHS  ${HDF5_DIR}/include/
-           NO_DEFAULT_PATH
-           NO_CMAKE_ENVIRONMENT_PATH
-           NO_CMAKE_PATH
-           NO_SYSTEM_ENVIRONMENT_PATH
-           NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
-
-include(FindPackageHandleStandardArgs)
-
-# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
-# listed variables are TRUE
-find_package_handle_standard_args(HDF5  DEFAULT_MSG
-                                  HDF5_INCLUDE_DIRS
-                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY
-                                  HDF5_LIBRARIES )
@@ -14,6 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-find_package(umpire REQUIRED CONFIG)
-set(UMPIRE_FOUND ${umpire_FOUND})
-set(UMPIRE_LIBRARIES "umpire")
+include(MfemCmakeUtilities)
+mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
+  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")
@@ -43,14 +43,22 @@ function(convert_filenames_to_full_paths NAMES)
  set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()

-# Wrapper for add_executable
+# Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
-  add_executable(${NAME} ${ARGN})
+  if (MFEM_USE_HIP)
+    add_executable(${NAME} ${ARGN})
+  else()
+    add_executable(${NAME} ${ARGN})
+  endif()
 endmacro()

-# Wrapper for add_library
+# Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
-  add_library(${NAME} ${ARGN})
+  if (MFEM_USE_HIP)
+    add_library(${NAME} ${ARGN})
+  else()
+    add_library(${NAME} ${ARGN})
+  endif()
 endmacro()

 # Simple shortcut to add_custom_target() with option to add the target to the
@@ -31,11 +31,9 @@

 // Windows specific options
 #ifdef _WIN32
-#ifndef _USE_MATH_DEFINES
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().
@@ -179,7 +179,7 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
 endif

 # ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
 # the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
 ifeq ($(MFEM_USE_HIP),YES)
   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
@@ -251,16 +251,12 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-# SUNDIALS >= 6.4.0 requires C++14:
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   BASE_FLAGS = -std=c++14
-endif
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
- $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
- -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
 -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+
 ifeq ($(MFEM_USE_MPI),YES)
   SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
@@ -313,7 +309,7 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortran library:
+# Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran

 # MUMPS library configuration
@@ -554,14 +554,15 @@ function go()
   local cmd_line="${1##+( )}"
   cmd_line="${cmd_line%%+( )}"
   shopt -u extglob
+   eval local cmd=(${cmd_line})
   local res=""
   echo $sep
   echo "<${group}>" "${cmd_line}"
   echo $sep
   if [ "${timing}" == "yes" ]; then
-      timed_run eval "${cmd_line}"
+      timed_run "${cmd[@]}"
   else
-      eval "${cmd_line}"
+      "${cmd[@]}"
   fi
   if [ "$?" -eq 0 ]; then
      res="${green}  OK  ${none}"
@@ -1,8 +0,0 @@
--- a/CMakeLists.txt	Wed Dec 21 18:24:22 2016
-+++ b/CMakeLists.txt	Wed Dec 21 18:24:26 2016
-@@ -20,4 +20,4 @@
- # Recursively look for CMakeLists.txt in subdirs.
- add_subdirectory("include")
- add_subdirectory("libmetis")
-add_subdirectory("programs")
-+# add_subdirectory("programs")
@@ -1,15 +0,0 @@
--- a/CMakeLists.txt	Sat Mar 30 17:24:45 2013
-+++ b/CMakeLists.txt	Wed Dec 21 18:23:43 2016
-@@ -4,11 +4,7 @@
- set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
- set(SHARED FALSE CACHE BOOL "build a shared library")
- 
-if(MSVC)
-  set(METIS_INSTALL FALSE)
-else()
-  set(METIS_INSTALL TRUE)
-endif()
-+set(METIS_INSTALL TRUE)
- 
- # Configure libmetis library.
- if(SHARED)
@@ -1,34 +0,0 @@
-diff --git a/include/metis.h b/include/metis.h
-index dc5406a..7732437 100644
--- a/include/metis.h
-+++ b/include/metis.h
-@@ -72,10 +72,14 @@ typedef __int64 int64_t;
- #define PRId64       "I64d"
- #define SCNd32       "ld"
- #define SCNd64       "I64d"
-+#ifdef _WIN32
-+#include <stdint.h>
-+#else
- #define INT32_MIN    ((int32_t)_I32_MIN)
- #define INT32_MAX    _I32_MAX
- #define INT64_MIN    ((int64_t)_I64_MIN)
- #define INT64_MAX    _I64_MAX
-+#endif
- #else
- #include <inttypes.h>
- #endif
-diff --git a/GKlib/gk_arch.h b/GKlib/gk_arch.h
-index 78b1431..7258763 100644
--- a/GKlib/gk_arch.h
-+++ b/GKlib/gk_arch.h
-@@ -32,8 +32,8 @@
- 
- 
- #ifdef __MSC__ 
-  #include "ms_stdint.h"
-  #include "ms_inttypes.h"
-+  #include <stdint.h>
-+  #include <inttypes.h>
-   #include "ms_stat.h"
- #else
- #ifndef SUNOS
@@ -1,11 +0,0 @@
--- a/GKlib/gk_arch.h	Wed Dec 21 18:34:18 2016
-+++ b/GKlib/gk_arch.h	Wed Dec 21 18:30:49 2016
-@@ -58,7 +58,7 @@
- #define PTRDIFF_MAX  INT64_MAX
- #endif
- 
-#ifdef __MSC__
-+#if defined(__MSC__) && (_MSC_VER < 1900)
- /* MSC does not have rint() function */
- #define rint(x) ((int)((x)+0.5))  
- 
@@ -1,14 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index e94f050..b9613a7 100644
--- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -1,7 +1,8 @@
- cmake_minimum_required(VERSION 2.8)
- project(METIS)
- 
-set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
-+set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
-+
- set(SHARED FALSE CACHE BOOL "build a shared library")
- 
- set(METIS_INSTALL TRUE)
@@ -1,11 +0,0 @@
--- a/libmetis/metislib.h	Sat Mar 30 17:24:45 2013
-+++ b/libmetis/metislib.h	Wed Dec 21 18:30:59 2016
-@@ -31,7 +31,7 @@
- #include <proto.h>
- 
- 
-#if defined(COMPILER_MSC)
-+#if defined(COMPILER_MSC) && (_MSC_VER < 1900)
- #if defined(rint)
-   #undef rint
- #endif
@@ -1,10 +0,0 @@
--- a/libmetis/CMakeLists.txt	Sat Mar 30 17:24:45 2013
-+++ b/libmetis/CMakeLists.txt	Wed Dec 21 17:41:37 2016
-@@ -11,6 +11,6 @@
- if(METIS_INSTALL)
-   install(TARGETS metis
-     LIBRARY DESTINATION lib
-    RUNTIME DESTINATION lib
-+    RUNTIME DESTINATION bin
-     ARCHIVE DESTINATION lib)
- endif()
@@ -1,44 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index b9613a7..e43ffee 100644
--- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -22,3 +22,23 @@ include_directories(include)
- add_subdirectory("include")
- add_subdirectory("libmetis")
- # add_subdirectory("programs")
-+
-+if(METIS_INSTALL)
-+  set(PRJ_NAME metis)
-+  set(PRJ_VER 5.1.0)
-+  install(EXPORT metisTargets 
-+    FILE ${PRJ_NAME}Targets.cmake
-+    DESTINATION lib/cmake/${PRJ_NAME})
-+  include(CMakePackageConfigHelpers)
-+  write_basic_package_version_file(
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
-+    VERSION ${PRJ_VER}
-+    COMPATIBILITY SameMajorVersion)
-+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
-+    "include(\${CMAKE_CURRENT_LIST_DIR}/${PRJ_NAME}Targets.cmake)")
-+  install(FILES 
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
-+    DESTINATION lib/cmake/${PRJ_NAME})
-+endif()
-+
-diff --git a/libmetis/CMakeLists.txt b/libmetis/CMakeLists.txt
-index 7a5fc74..5a68cf0 100644
--- a/libmetis/CMakeLists.txt
-+++ b/libmetis/CMakeLists.txt
-@@ -9,8 +9,9 @@ if(UNIX)
- endif()
- 
- if(METIS_INSTALL)
-  install(TARGETS metis
-+  install(TARGETS metis EXPORT metisTargets
-     LIBRARY DESTINATION lib
-     RUNTIME DESTINATION bin
-    ARCHIVE DESTINATION lib)
-+    ARCHIVE DESTINATION lib
-+    INCLUDES DESTINATION include)
- endif()
@@ -1,41 +0,0 @@
-vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
-set(OPTIONS -DSHARED=OFF)
-
-set(METIS_VERSION 5.1.0)
-
-vcpkg_download_distfile(ARCHIVE
-    URLS "https://github.com/mfem/tpls/raw/gh-pages/metis-${METIS_VERSION}.tar.gz"
-    FILENAME "metis-${METIS_VERSION}.tar.gz"
-    SHA512 deea47749d13bd06fbeaf98a53c6c0b61603ddc17a43dae81d72c8015576f6495fd83c11b0ef68d024879ed5415c14ebdbd87ce49c181bdac680573bea8bdb25
-)
-
-vcpkg_extract_source_archive_ex(
-    OUT_SOURCE_PATH SOURCE_PATH
-    ARCHIVE ${ARCHIVE}
-    REF ${METIS_VERSION}
-    PATCHES
-        enable-install.patch
-        disable-programs.patch
-        fix-runtime-install-destination.patch
-        fix-metis-vs14-math.patch
-        fix-gklib-vs14-math.patch
-        fix-linux-build-error.patch
-        install-metisConfig.patch
-        fix-INT_MIN_define.patch
-)
-
-vcpkg_configure_cmake(
-    SOURCE_PATH ${SOURCE_PATH}
-    PREFER_NINJA
-    OPTIONS ${OPTIONS}
-)
-
-vcpkg_install_cmake()
-vcpkg_copy_pdbs()
-vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/metis)
-
-file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
-
-# Handle copyright
-file(COPY ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/metis)
-file(INSTALL ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
@@ -1,7 +0,0 @@
-{
-  "name": "metis-mfem",
-  "version-string": "5.1.0",
-  "port-version": 0,
-  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
-  "homepage": "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
-}
@@ -1,7 +1,7 @@
 MFEM mesh v1.0

 #
-# MFEM Geometry Types (see mesh/geom.hpp):
+# MFEM Geomety Types (see mesh/geom.hpp):
 #
 # POINT       = 0
 # SEGMENT     = 1
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.5.0
+PROJECT_NUMBER         = v4.4.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -2349,7 +2349,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.

-DOT_GRAPH_MAX_NODES    = 100
+DOT_GRAPH_MAX_NODES    = 50

 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the
@@ -0,0 +1,310 @@
+// Example run: ./FOSLS2D_maxwell -ref 4  -o 3 -sol 1 -k 3.0
+
+//  ∇ × E - ω H = 0
+// -ω E + ∇ × H = J
+
+// --------------------------------------------------------------------------
+// |   |            E             |             H              |     RHS    | 
+// --------------------------------------------------------------------------
+// | F | (∇ × E,∇ × F)+ ω^2 (E,F) | - ω (∇ × H,F) - ω (H,curF) | - ω (J,F)  |
+// |   |                          |                            |            |
+// | G |-ω (E,∇ × G)-ω (∇ × E,G)  | (∇ × H,∇ × G)+ ω^2(H,G)    | (J,∇ × G)  |
+
+// for E in H1 (scalar) we have ∇ × E = [0 1;-1 0] ∇ E
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Define exact solution
+double E_exact(const Vector &x);
+void H_exact(const Vector &x, Vector &H);
+double frhs(const Vector &x);
+void fvrhs(const Vector &x, Vector &f);
+void get_maxwell_solution(const Vector &x, double & E, Vector & curlE, double & curl2E);
+
+int dim;
+double omega;
+int isol = 0;
+
+int main(int argc, char *argv[])
+{
+   StopWatch chrono;
+
+   // 1. Parse command-line options.
+   // geometry file
+   const char *mesh_file = "../data/star.mesh";
+   // finite element order of approximation
+   int order = 1;
+   // visualization flag
+   bool visualization = 1;
+   int ref = 1;
+   // number of wavelengths
+   double k = 0.6;
+   // optional command line inputs
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&ref, "-ref", "--init-refinements",
+                  "Number of initial mesh refinements");
+   args.AddOption(&k, "-k", "--wavelengths",
+                  "Number of wavelengths.");
+   args.AddOption(&isol, "-sol", "--solution",
+                  "Exact Solution: 0) Polynomial, 1) Sinusoidal.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * k;
+
+   // Mesh mesh(1, 1, Element::QUADRILATERAL, true, 1.0, 1.0, false);
+   Mesh mesh(mesh_file, 1, 1);
+
+   dim = mesh.Dimension();
+   if (dim == 3) {MFEM_ABORT("This is 2D Maxwell")};
+
+   for (int i = 0; i < ref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   H1_FECollection H1fec(order,dim);
+   FiniteElementSpace H1fes(&mesh, &H1fec);
+
+   ND_FECollection NDfec(order, dim);
+   FiniteElementSpace NDfes(&mesh, &NDfec);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      // Essential BC on E. Nothing on H
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+   Array<int> block_offsets(3);
+   block_offsets[0] = 0;
+   block_offsets[1] = H1fes.GetVSize();
+   block_offsets[2] = NDfes.GetVSize();
+   block_offsets.PartialSum();
+
+   BlockVector x(block_offsets), b(block_offsets);
+   x = 0.0;
+   b = 0.0;
+
+   FunctionCoefficient Eex(E_exact);
+   VectorFunctionCoefficient Hex(dim, H_exact);
+   GridFunction E_gf;
+   GridFunction H_gf;
+   E_gf.MakeRef(&H1fes, x.GetBlock(0));
+   E_gf.ProjectBdrCoefficient(Eex,ess_bdr);
+   H_gf.MakeRef(&NDfes, x.GetBlock(1));
+
+   FunctionCoefficient f(frhs);
+   ProductCoefficient f_E(-omega, f);
+   VectorFunctionCoefficient f_H(1,fvrhs);
+   LinearForm b_E;
+   b_E.Update(&H1fes, b.GetBlock(0), 0);
+   b_E.AddDomainIntegrator(new DomainLFIntegrator(f_E));
+   b_E.Assemble();
+
+   LinearForm b_H;
+   b_H.Update(&NDfes, b.GetBlock(1), 0);
+   b_H.AddDomainIntegrator(new VectorFEDomainLFCurlIntegrator(f_H));
+   b_H.Assemble();
+
+   // 7. Bilinear form a(.,.) on the finite element space
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient omeg2(pow(omega, 2));
+   ConstantCoefficient negomega(-(omega));
+   DenseMatrix mat(2);
+   mat(0,0) = 0.; mat(0,1) = 1.;
+   mat(1,0) = -1.; mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(mat);
+
+   BilinearForm a_EE(&H1fes);
+   a_EE.AddDomainIntegrator(new DiffusionIntegrator(one));
+   a_EE.AddDomainIntegrator(new MassIntegrator(omeg2));
+   a_EE.Assemble();
+   a_EE.EliminateEssentialBC(ess_bdr, x.GetBlock(0), b.GetBlock(0));
+   a_EE.Finalize();
+   SparseMatrix &A_EE = a_EE.SpMat();
+
+   ScalarMatrixProductCoefficient c1(-omega, rot);
+   MixedBilinearForm a_EH(&H1fes,&NDfes);
+   // - omega (rot grad E, G) - (omega E, curl G)
+   a_EH.AddDomainIntegrator(new MixedVectorGradientIntegrator(c1));
+   a_EH.AddDomainIntegrator(new MixedScalarWeakCurlIntegrator(negomega));
+   a_EH.Assemble();
+   a_EH.EliminateTrialDofs(ess_bdr, x.GetBlock(0), b.GetBlock(1));
+   a_EH.Finalize();
+   SparseMatrix &A_EH = a_EH.SpMat();
+   SparseMatrix * A_HE = Transpose(A_EH);
+
+   BilinearForm a_HH(&NDfes);
+   a_HH.AddDomainIntegrator(new CurlCurlIntegrator(one)); // one is the coeff
+   a_HH.AddDomainIntegrator(new VectorFEMassIntegrator(omeg2)); // one is the coeff
+   a_HH.Assemble();
+   a_HH.Finalize();
+   SparseMatrix &A_HH = a_HH.SpMat();
+
+   BlockMatrix LS_Maxwellop(block_offsets);
+   LS_Maxwellop.SetBlock(0, 0, &A_EE);
+   LS_Maxwellop.SetBlock(0, 1, A_HE);
+   LS_Maxwellop.SetBlock(1, 0, &A_EH);
+   LS_Maxwellop.SetBlock(1, 1, &A_HH);
+
+   UMFPackSolver invE;
+   invE.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invE.SetOperator(LS_Maxwellop.GetBlock(0,0));
+
+   UMFPackSolver invH;
+   invH.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
+   invH.SetOperator(LS_Maxwellop.GetBlock(1,1));
+
+   BlockDiagonalPreconditioner prec(block_offsets);
+   prec.SetDiagonalBlock(0, &invE);
+   prec.SetDiagonalBlock(1, &invH);
+
+   int maxit(5000);
+   double rtol(1.e-16);
+   double atol(0.0);
+
+   CGSolver pcg;
+   pcg.SetAbsTol(atol);
+   pcg.SetRelTol(rtol);
+   pcg.SetMaxIter(maxit);
+   pcg.SetOperator(LS_Maxwellop);
+   pcg.SetPreconditioner(prec);
+   pcg.SetPrintLevel(3);
+   pcg.Mult(b, x);
+
+   int order_quad = max(2, 2 * order + 1);
+   const IntegrationRule *irs[Geometry::NumGeom];
+   for (int i = 0; i < Geometry::NumGeom; ++i)
+   {
+      irs[i] = &(IntRules.Get(i, order_quad));
+   }
+
+   double Error_E = E_gf.ComputeL2Error(Eex, irs);
+   double Error_H = H_gf.ComputeL2Error(Hex, irs);
+   
+   cout << "|| E_h - E || = " << Error_E << "\n";
+   cout << "|| H_h - H || = " << Error_H << "\n";
+   cout << "Total error = " << sqrt(Error_H*Error_H+Error_E*Error_E) << "\n";
+
+   GridFunction E_exgf(&H1fes);
+   E_exgf.ProjectCoefficient(Eex);
+
+   GridFunction H_exgf(&NDfes);
+   H_exgf.ProjectCoefficient(Hex);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int visport = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      socketstream ex_sock(vishost, visport);
+      ex_sock.precision(8);
+      socketstream sol_sockH(vishost, visport);
+      sol_sockH.precision(8);
+      socketstream ex_sockH(vishost, visport);
+      ex_sockH.precision(8);
+      sol_sock << "solution\n"
+               << mesh << E_gf << "window_title 'Numerical E'" << "keys rRljc\n"
+               << flush;
+      ex_sock << "solution\n"
+               << mesh << E_exgf << "window_title 'Exact E'" << "keys rRljc\n"
+               << flush;
+      sol_sockH << "solution\n"
+               << mesh << H_gf << "window_title 'Numerical H'" << "keys rRljc\n"
+               << flush;
+      ex_sockH << "solution\n"
+               << mesh << H_exgf << "window_title 'Exact H'" << "keys rRljc\n"
+               << flush;        
+   }
+   delete A_HE;
+   return 0;
+}
+
+double E_exact(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   return E;  //Scalar
+}
+
+//define exact solution
+void H_exact(const Vector &x, Vector &H)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   H[0] = curlE[0]/omega;
+   H[1] = curlE[1]/omega;
+}
+
+double frhs(const Vector &x)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+
+   // - omega E + curl H = f
+   // - omega E + curl (curl E) / omega = f
+   double f = - omega * E + curl2E / omega;
+   return f;
+}
+
+void fvrhs(const Vector &x, Vector &f)
+{
+   double E, curl2E;
+   Vector curlE(2);
+   get_maxwell_solution(x, E, curlE, curl2E);
+   f[0] = - omega * E + curl2E / omega;
+}
+
+
+void get_maxwell_solution(const Vector &X, double & E, Vector & curlE, double & curl2E)
+{
+   double x = X[0];
+   double y = X[1];
+   double Ex, Ey, Exx, Eyy;
+   if (isol == 0) // polynomial
+   {
+      E = x * (1.0 - x) * y * (1.0 - y);
+      Ex = (1.0 - 2.0 * x) * y * (1.0 - y);
+      Ey = x * (1.0 - x) * (1.0 - 2.0 * y);
+
+      Exx = -2.0 * y * (1.0 - y);
+      Eyy = -2.0 * x * (1.0 - x);
+   }
+   else 
+   {
+      double s = omega * (y+x);
+      E = cos(s);
+      Ex = -omega * sin(s);  
+      Ey = Ex;
+      Exx = - omega * omega * E;  
+      Eyy = Exx;  
+   }
+   curlE[0] = Ey;
+   curlE[1] = -Ex;
+   curl2E = -Exx - Eyy;
+}
@@ -0,0 +1,61 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/EM-diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = primal_dpg
+	 				
+PAR_EXAMPLES = 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf ParaView
+
+clean-exec:
@@ -0,0 +1,201 @@
+//                     MFEM primal_dpg example
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+void E_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+int dim;
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+   int ref = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.AddOption(&freq, "-f", "--frequency", "Set the frequency for the exact"
+                  " solution.");   
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");
+   args.AddOption(&ref, "-ref", "--refinements",
+                  "Number of refinements.");                         
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   kappa = freq * M_PI;
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+
+   for (int i = 0; i<ref; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   dim = mesh.Dimension();
+   int sdim = mesh.SpaceDimension();
+   // 3. Define a finite element space on the mesh. Here we use H1 continuous
+   //    high-order Lagrange finite elements of the given order.
+   ND_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace NDfes(&mesh, &fec);
+
+   FiniteElementCollection * trace_fec = nullptr;
+   if (dim == 3)
+   {
+      trace_fec = new ND_Trace_FECollection(order,mesh.Dimension());
+   }
+   else
+   {
+      trace_fec = new H1_Trace_FECollection(order,mesh.Dimension());
+   }
+   FiniteElementSpace trace_fes(&mesh, trace_fec);
+
+   int test_order = order+1;
+
+   ND_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+
+   trial_fes.Append(&NDfes);
+   trial_fes.Append(&trace_fes);
+   test_fecs.Append(&test_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+
+
+   ConstantCoefficient one(1.0);
+   a->AddTrialIntegrator(new CurlCurlIntegrator(one),0,0);
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(one),0,0);
+   a->AddTrialIntegrator(new TangentTraceIntegrator,1,0);
+   
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),0,0);
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+
+   VectorFunctionCoefficient f(sdim, f_exact);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
+
+
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      NDfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+
+   Vector X,B;
+   OperatorPtr Ah;
+
+   VectorFunctionCoefficient E(sdim, E_exact);
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = NDfes.GetVSize();
+   offsets[2] = trace_fes.GetVSize();
+   offsets.PartialSum();
+
+   BlockVector x(offsets);
+   x = 0.;
+
+
+   GridFunction E_gf(&NDfes);
+   E_gf.MakeRef(&NDfes,x.GetBlock(0));
+   E_gf.ProjectBdrCoefficientTangent(E,ess_bdr);
+   E_gf.ProjectCoefficient(E);
+
+
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+
+
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   GMRESSolver cg;
+   cg.SetRelTol(1e-8);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   E_gf.MakeRef(&NDfes,x.GetData());
+
+   double L2Error = E_gf.ComputeL2Error(E);
+   mfem::out << "L2_error = " << L2Error << endl;
+
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << E_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+   delete trace_fec;
+   return 0;
+}
+
+
+void E_exact(const Vector &x, Vector &E)
+{
+   if (dim == 3)
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(2));
+      E(2) = sin(kappa * x(0));
+   }
+   else
+   {
+      E(0) = sin(kappa * x(1));
+      E(1) = sin(kappa * x(0));
+      if (x.Size() == 3) { E(2) = 0.0; }
+   }
+}
+
+void f_exact(const Vector &x, Vector &f)
+{
+   if (dim == 3)
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(2));
+      f(2) = (1. + kappa * kappa) * sin(kappa * x(0));
+   }
+   else
+   {
+      f(0) = (1. + kappa * kappa) * sin(kappa * x(1));
+      f(1) = (1. + kappa * kappa) * sin(kappa * x(0));
+      if (x.Size() == 3) { f(2) = 0.0; }
+   }
+}
@@ -0,0 +1,677 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make uw_dpg
+//
+
+//     - Δ p - ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//  ∇ p + i ω u = 0, in Ω
+//  ∇⋅u + i ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/(i ω) 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) + i ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+// -(u , ∇ q) + i ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂  = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | i ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | i ω (p,q) |-(u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void acoustics_solution(const Vector & X, complex<double> & p, 
+                        vector<complex<double>> &dp, complex<double> & d2p);
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);      
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);                                                
+
+double p_exact_r(const Vector &x);
+double p_exact_i(const Vector &x);
+void u_exact_r(const Vector &x, Vector & u);
+void u_exact_i(const Vector &x, Vector & u);
+double rhs_func_r(const Vector &x);
+double rhs_func_i(const Vector &x);
+void gradp_exact_r(const Vector &x, Vector &gradu);
+void gradp_exact_i(const Vector &x, Vector &gradu);
+double divu_exact_r(const Vector &x);
+double divu_exact_i(const Vector &x);
+double d2_exact_r(const Vector &x);
+double d2_exact_i(const Vector &x);
+double hatp_exact_r(const Vector & X);
+double hatp_exact_i(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+void hatu_exact_r(const Vector & X, Vector & hatu);
+void hatu_exact_i(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                     
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   mfem::out << "p_fes space true dofs = " << p_fes->GetTrueVSize() << endl;
+   mfem::out << "u_fes space true dofs = " << u_fes->GetTrueVSize() << endl;
+   mfem::out << "hatp_fes space true dofs = " << hatp_fes->GetTrueVSize() << endl;
+   mfem::out << "hatu_fes space true dofs = " << hatu_fes->GetTrueVSize() << endl;
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ComplexNormalEquations * a = new ComplexNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices();
+
+   // i ω (p,q)
+   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omeg),0,0);
+
+// -(u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(negone)),nullptr,1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),nullptr,0,1);
+
+//  i ω (u,v)
+   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(omeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,nullptr,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,3,0);
+
+   // for impedence condition (only on the boundary)
+   // TODO
+   // a->AddTrialIntegrator(new TraceIntegrator,nullptr,2,0);
+
+
+// test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),nullptr,1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -i ω (∇q,δv)
+      a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negomeg),0,1);
+      // i ω (v,∇ δq)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakDivergenceIntegrator(negomeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),nullptr,1,1);
+
+      // - i ω (∇⋅v,δq)   
+      a->AddTestIntegrator(nullptr,new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // i ω (q,∇⋅v)   
+      a->AddTestIntegrator(nullptr,new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),nullptr,0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs_r(rhs_func_r);
+   FunctionCoefficient f_rhs_i(rhs_func_i);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),new DomainLFIntegrator(f_rhs_i),0);
+   
+   
+   FunctionCoefficient hatpex_r(hatp_exact_r);
+   FunctionCoefficient hatpex_i(hatp_exact_i);
+   VectorFunctionCoefficient hatuex_r(dim,hatu_exact_r);
+   VectorFunctionCoefficient hatuex_i(dim,hatu_exact_i);
+   Array<int> elements_to_refine;
+
+   socketstream p_out_r;
+   socketstream p_out_i;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out_r.open(vishost, visport);
+      p_out_i.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // ess_bdr[1] = 0;
+         // ess_bdr[2] = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         // hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+                        //   + hatp_fes->GetTrueVSize(); 
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ComplexGridFunction hatp_gf(hatp_fes);
+      hatp_gf.real().MakeRef(hatp_fes,&xdata[offsets[2]]);
+      hatp_gf.imag().MakeRef(hatp_fes,&xdata[offsets.Last()+ offsets[2]]);
+      hatp_gf.ProjectBdrCoefficient(hatpex_r,hatpex_i, ess_bdr);
+
+      // ComplexGridFunction hatu_gf(hatu_fes);
+      // hatu_gf.real().MakeRef(hatu_fes,&xdata[offsets[3]]);
+      // hatu_gf.imag().MakeRef(hatu_fes,&xdata[offsets.Last()+ offsets[3]]);
+      // hatu_gf.ProjectBdrCoefficientNormal(hatuex_r,hatuex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      SparseMatrix * Ar = dynamic_cast<BlockMatrix *>(&Ahc->real())->CreateMonolithic();
+      SparseMatrix * Ai = dynamic_cast<BlockMatrix *>(&Ahc->imag())->CreateMonolithic();
+
+      ComplexSparseMatrix Ac(Ar,Ai,true,true);
+      SparseMatrix * A = Ac.GetSystemMatrix();
+
+      mfem::out << "Size of the linear system: " << A->Height() << std::endl;
+
+
+      UMFPackSolver umf(*A);
+      umf.Mult(B,X);
+
+      delete A;
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      double residual = residuals.Norml2();
+
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ComplexGridFunction p(p_fes);
+      p.real().MakeRef(p_fes,x.GetData());
+      p.imag().MakeRef(p_fes,&x.GetData()[offsets.Last()]);
+
+      ComplexGridFunction pgf_ex(p_fes);
+      FunctionCoefficient p_ex_r(p_exact_r);
+      FunctionCoefficient p_ex_i(p_exact_i);
+      pgf_ex.ProjectCoefficient(p_ex_r, p_ex_i);
+
+      int dofs = X.Size()/2;
+
+      double p_err_r = p.real().ComputeL2Error(p_ex_r);
+      double p_err_i = p.imag().ComputeL2Error(p_ex_i);
+
+      double L2Error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  0.0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::setw(10) << std::scientific 
+                << std::endl;
+
+      if (visualization)
+      {
+         p_out_r.precision(8);
+         p_out_r << "solution\n" << mesh << p.real() <<
+                  "window_title 'Real Numerical presure' "
+                  << flush;
+
+         p_out_i.precision(8);
+         p_out_i << "solution\n" << mesh << p.imag() <<
+                  "window_title 'Imag Numerical presure' "
+                  << flush;         
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double p_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return p;
+}
+
+double p_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return p;
+}
+
+double hatp_exact_r(const Vector & X)
+{
+   return p_exact_r(X);
+}
+
+double hatp_exact_i(const Vector & X)
+{
+   return p_exact_i(X);
+}
+
+void gradp_exact_r(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_r(x,p,grad,d2p);
+}
+
+void gradp_exact_i(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_i(x,p,grad,d2p);
+}
+
+double d2_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return d2p;
+}
+
+double d2_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return d2p;
+}
+
+//  u = - ∇ p / (i ω )
+//    = i (∇ p_r + i * ∇ p_i)  / ω 
+//    = - ∇ p_i / ω + i ∇ p_r / ω 
+void u_exact_r(const Vector &x, Vector & u)
+{
+   gradp_exact_i(x,u);
+   u *= -1./omega;
+}
+
+void u_exact_i(const Vector &x, Vector & u)
+{
+   gradp_exact_r(x,u);
+   u *= 1./omega;
+}
+
+void hatu_exact_r(const Vector & X, Vector & hatu)
+{
+   u_exact_r(X,hatu);
+}
+void hatu_exact_i(const Vector & X, Vector & hatu)
+{
+   u_exact_i(X,hatu);
+}
+
+//  ∇⋅u = i Δ p / ω
+//      = i (Δ p_r + i * Δ p_i)  / ω 
+//      = - Δ p_i / ω + i Δ p_r / ω 
+
+double divu_exact_r(const Vector &x)
+{
+   return -d2_exact_i(x)/omega;
+}
+
+double divu_exact_i(const Vector &x)
+{
+   return d2_exact_r(x)/omega;
+}
+
+// f = ∇⋅u + i ω p 
+// f_r = ∇⋅u_r - ω p_i  
+double rhs_func_r(const Vector &x)
+{
+   double p = p_exact_i(x);
+   double divu = divu_exact_r(x);
+   return divu - omega * p;
+}
+
+// f_i = ∇⋅u_i + ω p_r
+double rhs_func_i(const Vector &x)
+{
+   double p = p_exact_r(x);
+   double divu = divu_exact_i(x);
+   return divu + omega * p;
+}
+
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.real();
+   d2p = d2zp.real();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].real();
+   }
+}                           
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.imag();
+   d2p = d2zp.imag();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].imag();
+   }
+}
+
+
+void acoustics_solution(const Vector & X, complex<double> & p, vector<complex<double>> & dp, 
+         complex<double> & d2p)
+{
+   dp.resize(X.Size());
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      double beta = omega/std::sqrt((double)X.Size());
+      complex<double> alpha = beta * zi * X.Sum();
+      p = exp(-alpha);
+      d2p = - dim * beta * beta * p;
+      for (int i = 0; i<X.Size(); i++)
+      {
+         dp[i] = - zi * beta * p;
+      }
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> zi = complex<double>(0., 1.);
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp;
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
+
+      d2p = (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+          + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
+   }
+   break;
+   }
+
+}
@@ -0,0 +1,294 @@
+//                 MFEM FOSLS acoustics Example
+//
+// Compile with: make fosls
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// FOSLS:
+//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
+
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// -------------------------------------------------------------------
+// |   |            p           |            u             |   RHS    | 
+// -------------------------------------------------------------------
+// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
+// |   |                        |                          |          |
+// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
+
+// where (q,v) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int sr = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
+   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = H1fes;
+   fespaces[1] = RTfes;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
+   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega*omega);
+
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+
+   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
+   SumIntegrator * integ00 = new SumIntegrator();
+   integ00->AddIntegrator(new DiffusionIntegrator(one));
+   integ00->AddIntegrator(new MassIntegrator(omeg2));
+   blfi(0,0) = integ00;
+
+   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
+   SumIntegrator * integ01 = new SumIntegrator();
+#ifdef DEFINITE
+   // -ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+#else   
+   // ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+#endif
+   // - ω (u, ∇ q)
+   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   blfi(0,1) = integ01;
+
+   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
+   SumIntegrator * integ10 = new SumIntegrator();
+#ifdef DEFINITE
+   // - ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+#else
+   // ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+#endif
+   // - ω (∇ p,v)
+   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
+   blfi(1,0) = integ10;
+
+   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
+   SumIntegrator * integ11 = new SumIntegrator();
+   integ11->AddIntegrator(new DivDivIntegrator(one));
+   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
+   blfi(1,1) = integ11;
+
+
+   BlockLinearForm b(fespaces);
+   Array<LinearFormIntegrator * > lfi(2);
+   // ± ω (f,q)
+   FunctionCoefficient f_rhs(rhs_func);
+#ifdef DEFINITE  
+   ProductCoefficient w_f(omeg,f_rhs);
+#else   
+   ProductCoefficient w_f(negomeg,f_rhs);
+#endif   
+   // lfi[0] = new DomainLFIntegrator(w_f);
+   lfi[0] = new DomainLFIntegrator(w_f);
+
+
+   //  -(f,∇⋅v)
+   ProductCoefficient neg_f(negone,f_rhs);
+   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
+   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(H1fes);   
+
+   p_gf.MakeRef(H1fes,x,0);
+   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+   p_gf.ProjectCoefficient(p_ex);
+   pex_gf.ProjectCoefficient(p_ex);
+
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+   u_gf = 0.;
+
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-10);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   p_gf.MakeRef(H1fes,x,0);
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << p_gf <<
+                "window_title 'Numerical p' "
+                << flush;
+      // socketstream sols_sock(vishost, visport);
+      // sols_sock.precision(8);
+      // sols_sock << "solution\n" << mesh << u_gf <<
+               //  "window_title 'Numerical sigma' "
+               //  << flush;
+
+      socketstream solex_sock(vishost, visport);
+      solex_sock.precision(8);
+      solex_sock << "solution\n" << mesh << pex_gf <<
+                    "window_title 'Exact p' "
+                    << flush;          
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = fosls uw_dpg  strong_dpg complex_uw_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
@@ -0,0 +1,837 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make pcomplex_uw_dpg
+//
+// sample runs
+// ./pcomplex_uw_dpg -o 3 -m ../../../data/inline-quad.mesh -sref 2 -pref 3 -rnum 4.1 -prob 0 -sc -graph-norm
+
+//     - Δ p - ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//  ∇ p + i ω u = 0, in Ω
+//  ∇⋅u + i ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/(i ω) 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) + i ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+// -(u , ∇ q) + i ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂  = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | i ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | i ω (p,q) |-(u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void acoustics_solution(const Vector & X, complex<double> & p, 
+                        vector<complex<double>> &dp, complex<double> & d2p);
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);      
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p);                                                
+
+double p_exact_r(const Vector &x);
+double p_exact_i(const Vector &x);
+void u_exact_r(const Vector &x, Vector & u);
+void u_exact_i(const Vector &x, Vector & u);
+double rhs_func_r(const Vector &x);
+double rhs_func_i(const Vector &x);
+void gradp_exact_r(const Vector &x, Vector &gradu);
+void gradp_exact_i(const Vector &x, Vector &gradu);
+double divu_exact_r(const Vector &x);
+double divu_exact_i(const Vector &x);
+double d2_exact_r(const Vector &x);
+double d2_exact_i(const Vector &x);
+double hatp_exact_r(const Vector & X);
+double hatp_exact_i(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+void hatu_exact_r(const Vector & X, Vector & hatu);
+void hatu_exact_i(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+   int sr = 0;
+   int pr = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");     
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                    
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of parallel refinements.");                                              
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");  
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   dim = mesh.Dimension();
+   mesh.EnsureNCMesh();
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   // if (myid == 0)
+   // {
+   //    mfem::out << "p_fes space true dofs = " << p_fes->GetTrueVSize() << endl;
+   //    mfem::out << "u_fes space true dofs = " << u_fes->GetTrueVSize() << endl;
+   //    mfem::out << "hatp_fes space true dofs = " << hatp_fes->GetTrueVSize() << endl;
+   //    mfem::out << "hatu_fes space true dofs = " << hatu_fes->GetTrueVSize() << endl;
+   // }
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ComplexParNormalEquations * a = new ComplexParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices();
+   // i ω (p,q)
+   a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(omeg),0,0);
+
+// -(u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(negone)),nullptr,1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),nullptr,0,1);
+
+//  i ω (u,v)
+   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(omeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,nullptr,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,nullptr,3,0);
+
+// test integrators 
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),nullptr,1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -i ω (∇q,δv)
+      a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negomeg),0,1);
+      // i ω (v,∇ δq)
+      a->AddTestIntegrator(nullptr,new MixedVectorWeakDivergenceIntegrator(negomeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),nullptr,1,1);
+
+      // - i ω (∇⋅v,δq)   
+      a->AddTestIntegrator(nullptr,new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // i ω (q,∇⋅v)   
+      a->AddTestIntegrator(nullptr,new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),nullptr,0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs_r(rhs_func_r);
+   FunctionCoefficient f_rhs_i(rhs_func_i);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs_r),new DomainLFIntegrator(f_rhs_i),0);
+   
+
+   FunctionCoefficient hatpex_r(hatp_exact_r);
+   FunctionCoefficient hatpex_i(hatp_exact_i);
+
+   VectorFunctionCoefficient hatuex_r(dim,hatu_exact_r);
+   VectorFunctionCoefficient hatuex_i(dim,hatu_exact_i);
+   Array<int> elements_to_refine;
+
+   socketstream p_out_r;
+   socketstream p_out_i;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out_r.open(vishost, visport);
+      p_out_i.open(vishost, visport);
+   }
+
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << "\n  Ref |" 
+                << "       Mesh       |"
+                << "    Dofs    |" 
+                << "   ω   |" 
+                << "  L2 Error  |" 
+                << " Relative % |" 
+                << "  Rate  |" 
+                << "  Residual  |" 
+                << "  Rate  |" 
+                << " PCG it |"
+                << " PCG time |"  << endl;
+      mfem::out << " --------------------"      
+                <<  "---------------------"    
+                <<  "---------------------"    
+                <<  "---------------------"    
+                <<  "---------------------"    
+                <<  "-------------------" << endl;      
+   }
+
+
+   for (int it = 0; it<pr; it++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         // ess_bdr[1] = 0;
+         // ess_bdr[2] = 0;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         // hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+                           //   + hatp_fes->GetTrueVSize(); 
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ParComplexGridFunction hatp_gf(hatp_fes);
+      hatp_gf.real().MakeRef(hatp_fes,&xdata[offsets[2]]);
+      hatp_gf.imag().MakeRef(hatp_fes,&xdata[offsets.Last()+ offsets[2]]);
+      hatp_gf.ProjectBdrCoefficient(hatpex_r,hatpex_i, ess_bdr);
+      // ParComplexGridFunction hatu_gf(hatu_fes);
+      // hatu_gf.real().MakeRef(hatu_fes,&xdata[offsets[3]]);
+      // hatu_gf.imag().MakeRef(hatu_fes,&xdata[offsets.Last()+ offsets[3]]);
+      // hatu_gf.ProjectCoefficientNormal(hatuex_r,hatuex_i, ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+      BlockOperator * BlockA_r = dynamic_cast<BlockOperator *>(&Ahc->real());
+      BlockOperator * BlockA_i = dynamic_cast<BlockOperator *>(&Ahc->imag());
+
+      int num_blocks = BlockA_r->NumRowBlocks();
+      Array<int> tdof_offsets(2*num_blocks+1);
+
+      tdof_offsets[0] = 0;
+      int skip = (static_cond) ? 0 : 2;
+      int k = (static_cond) ? 2 : 0;
+      for (int i=0; i<num_blocks;i++)
+      {
+         tdof_offsets[i+1] = trial_fes[i+k]->GetTrueVSize(); 
+         tdof_offsets[num_blocks+i+1] = trial_fes[i+k]->GetTrueVSize(); 
+      }
+      tdof_offsets.PartialSum();
+
+      BlockOperator blockA(tdof_offsets);
+      for (int i = 0; i<num_blocks; i++)
+      {
+         for (int j = 0; j<num_blocks; j++)
+         {
+            blockA.SetBlock(i,j,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i,j+num_blocks,&BlockA_i->GetBlock(i,j), -1.0);
+            blockA.SetBlock(i+num_blocks,j+num_blocks,&BlockA_r->GetBlock(i,j));
+            blockA.SetBlock(i+num_blocks,j,&BlockA_i->GetBlock(i,j));
+         }
+      }
+
+
+      X = 0.;
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(tdof_offsets);
+
+
+      if (!static_cond)
+      {
+         HypreBoomerAMG * solver_p = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(0,0));
+         solver_p->SetPrintLevel(0);
+         solver_p->SetSystemsOptions(dim);
+         HypreBoomerAMG * solver_u = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(1,1));
+         solver_u->SetPrintLevel(0);
+         solver_u->SetSystemsOptions(dim);
+         M->SetDiagonalBlock(0,solver_p);
+         M->SetDiagonalBlock(1,solver_u);
+         M->SetDiagonalBlock(num_blocks,solver_p);
+         M->SetDiagonalBlock(num_blocks+1,solver_u);
+      }
+
+
+      HypreBoomerAMG * solver_hatp = new HypreBoomerAMG((HypreParMatrix &)BlockA_r->GetBlock(skip,skip));
+      // amg->SetCycleNumSweeps(5, 5);
+      solver_hatp->SetPrintLevel(0);
+      
+      HypreSolver * solver_hatu = nullptr;
+      if (dim == 2)
+      {
+         solver_hatu = new HypreAMS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1),hatu_fes);
+         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
+      }
+      else
+      {
+         solver_hatu = new HypreADS((HypreParMatrix &)BlockA_r->GetBlock(skip+1,skip+1), hatu_fes);
+         dynamic_cast<HypreAMS*>(solver_hatu)->SetPrintLevel(0);
+      }
+      
+
+      M->SetDiagonalBlock(skip,solver_hatp);
+      M->SetDiagonalBlock(skip+1,solver_hatu);
+      M->SetDiagonalBlock(skip+num_blocks,solver_hatp);
+      M->SetDiagonalBlock(skip+num_blocks+1,solver_hatu);
+
+      StopWatch chrono;
+      
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-7);
+      cg.SetAbsTol(1e-7);
+      cg.SetMaxIter(10000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M); 
+      cg.SetOperator(blockA);
+      chrono.Clear();
+      chrono.Start();
+      cg.Mult(B, X);
+      chrono.Stop();
+      delete M;
+
+      int ne = pmesh.GetNE();
+      MPI_Allreduce(MPI_IN_PLACE,&ne,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+      int ne_x = (dim == 2) ? (int)sqrt(ne) : (int)cbrt(ne);
+      ostringstream oss;
+      double pcg_time = chrono.RealTime();
+      if (myid == 0)
+      {
+         if (dim == 2)
+         {
+            oss << ne_x << " x " << ne_x ;
+         }
+         else
+         {
+            oss << ne_x << " x " << ne_x << " x " << ne_x ;
+         }
+      }
+
+      int num_iter = cg.GetNumIterations();
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParComplexGridFunction p(p_fes);
+      p.real().MakeRef(p_fes,x.GetData());
+      p.imag().MakeRef(p_fes,&x.GetData()[offsets.Last()]);
+
+      ParComplexGridFunction u(u_fes);
+      u.real().MakeRef(u_fes,&x.GetData()[offsets[1]]);
+      u.imag().MakeRef(u_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+
+      // Error in pressure 
+      ParComplexGridFunction pgf_ex(p_fes);
+      FunctionCoefficient p_ex_r(p_exact_r);
+      FunctionCoefficient p_ex_i(p_exact_i);
+      pgf_ex.ProjectCoefficient(p_ex_r, p_ex_i);
+
+      double p_err_r = p.real().ComputeL2Error(p_ex_r);
+      double p_err_i = p.imag().ComputeL2Error(p_ex_i);
+      double p_error = sqrt(p_err_r*p_err_r + p_err_i*p_err_i);
+      double p_norm_r = pgf_ex.real().ComputeL2Error(zero);
+      double p_norm_i = pgf_ex.imag().ComputeL2Error(zero);
+      double p_norm = sqrt(p_norm_r*p_norm_r + p_norm_i*p_norm_i);
+
+      // Error in velocity
+      ParComplexGridFunction ugf_ex(u_fes);
+      VectorFunctionCoefficient u_ex_r(dim,u_exact_r);
+      VectorFunctionCoefficient u_ex_i(dim,u_exact_i);
+
+      double u_err_r = u.real().ComputeL2Error(u_ex_r);
+      double u_err_i = u.imag().ComputeL2Error(u_ex_i);
+      double u_error = sqrt(u_err_r*u_err_r + u_err_i*u_err_i);
+      double u_norm_r = pgf_ex.real().ComputeL2Error(vzero);
+      double u_norm_i = pgf_ex.imag().ComputeL2Error(vzero);
+      double u_norm = sqrt(u_norm_r*u_norm_r + u_norm_i*u_norm_i);
+
+
+      double L2Error = sqrt(p_error*p_error + u_error*u_error);
+      double L2norm = sqrt(p_norm*p_norm + u_norm*u_norm);
+
+      double rel_err = L2Error/L2norm;
+
+      int dofs = p_fes->GlobalTrueVSize()
+               + u_fes->GlobalTrueVSize()
+               + hatp_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize();
+
+
+      double rate_err = (it) ? dim*log(err0/rel_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (it) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = rel_err;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(5) << it << " | " 
+                  << std::setw(16) << oss.str() << " | " 
+                  << std::setw(10) <<  dof0 << " | " 
+                  << std::setprecision(0) << std::fixed
+                  << std::setw(2) <<  2*rnum << " π  | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  err0 << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::fixed <<  rel_err * 100. << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_err << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  res0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_res << " | " 
+                  << std::setw(6) << std::fixed << num_iter << " | " 
+                  << std::setprecision(5) 
+                  << std::setw(8) << std::fixed << pcg_time << " | " 
+                  << std::scientific 
+                  << std::endl;
+      }   
+
+      if (visualization)
+      {
+         p_out_r << "parallel " << num_procs << " " << myid << "\n";
+         p_out_r.precision(8);
+         p_out_r << "solution\n" << pmesh << p.real() <<
+                  "window_title 'Real Numerical presure' "
+                  << flush;
+
+         p_out_i << "parallel " << num_procs << " " << myid << "\n";
+         p_out_i.precision(8);
+         p_out_i << "solution\n" << pmesh << p.imag() <<
+                  "window_title 'Imag Numerical presure' "
+                  << flush;         
+      }
+
+      if (it == pr)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();   
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double p_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return p;
+}
+
+double p_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return p;
+}
+
+double hatp_exact_r(const Vector & X)
+{
+   return p_exact_r(X);
+}
+
+double hatp_exact_i(const Vector & X)
+{
+   return p_exact_i(X);
+}
+
+void gradp_exact_r(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_r(x,p,grad,d2p);
+}
+
+void gradp_exact_i(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   double p,d2p;
+   acoustics_solution_i(x,p,grad,d2p);
+}
+
+double d2_exact_r(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_r(x,p,dp,d2p);
+   return d2p;
+}
+
+double d2_exact_i(const Vector &x)
+{
+   double p,d2p;
+   Vector dp;
+   acoustics_solution_i(x,p,dp,d2p);
+   return d2p;
+}
+
+//  u = - ∇ p / (i ω )
+//    = i (∇ p_r + i * ∇ p_i)  / ω 
+//    = - ∇ p_i / ω + i ∇ p_r / ω 
+void u_exact_r(const Vector &x, Vector & u)
+{
+   gradp_exact_i(x,u);
+   u *= -1./omega;
+}
+
+void u_exact_i(const Vector &x, Vector & u)
+{
+   gradp_exact_r(x,u);
+   u *= 1./omega;
+}
+
+void hatu_exact_r(const Vector & X, Vector & hatu)
+{
+   u_exact_r(X,hatu);
+}
+void hatu_exact_i(const Vector & X, Vector & hatu)
+{
+   u_exact_i(X,hatu);
+}
+
+//  ∇⋅u = i Δ p / ω
+//      = i (Δ p_r + i * Δ p_i)  / ω 
+//      = - Δ p_i / ω + i Δ p_r / ω 
+
+double divu_exact_r(const Vector &x)
+{
+   return -d2_exact_i(x)/omega;
+}
+
+double divu_exact_i(const Vector &x)
+{
+   return d2_exact_r(x)/omega;
+}
+
+// f = ∇⋅u + i ω p 
+// f_r = ∇⋅u_r - ω p_i  
+double rhs_func_r(const Vector &x)
+{
+   double p = p_exact_i(x);
+   double divu = divu_exact_r(x);
+   return divu - omega * p;
+}
+
+// f_i = ∇⋅u_i + ω p_r
+double rhs_func_i(const Vector &x)
+{
+   double p = p_exact_r(x);
+   double divu = divu_exact_i(x);
+   return divu + omega * p;
+}
+
+
+void acoustics_solution_r(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.real();
+   d2p = d2zp.real();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].real();
+   }
+}                           
+
+void acoustics_solution_i(const Vector & X, double & p, 
+                          Vector &dp, double & d2p)
+{
+   complex<double> zp, d2zp;
+   vector<complex<double>> dzp;
+   acoustics_solution(X,zp,dzp,d2zp);
+   p = zp.imag();
+   d2p = d2zp.imag();
+   dp.SetSize(X.Size());
+   for (int i = 0; i<X.Size(); i++)
+   {
+      dp[i] = dzp[i].imag();
+   }
+}
+
+
+void acoustics_solution(const Vector & X, complex<double> & p, vector<complex<double>> & dp, 
+         complex<double> & d2p)
+{
+   dp.resize(X.Size());
+   complex<double> zi = complex<double>(0., 1.);
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      double beta = omega/std::sqrt((double)X.Size());
+      complex<double> alpha = beta * zi * X.Sum();
+      p = exp(-alpha);
+      d2p = - dim * beta * beta * p;
+      for (int i = 0; i<X.Size(); i++)
+      {
+         dp[i] = - zi * beta * p;
+      }
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp;
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim);
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim);
+
+      d2p = (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+          + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim;
+   }
+   break;
+   }
+
+}
@@ -0,0 +1,271 @@
+//                 MFEM DPG_strong acoustics Example
+//
+// Compile with: make strong_dpg
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// Strong DPG formulation
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// 
+//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
+//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
+//                    p = p_0, in ∂Ω
+// 
+// ------------------------------------
+// |   |      p    |     u    |  RHS  | 
+// ------------------------------------
+// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
+// |   |           |          |       |
+// | v | (∇ p, v)  | -ω (u,v) |       |
+
+// where (q,v) ∈ L^2  × (L^2)^dim
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");                  
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   // Define spaces
+   // H1 space for p
+   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
+   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
+
+   // H(div) for u
+   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->SetTestFECollVdim(1,dim);
+   
+   a->StoreMatrices(true);
+
+//  ± ω (p, q)
+#ifdef DEFINITE
+//  ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
+#else
+// -ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
+#endif
+
+// -(∇⋅u, q) 
+   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
+
+// -ω (u,v)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
+
+// (∇ p, v)
+   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
+
+// (v,δv)
+   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
+    
+// (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(p_fes);   
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = p_fes->GetVSize();
+   offsets[2] = u_fes->GetVSize();
+   offsets.PartialSum();
+   BlockVector x(offsets);
+   x = 0.0;
+
+   p_gf.MakeRef(p_fes,x.GetBlock(0));
+   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+
+   u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = Ah.As<BlockMatrix>();
+
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream p_out;
+      socketstream u_out;
+      p_out.open(vishost, visport);
+      u_out.open(vishost, visport);
+      p_out.precision(8);
+      p_out << "solution\n" << mesh << p_gf <<
+               "window_title 'Numerical p' "
+               << flush;
+
+      u_out.precision(8);
+      u_out << "solution\n" << mesh << u_gf <<
+            "window_title 'Numerical flux' "
+            << flush;
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
@@ -0,0 +1,546 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make uw_dpg
+//
+// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+double divu_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+   // (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+   // - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+   // < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   // test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   GridFunction hatp_gf;
+
+
+   socketstream p_out;
+   // socketstream u_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+      // u_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-8);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+      GridFunction pex_gf(p_fes);
+      GridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+
+      // Error
+      int dofs = X.Size();
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = pex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+      if (visualization)
+      {
+         p_out.precision(8);
+         p_out << "solution\n" << mesh << p_gf <<
+                  "window_title 'Numerical presure' "
+                  << flush;
+
+         // u_out.precision(8);
+         // u_out << "solution\n" << mesh << u_gf <<
+         //       "window_title 'Numerical velocity' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return p;
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   double p, d2p;
+   acoustics_solution(x,p,u,d2p);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return d2p/omega;
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}
+
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
+{
+   dp.SetSize(X.Size());
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      p = sin(omega*X.Sum());
+      dp = omega * cos(omega * X.Sum());
+      d2p = -dim * omega * omega * sin(omega*X.Sum());
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> zi = complex<double>(0., 1.);
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp.real();
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
+
+      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
+   }
+   break;
+   }
+
+}
@@ -0,0 +1,525 @@
+//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
+//
+// Compile with: make uw_dpgp
+//
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+//
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+// 
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int sr = 0;
+   int pr = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");      
+   args.AddOption(&sr, "-sref", "--serial_ref",
+                  "Number of parallel refinements.");                                              
+   args.AddOption(&pr, "-pref", "--parallel_ref",
+                  "Number of parallel refinements.");  
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                    
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   for (int i = 0; i<sr; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   dim = mesh.Dimension();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   Array<ParFiniteElementSpace * > trial_fes; 
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   Array<FiniteElementCollection * > test_fec; 
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // Integrators
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+// (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+// - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   // test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }   
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatp_gf;
+
+
+
+
+   socketstream p_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+   }
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << "\n Refinement |" 
+               << "    Dofs    |" 
+               << "   ω   |" 
+               << "  L2 Error  |" 
+               << " Relative % |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" 
+               << " PCG it |" << endl;
+      mfem::out << " --------------------"      
+               <<  "---------------------"    
+               <<  "---------------------"    
+               <<  "---------------------"    
+               <<  "----------------" << endl;   
+   }
+
+
+   for (int i = 0; i<pr; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         // amg0->SetRelaxType(16);
+         // amg1->SetRelaxType(16);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip = 2;
+      }
+
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      // amg2->SetRelaxType(16);
+      M->SetDiagonalBlock(skip,amg2);
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatu_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatu_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-7);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      int num_iter = cg.GetNumIterations();
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ParGridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+
+      ParGridFunction pex_gf(p_fes);
+      ParGridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+      int dofs = p_fes->GlobalTrueVSize()
+               + u_fes->GlobalTrueVSize()
+               + hatp_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize();
+
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = pex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+      std::ios oldState(nullptr);
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                   << std::setw(10) <<  dof0 << " | " 
+                   << std::setprecision(0) << std::fixed
+                   << std::setw(2) <<  2*rnum << " π  | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  err0 << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_err << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  res0 << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_res << " | " 
+                   << std::setw(6) << std::fixed << num_iter << " | " 
+                   << std::setprecision(5) 
+                   << std::scientific 
+                   << std::endl;
+      }    
+
+      if (visualization)
+      {
+         p_out << "parallel " << num_procs << " " << myid << "\n";
+         p_out.precision(8);
+         p_out << "solution\n" << pmesh << p_gf <<
+                  "window_title 'Numerical pressure' "
+                  << flush;
+      }
+
+      if (i == pr)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/convection-diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = uw_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
@@ -0,0 +1,652 @@
+//                     MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");        
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                             
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: EJ ,2: General");           
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");       
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");                                              
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatf_fes = new FiniteElementSpace(&mesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   FiniteElementSpace *coeff_fes = new FiniteElementSpace(&mesh,coeff_fec);
+   GridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         // (∇v,∇δv)
+         mfem::out << "\n Test norm: Standard" << endl;
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         mfem::out << "\n Test norm: Robust" << endl;
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+   GridFunction hatu_gf;
+   GridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<=ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(mesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int j = 0; j < n; j++)
+      {
+         ess_tdof_list[j] = ess_tdof_list_uhat[j] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int j = 0; j < m; j++)
+      {
+         ess_tdof_list[j+n] = ess_tdof_list_fhat[j] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      // BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      // M->owns_blocks = 1;
+      // for (int i=0; i<A->NumRowBlocks(); i++)
+      // {
+      //    M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      // }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(0);
+      // cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      // delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }   
+
+      GridFunction uex_gf(u_fes);
+      uex_gf.ProjectCoefficient(uex);
+
+      GridFunction sigmaex_gf(sigma_fes);
+      sigmaex_gf.ProjectCoefficient(sigmaex);
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = X.Size();
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(zero);
+      // mfem::out << "u_err = " << u_err << endl;
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double sigma_norm = sigmaex_gf.ComputeL2Error(vzero);
+      // mfem::out << "sigma_err = " << sigma_err << endl;
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+      double L2norm = sqrt(u_norm * u_norm + sigma_norm * sigma_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "solution\n" << mesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+         u_out.precision(8);
+         u_out << "solution\n" << mesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> dofs;
+         for (int i = 0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+      }
+
+   }
+
+   delete coeff_fes;
+   delete coeff_fec;
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fes;
+   delete sigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}
@@ -0,0 +1,704 @@
+//                                MFEM Ultraweak DPG example
+//
+// Compile with: make  uw_dpgp
+//
+// sample runs 
+// mpirun -np 6 ./uw_dpgp  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, f̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+
+int main(int argc, char *argv[])
+{
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   bool static_cond = false;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                           
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");                         
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");     
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");          
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatf_fes = new ParFiniteElementSpace(&pmesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   ParFiniteElementSpace *coeff_fes = new ParFiniteElementSpace(&pmesh,coeff_fec);
+   ParGridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Standard" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Robust" << endl;
+         }
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+
+   ParGridFunction hatu_gf;
+   ParGridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << " Refinement |" 
+               << "    Dofs    |" 
+               << "  L2 Error  |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" 
+               << "  CG iter  |" << endl;
+      mfem::out << " --------------------"      
+               <<  "-------------------"    
+               <<  "-------------------"    
+               <<  "-------------------" << endl;   
+   }
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(pmesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int j = 0; j < n; j++)
+      {
+         ess_tdof_list[j] = ess_tdof_list_uhat[j] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int j = 0; j < m; j++)
+      {
+         ess_tdof_list[j+n] = ess_tdof_list_fhat[j] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip = 2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(-1);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      int num_iter = cg.GetNumIterations();
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max(); 
+
+      double gresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&gresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      gresidual = sqrt(gresidual);
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = u_fes->GlobalTrueVSize()
+               + sigma_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize()
+               + hatf_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeL2Error(uex);
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/gresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = gresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                  << std::setw(10) <<  dof0 << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  err0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_err << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  res0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_res << " | " 
+                  << std::setw(6) << std::fixed << num_iter << " | " 
+                  << std::resetiosflags(std::ios::showbase)
+                  << std::endl;
+      }
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "parallel " << num_procs << " " << myid << "\n";
+         // uex_out << "solution\n" << pmesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << pmesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref-1)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> edofs;
+         for (int i = 0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,edofs);
+            c1_gf.SetSubVector(edofs,c1);
+            c2_gf.SetSubVector(edofs,c2);
+         }
+      }
+   }
+
+   delete coeff_fes;
+   delete coeff_fec;
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}
@@ -0,0 +1,203 @@
+//                 MFEM Fosls 1
+//
+// Compile with: make blkfosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec0 = new H1_FECollection(order, dim);
+   FiniteElementCollection *fec1 = new RT_FECollection(order-1, dim);
+   FiniteElementSpace fespace0(&mesh, fec0);
+   FiniteElementSpace fespace1(&mesh, fec1);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = &fespace0;
+   fespaces[1] = &fespace1;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+
+   cout << "H1 fespace = " << fespace0.GetVSize() << endl;
+   cout << "RT fespace = " << fespace1.GetVSize() << endl;
+
+   FiniteElementCollection *fec2 = new RT_Trace_FECollection(order-1, dim);
+   FiniteElementSpace RT_trace_fes(&mesh, fec2);
+   cout << "RT trace = " << RT_trace_fes.GetVSize() << endl;
+
+   // for (int i = 0; i<mesh.GetNE(); i++)
+   // {
+   //    // const FiniteElement * fe = fespace1.GetFE(i);
+   //    // fespace1.GetTraceElement()
+   //    Array<int> faces, ori;
+   //    mesh.GetElementEdges(i, faces, ori);
+   //    for (int f = 0; f<faces.Size(); f++)
+   //    {
+   //       const FiniteElement * fe_trace = RT_trace_fes.GetFaceElement(faces[f]);
+   //       cout << fe_trace->GetDof() << endl;
+   //       Array<int> face_dofs;
+   //       RT_trace_fes.GetFaceDofs(faces[f],face_dofs);
+   //       cout << "face dofs = " << endl;
+   //       face_dofs.Print();
+   //    }
+
+   //    // cout << fe->GetGeomType() << endl;
+   //    Array<int> vdofs;
+   //    RT_trace_fes.GetElementVDofs(i, vdofs);
+   //    cout << "trace dofs = " << endl;
+   //    vdofs.Print();
+   //    fespace1.GetElementVDofs(i, vdofs);
+   //    cout << "elem dofs = " << endl;
+   //    vdofs.Print();
+   //    cin.get();
+   // }
+
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+   blfi(0,0) = new DiffusionIntegrator(one);
+   blfi(0,1) = new MixedVectorWeakDivergenceIntegrator(one);
+   blfi(1,0) = new MixedVectorGradientIntegrator(negone);
+
+   BilinearFormIntegrator * divdiv = new DivDivIntegrator(one);
+   BilinearFormIntegrator * mass = new VectorFEMassIntegrator(one);
+   SumIntegrator * suminteg = new SumIntegrator();
+   suminteg->AddIntegrator(divdiv);
+   suminteg->AddIntegrator(mass);
+   blfi(1,1) = suminteg;
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+
+   BlockLinearForm b(fespaces);
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   Array<LinearFormIntegrator * > lfi(2);
+   lfi[0] = nullptr;
+   lfi[1] = new VectorFEDomainLFDivIntegrator(negone);
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+
+   // need to implement blkgridfunction later but for now Vector would do
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(200);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   GridFunction u_gf, sigma_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(fespaces[0],&data[0]);
+   sigma_gf.MakeRef(fespaces[1],&data[fespaces[0]->GetVSize()]);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   delete fec0;
+
+   return 0;
+}
@@ -0,0 +1,223 @@
+//                   MFEM Fosls example
+//
+// Compile with: make fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order,dim);
+   FiniteElementSpace *H1fes = new FiniteElementSpace(&mesh, H1fec);
+
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1,dim);
+   FiniteElementSpace *RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Linear forms
+   LinearForm b_0(H1fes);
+   // (f,∇⋅τ )
+   LinearForm b_1(RTfes);
+   b_1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(negone));
+
+   // Bilinear forms
+   // (∇u,∇v)
+   BilinearForm a_00(H1fes);
+   a_00.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   //   -(σ,∇v)
+   MixedBilinearForm a_01(RTfes, H1fes);
+   a_01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(
+                               one)); // (-1 is included)
+
+   //    // -(∇u,τ)
+   // MixedBilinearForm()
+   MixedBilinearForm a_10(H1fes, RTfes);
+   a_10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negone));
+
+   // (∇⋅σ, ∇⋅τ) + (σ,τ)
+
+   BilinearForm a_11(RTfes);
+   a_11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a_11.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> block_Toffsets(3);
+   block_Toffsets[0] = 0;
+   block_Toffsets[1] = H1fes->GetTrueVSize();
+   block_Toffsets[2] = RTfes->GetTrueVSize();
+   block_Toffsets.PartialSum();
+
+   Vector rhs_H1(H1fes->GetVSize()); rhs_H1 = 0.;
+   Vector rhs_RT(RTfes->GetVSize()); rhs_RT = 0.;
+
+   Vector x_H1(H1fes->GetVSize()); x_H1 = 0.;
+   Vector x_RT(RTfes->GetVSize()); x_RT = 0.;
+
+
+   Vector RHS_H1(H1fes->GetTrueVSize()); RHS_H1 = 0.0;
+   Vector RHS_RT(RTfes->GetTrueVSize()); RHS_RT = 0.0;
+
+   Vector X_H1(H1fes->GetTrueVSize()); X_H1 = 0.0;
+   Vector X_RT(RTfes->GetTrueVSize()); X_RT = 0.0;
+
+
+   b_0.Update(H1fes,rhs_H1,0);
+   b_0.Assemble();
+
+   b_1.Update(RTfes,rhs_RT,0);
+   b_1.Assemble();
+
+
+   // Assembly and BC
+   a_00.Assemble();
+   SparseMatrix A_00;
+   a_00.FormLinearSystem(ess_tdof_list,x_H1,rhs_H1,
+                         A_00,X_H1,RHS_H1);
+
+   a_01.Assemble();
+   SparseMatrix A_01;
+   Array<int> empty;
+   a_01.FormRectangularSystemMatrix(empty, ess_tdof_list,A_01);
+
+
+   a_10.Assemble();
+   SparseMatrix A_10;
+
+   a_10.FormRectangularLinearSystem(ess_tdof_list,empty,x_H1,rhs_RT,
+                                    A_10,X_H1,RHS_RT);
+
+   a_11.Assemble();
+   SparseMatrix A_11;
+   a_11.FormSystemMatrix(empty,A_11);
+
+
+   BlockMatrix BlockA(block_Toffsets);
+   BlockA.SetBlock(0,0,&A_00);
+   BlockA.SetBlock(0,1,&A_01);
+   BlockA.SetBlock(1,0,&A_10);
+   BlockA.SetBlock(1,1,&A_11);
+
+
+   BlockVector RHS(block_Toffsets);
+   RHS.GetBlock(0) = RHS_H1;
+   RHS.GetBlock(1) = RHS_RT;
+
+   BlockVector X(block_Toffsets);
+   X.GetBlock(0) = X_H1;
+   X.GetBlock(1) = X_RT;
+
+
+   SparseMatrix * A = BlockA.CreateMonolithic();
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(RHS, X);
+
+   GridFunction u_gf(H1fes), sigma_gf(RTfes);
+   u_gf = 0.;
+   sigma_gf = 0.;
+
+   const SparseMatrix * P = H1fes->GetConformingProlongation();
+   if (P)
+   {
+      a_00.RecoverFEMSolution(X.GetBlock(0),rhs_H1,u_gf);
+      a_11.RecoverFEMSolution(X.GetBlock(1),rhs_RT,sigma_gf);
+   }
+   else
+   {
+      u_gf.MakeRef(X.GetBlock(0),0);
+      sigma_gf.MakeRef(X.GetBlock(1),0);
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   return 0;
+}
@@ -0,0 +1,61 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = blkfosls fosls primal_dpg \
+	 				uw_dpg 
+PAR_EXAMPLES = uw_dpgp 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf ParaView
+
+clean-exec:
@@ -0,0 +1,179 @@
+//                     MFEM primal_dpg example
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");   
+   args.ParseCheck();
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   // 3. Define a finite element space on the mesh. Here we use H1 continuous
+   //    high-order Lagrange finite elements of the given order.
+   H1_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace H1fes(&mesh, &fec);
+
+   RT_Trace_FECollection trace_fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTtrace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   H1_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+
+   trial_fes.Append(&H1fes);
+   trial_fes.Append(&RTtrace_fes);
+   test_fecs.Append(&test_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+
+
+   ConstantCoefficient one(1.0);
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
+   a->AddTrialIntegrator(new TraceIntegrator,1,0);
+
+   BilinearFormIntegrator * diffusion = new DiffusionIntegrator(one);
+   BilinearFormIntegrator * mass = new MassIntegrator(one);
+   a->AddTestIntegrator(diffusion,0,0);
+   a->AddTestIntegrator(mass,0,0);
+
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(one),0);
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+   OperatorPtr Ah;
+
+   int size = H1fes.GetVSize() + RTtrace_fes.GetVSize();
+
+   Vector x(size);
+   x = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+
+   a->RecoverFEMSolution(X,x);
+
+   GridFunction u_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(&H1fes,data);
+
+   GridFunction s_gf;
+   s_gf.MakeRef(&RTtrace_fes,&data[H1fes.GetVSize()]);
+
+
+
+   RT_FECollection RTfec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   GridFunction sigma_gf(&RTfes);
+   sigma_gf = 0.0;
+   for (int i = 0; i<mesh.GetNE(); i++)
+   {
+      Array<int> strace_dofs;
+      Array<int> trace_dofs;
+      Vector dofs;
+      RTtrace_fes.GetElementDofs(i,trace_dofs);
+      strace_dofs.SetSize(trace_dofs.Size());
+      // shift dofs;
+      for (int j = 0; j< trace_dofs.Size(); j++)
+      {
+         int offset = trace_dofs[j] < 0 ? -H1fes.GetVSize() : H1fes.GetVSize();
+         strace_dofs[j] = offset + trace_dofs[j];
+      }
+      x.GetSubVector(strace_dofs, dofs);
+      sigma_gf.SetSubVector(trace_dofs,dofs);
+
+   }
+
+
+
+
+
+   ParaViewDataCollection paraview_dc("DPG_example", &mesh);
+   paraview_dc.SetPrefixPath("ParaView");
+   paraview_dc.SetLevelsOfDetail(order);
+   paraview_dc.SetCycle(0);
+   paraview_dc.SetDataFormat(VTKFormat::BINARY);
+   paraview_dc.SetHighOrderOutput(true);
+   paraview_dc.SetTime(0.0); // set the time
+   paraview_dc.RegisterField("field",&u_gf);
+   paraview_dc.RegisterField("flux",&sigma_gf);
+   // paraview_dc.RegisterField("flux",&s_gf);
+   paraview_dc.Save();
+
+
+
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << u_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << sigma_gf <<
+                 "window_title 'Flux sigma_n' "
+                 << flush;
+
+
+
+}
@@ -0,0 +1,403 @@
+//                       MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg -m ../lshape2.mesh -o 2 -ref 20 -graph-norm -do 1 -prob 0
+
+//     - Δ u = f,   in Ω
+//         u = u_0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) - (σ , τ) + < û, τ⋅n> = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) + < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//                                û = 0       on ∂Ω 
+
+// Note: 
+// û := u
+// σ̂ := -σ
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           |  <σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u);
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ∇ u
+   sigma = du;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return exact_u(X);
+}
+
+void exact_hatsigma(const Vector & X, Vector & hatsigma)
+{
+   exact_sigma(X,hatsigma);
+   hatsigma *= -1.;
+}
+
+double f_exact(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return -d2u;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");               
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space"); 
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatsigma_fes = new FiniteElementSpace(&mesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negone)),1,0);
+
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   //  <σ̂,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f(f_exact);
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),1);
+   }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   Array<int> elements_to_refine;
+   GridFunction hatu_gf;
+
+
+   socketstream u_out;
+   // socketstream sigma_out;
+   socketstream mesh_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+      mesh_out.open(vishost, visport);
+   }
+
+
+   for (int iref = 0; iref<ref; iref++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new GSSmoother(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      cout << "Residual = " << residual << endl;
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      double theta = 0.7;
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out.precision(8);
+         string keys = (iref == 0) ? "keys em\n" : "keys";
+         u_out << "solution\n" << mesh << u_gf 
+               << "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+
+         mesh_out.precision(8);
+         mesh_out << "mesh\n" << mesh 
+                  << keys
+                  << "window_title 'Mesh' "
+                  << flush;
+
+      }
+
+      mesh.GeneralRefinement(elements_to_refine);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case lshape:
+      {
+         double r = sqrt(x*x + y*y);
+         double alpha = 2./3.;
+         double theta = atan2(y,x);
+         if (theta < 0) theta += 2*M_PI;
+         u = pow(r,alpha) * sin(alpha * theta);
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
@@ -0,0 +1,404 @@
+//                 MFEM UW DPG parallel example
+//
+// Compile with: make poisson_fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) + < û, τ⋅n> - (σ , τ) = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) - < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//            û = 0        on ∂Ω 
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           | -<σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+double exact(const Vector & X)
+{
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   return pow(r,alpha) * sin(alpha * theta);
+}
+
+void gradexact(const Vector & X, Vector & grad)
+{
+   grad.SetSize(2);
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   double r_x = x/r;
+   double r_y = y/r;
+   double theta_x = - y / (r*r);
+   double theta_y = x / (r*r);
+   double beta = alpha * pow(r,alpha - 1.);
+   grad[0] = beta*(r_x * sin(alpha*theta) + r * theta_x * cos(alpha*theta));
+   grad[1] = beta*(r_y * sin(alpha*theta) + r * theta_y * cos(alpha*theta));
+}
+
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+   double theta = 0.7;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");    
+   args.AddOption(&theta, "-theta", "--theta_factor",
+                  "Refinement factor");                              
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");   
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   TransposeIntegrator * mass = new TransposeIntegrator(new VectorFEMassIntegrator(negone));
+   a->AddTrialIntegrator(mass,1,0);
+
+   // (σ,∇ v)
+   TransposeIntegrator * grad = new TransposeIntegrator(new GradientIntegrator(one));
+   a->AddTrialIntegrator(grad,1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+   // RHS
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(one),1);
+   }
+
+   FunctionCoefficient uex(exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatu_gf;
+
+
+   socketstream u_out;
+   socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      sigma_out.open(vishost, visport);
+   }
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      if (prob == prob_type::lshape)
+      {
+         hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+         hatu_gf.ProjectBdrCoefficient(uex,ess_bdr);
+      }
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip=2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      if (myid == 0)
+      {
+         cout << "Global Residual = " << globalresidual << endl;
+      }
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         sigma_out.precision(8);
+         sigma_out << "solution\n" << pmesh << sigma_gf <<
+                  "window_title 'Numerical flux' "
+                  << flush;
+      }
+
+
+      if (i == ref-1)
+      {
+         break;
+      }
+
+      pmesh.GeneralRefinement(elements_to_refine);
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/grad-div,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = primal_dpg 
+PAR_EXAMPLES = 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
@@ -0,0 +1,176 @@
+//                                MFEM primal dpg example for grad-dic problem
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.ParseCheck();
+
+   kappa = freq * M_PI;
+
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   RT_FECollection fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &fec);
+
+   H1_Trace_FECollection trace_fec(order, mesh.Dimension());
+   FiniteElementSpace H1trace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   RT_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace *> trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+   
+   trial_fes.Append(&RTfes);
+   trial_fes.Append(&H1trace_fes);
+   test_fecs.Append(&test_fec);
+
+
+   GridFunction rt_gf(&RTfes);
+   VectorFunctionCoefficient F(dim, F_exact);
+   rt_gf.ProjectCoefficient(F);
+
+   Vector x(RTfes.GetVSize()+H1trace_fes.GetVSize());
+   x = 0.;
+   x.SetVector(rt_gf,0);
+
+
+   ConstantCoefficient alpha(1.0);
+   ConstantCoefficient beta(1.0);
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+   a->AddTrialIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(beta),0,0);
+   a->AddTrialIntegrator(new NormalTraceIntegrator,1,0);
+   a->AddTestIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTestIntegrator(new VectorFEMassIntegrator(beta),0,0);
+
+
+   VectorFunctionCoefficient f(dim, f_exact);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      RTfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+
+   OperatorPtr Ah;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   // GridFunction u_gf;
+   double *data = x.GetData();
+   rt_gf.MakeRef(&RTfes,data);
+
+   GridFunction exact_gf(&RTfes);
+   exact_gf.ProjectCoefficient(F);
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << rt_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << exact_gf <<
+                 "window_title 'Exact' "
+                 << flush;
+
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+
+   F(0) = cos(kappa*x)*sin(kappa*y);
+   F(1) = cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      F(2) = 0.0;
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+
+   double temp = 1 + 2*kappa*kappa;
+
+   f(0) = temp*cos(kappa*x)*sin(kappa*y);
+   f(1) = temp*cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      f(2) = 0;
+   }
+}
@@ -0,0 +1,51 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+3
+1 3 0 1 4 3
+1 3 3 4 7 6
+1 3 1 2 5 4
+
+boundary
+8
+1 1 0 1
+1 1 1 2
+1 1 2 5
+2 1 5 4
+2 1 4 7
+1 1 7 6
+1 1 6 3
+1 1 3 0
+
+vertices
+8
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P1
+VDim: 2
+Ordering: 1
+
+-1 1
+-1 -0
+-1 -1
+0 1
+0 -0
+0 -1
+1 1
+1 -0
@@ -0,0 +1,907 @@
+//                   MFEM Ultraweak DPG Maxwell example
+//
+// Compile with: make complex_uw_dpg
+//
+
+//      ∇×(1/μ ∇×E) - ω^2 ϵ E = Ĵ ,   in Ω
+//                E×n = E_0, on ∂Ω
+
+// First Order System
+
+//  i ω μ H + ∇ × E = 0,   in Ω
+// -i ω ϵ E + ∇ × H = J,   in Ω
+//            E × n = E_0, on ∂Ω
+
+// note: Ĵ = -iωJ
+// in 2D 
+// E is vector valued and H is scalar. 
+//    (∇ × E, F) = (E, ∇ × F) + < n × E , F>
+// or (∇ ⋅ AE , F) = (AE, ∇ F) + < AE ⋅ n, F>
+// where A = A = [0 1; -1 0];
+
+// UW-DPG:
+// 
+// in 3D 
+// E,H ∈ (L^2(Ω))^3 
+// Ê ∈ H_0^1/2(Ω)(curl, Γ_h), Ĥ ∈ H^-1/2(curl, Γ_h)  
+//  i ω μ (H,F) + (E,∇ × F) + < Ê, F × n > = 0,      ∀ F ∈ H(curl,Ω)      
+// -i ω ϵ (E,G) + (H,∇ × G) + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)      
+//                                   Ê × n = E_0     on ∂Ω 
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) | < n × Ê, F > |              |         |
+// |   |              |             |              |              |         |
+// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < n × Ĥ, G > |  (J,G)  |  
+// where (F,G) ∈  H(curl,Ω) × H(curl,Ω)
+
+// in 2D 
+// E ∈ L^2(Ω)^2, H ∈ L^2(Ω)
+// Ê ∈ H^-1/2(Ω)(Γ_h), Ĥ ∈ H^1/2(Γ_h)  
+//  i ω μ (H,F) + (E, ∇ × F) + < AÊ, F > = 0,      ∀ F ∈ H^1      
+// -i ω ϵ (E,G) + (H,∇ × G)  + < Ĥ, G × n > = (J,G)   ∀ G ∈ H(curl,Ω)      
+//                                    Ê = E_0     on ∂Ω 
+// -------------------------------------------------------------------------
+// |   |       E      |      H      |      Ê       |       Ĥ      |  RHS    |
+// -------------------------------------------------------------------------
+// | F |  (E,∇ × F)   | i ω μ (H,F) |   < Ê, F >   |              |         |
+// |   |              |             |              |              |         |
+// | G | -i ω ϵ (E,G) |  (H,∇ × G)  |              | < Ĥ, G × n > |  (J,G)  |  
+
+// where (F,G) ∈  H^1 × H(curl,Ω)
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void E_exact_r(const Vector &x, Vector & E_r);
+void E_exact_i(const Vector &x, Vector & E_i);
+
+void H_exact_r(const Vector &x, Vector & H_r);
+void H_exact_i(const Vector &x, Vector & H_i);
+
+
+void  rhs_func_r(const Vector &x, Vector & J_r);
+void  rhs_func_i(const Vector &x, Vector & J_i);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+void curlE_exact_i(const Vector &x, Vector &curlE_i);
+void curlH_exact_r(const Vector &x,Vector &curlH_r);
+void curlH_exact_i(const Vector &x,Vector &curlH_i);
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r);
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i);
+
+void hatE_exact_r(const Vector & X, Vector & hatE_r);
+void hatE_exact_i(const Vector & X, Vector & hatE_i);
+
+void hatH_exact_r(const Vector & X, Vector & hatH_r);
+void hatH_exact_i(const Vector & X, Vector & hatH_i);
+
+double hatH_exact_scalar_r(const Vector & X);
+double hatH_exact_scalar_i(const Vector & X);
+
+void maxwell_solution(const Vector & X, 
+                      std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE);
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                      Vector &curlE_r, 
+                      Vector &curlcurlE_r);
+
+void maxwell_solution_i(const Vector & X, Vector &E_i, 
+                      Vector &curlE_i, 
+                      Vector &curlcurlE_i);     
+
+int dim;
+int dimc;
+double omega;
+double mu = 1.0;
+double epsilon = 1.0;
+
+enum prob_type
+{
+   polynomial,
+   plane_wave,
+   fichera_oven  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-hex.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");    
+   args.AddOption(&mu, "-mu", "--permeability",
+                  "Permeability of free space (or 1/(spring constant)).");
+   args.AddOption(&epsilon, "-eps", "--permittivity",
+                  "Permittivity of free space (or mass constant).");                  
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: plane wave, 2: Gaussian beam");                     
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+   dimc = (dim == 3) ? 3 : 1;
+   int test_order = order+delta_order;
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *E_fes = new FiniteElementSpace(&mesh,E_fec,dim);
+
+   // Vector L2 space for H 
+   FiniteElementCollection *H_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *H_fes = new FiniteElementSpace(&mesh,H_fec, dimc); 
+
+   // H^-1/2 (curl) space for Ê   
+   FiniteElementCollection * hatE_fec = nullptr;
+   FiniteElementCollection * hatH_fec = nullptr; 
+   FiniteElementCollection * F_fec = nullptr;
+   if (dim == 3)
+   {
+      hatE_fec = new ND_Trace_FECollection(order,dim);
+      hatH_fec = new ND_Trace_FECollection(order,dim);   
+      F_fec = new ND_FECollection(test_order, dim);
+   }
+   else
+   {
+      hatE_fec = new RT_Trace_FECollection(order-1,dim);
+      hatH_fec = new H1_Trace_FECollection(order,dim);   
+      F_fec = new H1_FECollection(test_order, dim);
+   } 
+   FiniteElementSpace *hatE_fes = new FiniteElementSpace(&mesh,hatE_fec);
+   FiniteElementSpace *hatH_fes = new FiniteElementSpace(&mesh,hatH_fec);
+
+   FiniteElementCollection * G_fec = new ND_FECollection(test_order, dim);
+
+
+   mfem::out << "E_fes space true dofs = " << E_fes->GetTrueVSize() << endl;
+   mfem::out << "H_fes space true dofs = " << H_fes->GetTrueVSize() << endl;
+   mfem::out << "hatE_fes space true dofs = " << hatE_fes->GetTrueVSize() << endl;
+   mfem::out << "hatH_fes space true dofs = " << hatH_fes->GetTrueVSize() << endl;
+
+
+//    // Coefficients
+   Vector dim_zero(dim); dim_zero = 0.0;
+   Vector dimc_zero(dimc); dimc_zero = 0.0;
+   VectorConstantCoefficient E_zero(dim_zero);
+   VectorConstantCoefficient H_zero(dimc_zero);
+
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient eps2omeg2(epsilon*epsilon*omega*omega);
+   ConstantCoefficient mu2omeg2(mu*mu*omega*omega);
+   ConstantCoefficient muomeg(mu*omega);
+   ConstantCoefficient negepsomeg(-epsilon*omega);
+   ConstantCoefficient epsomeg(epsilon*omega);
+   ConstantCoefficient negmuomeg(-mu*omega);
+
+   DenseMatrix rot_mat(2);
+   rot_mat(0,0) = 0.; rot_mat(0,1) = 1.;
+   rot_mat(1,0) = -1.; rot_mat(1,1) = 0.;
+   MatrixConstantCoefficient rot(rot_mat);
+   ScalarMatrixProductCoefficient epsrot(epsomeg,rot);
+   ScalarMatrixProductCoefficient negepsrot(negepsomeg,rot);
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(E_fes);
+   trial_fes.Append(H_fes);
+   trial_fes.Append(hatE_fes);
+   trial_fes.Append(hatH_fes);
+
+   test_fec.Append(F_fec);
+   test_fec.Append(G_fec);
+
+   ComplexNormalEquations * a = new ComplexNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices();
+
+   // (E,∇ × F)
+   a->AddTrialIntegrator(new TransposeIntegrator(new CurlIntegrator(one)),nullptr,0,0);
+
+   // -i ω ϵ (E , G)
+   a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(negepsomeg)),0,1);
+
+   // i ω μ (H, F)
+   if (dim == 3)
+   {
+      a->AddTrialIntegrator(nullptr,new TransposeIntegrator(new VectorFEMassIntegrator(muomeg)),1,0);
+   }
+   else
+   {
+      a->AddTrialIntegrator(nullptr,new MixedScalarMassIntegrator(muomeg),1,0);
+   }
+   //  (H,∇ × G) 
+   a->AddTrialIntegrator(new TransposeIntegrator(new CurlIntegrator(one)),nullptr,1,1);
+
+   // < n×Ê,F>
+   if (dim == 3)
+   {
+      a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,2,0);
+   }
+   else
+   {
+      a->AddTrialIntegrator(new TraceIntegrator,nullptr,2,0);
+   }
+
+   // < n×Ĥ ,G>
+   a->AddTrialIntegrator(new TangentTraceIntegrator,nullptr,3,1);
+
+
+   // test integrators 
+
+   //space-induced norm for H(curl) × H(curl)
+   if (dim == 3)
+   {
+      // (∇×F,∇×δF)
+      a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,0,0);
+      // (F,δF)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,0,0);
+   }
+   else
+   {
+      // (∇F,∇δF)
+      a->AddTestIntegrator(new DiffusionIntegrator(one),nullptr,0,0);
+      // (F,δF)
+      a->AddTestIntegrator(new MassIntegrator(one),nullptr,0,0);
+   }
+
+   // (∇×G ,∇× δG)
+   a->AddTestIntegrator(new CurlCurlIntegrator(one),nullptr,1,1);
+   // (G,δG)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),nullptr,1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      if(dim == 3)
+      {
+         // μ^2 ω^2 (F,δF)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(mu2omeg2),nullptr,0,0);
+         // -i ω μ (F,∇ × δG) = (F, ω μ ∇ × δ G)
+         a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(negmuomeg),0,1);
+         // -i ω ϵ (∇ × F, δG)
+         a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(negepsomeg),0,1);
+         // i ω μ (∇ × G,δF)
+         a->AddTestIntegrator(nullptr,new MixedVectorCurlIntegrator(epsomeg),1,0);
+         // i ω ϵ (G, ∇ × δF )
+         a->AddTestIntegrator(nullptr,new MixedVectorWeakCurlIntegrator(muomeg),1,0);
+         // ϵ^2 ω^2 (G,δG)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);
+      }
+      else
+      {
+         // μ^2 ω^2 (F,δF)
+         a->AddTestIntegrator(new MassIntegrator(mu2omeg2),nullptr,0,0);
+
+         // -i ω μ (F,∇ × δG) = i (F, -ω μ ∇ × δ G)
+         a->AddTestIntegrator(nullptr,
+            new TransposeIntegrator(new CurlIntegrator(negmuomeg)),0,1);
+
+         // -i ω ϵ (∇ × F, δG) = i (- ω ϵ A ∇ F,δG), A = [0 1; -1; 0]
+         a->AddTestIntegrator(nullptr,new MixedVectorGradientIntegrator(negepsrot),0,1);   
+
+         // i ω μ (∇ × G,δF) = i (ω μ ∇ × G, δF )
+         a->AddTestIntegrator(nullptr,new CurlIntegrator(muomeg),1,0);
+
+         // i ω ϵ (G, ∇ × δF ) =  i (ω ϵ G, A ∇ δF) = i ( G , ω ϵ A ∇ δF) 
+         a->AddTestIntegrator(nullptr,
+                   new TransposeIntegrator(new MixedVectorGradientIntegrator(epsrot)),1,0);
+
+         // or    i ( ω ϵ A^t G, ∇ δF) = i (- ω ϵ A G, ∇ δF)
+         // a->AddTestIntegrator(nullptr,
+                  //  new MixedVectorWeakDivergenceIntegrator(epsrot),1,0);
+         // ϵ^2 ω^2 (G,δG)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2omeg2),nullptr,1,1);            
+      }
+   }
+
+   // RHS
+   VectorFunctionCoefficient f_rhs_r(dim,rhs_func_r);
+   VectorFunctionCoefficient f_rhs_i(dim,rhs_func_i);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f_rhs_r),
+                            new VectorFEDomainLFIntegrator(f_rhs_i),1);
+   
+   
+   VectorFunctionCoefficient hatEex_r(dim,hatE_exact_r);
+   VectorFunctionCoefficient hatEex_i(dim,hatE_exact_i);
+
+   VectorFunctionCoefficient hatHex_r(dimc,hatH_exact_r);
+   VectorFunctionCoefficient hatHex_i(dimc,hatH_exact_i);
+
+   FunctionCoefficient hatH_2D_ex_r(hatH_exact_scalar_r);
+   FunctionCoefficient hatH_2D_ex_i(hatH_exact_scalar_i);
+
+
+   Array<int> elements_to_refine;
+
+   socketstream E_out_r;
+   socketstream Eex_out_r;
+   // socketstream E_out_i;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      E_out_r.open(vishost, visport);
+      Eex_out_r.open(vishost, visport);
+      // E_out_i.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatE_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+         // hatH_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int j = 0; j < ess_tdof_list.Size(); j++)
+      {
+         ess_tdof_list[j] += E_fes->GetTrueVSize() + H_fes->GetTrueVSize();
+                           // + hatE_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = E_fes->GetVSize();
+      offsets[2] = H_fes->GetVSize();
+      offsets[3] = hatE_fes->GetVSize();
+      offsets[4] = hatH_fes->GetVSize();
+      offsets.PartialSum();
+
+      Vector x(2*offsets.Last());
+      x = 0.;
+      double * xdata = x.GetData();
+
+      ComplexGridFunction hatE_gf(hatE_fes);
+      hatE_gf.real().MakeRef(hatE_fes,&xdata[offsets[2]]);
+      hatE_gf.imag().MakeRef(hatE_fes,&xdata[offsets.Last()+ offsets[2]]);
+
+      ComplexGridFunction hatH_gf(hatH_fes);
+      hatH_gf.real().MakeRef(hatH_fes,&xdata[offsets[3]]);
+      hatH_gf.imag().MakeRef(hatH_fes,&xdata[offsets.Last()+ offsets[3]]);
+
+      if (dim == 3)
+      {
+         hatE_gf.ProjectBdrCoefficientTangent(hatEex_r,hatEex_i, ess_bdr);
+         // hatH_gf.ProjectBdrCoefficientTangent(hatHex_r,hatHex_i, ess_bdr);
+      }
+      else
+      {
+         hatE_gf.ProjectBdrCoefficientNormal(hatEex_r,hatEex_i, ess_bdr);
+         // hatH_gf.ProjectBdrCoefficient(hatH_2D_ex_r,hatH_2D_ex_i, ess_bdr);
+
+      }
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah, X,B);
+
+      ComplexOperator * Ahc = Ah.As<ComplexOperator>();
+
+      SparseMatrix * Ar = dynamic_cast<BlockMatrix *>(&Ahc->real())->CreateMonolithic();
+      SparseMatrix * Ai = dynamic_cast<BlockMatrix *>(&Ahc->imag())->CreateMonolithic();
+
+      ComplexSparseMatrix Ac(Ar,Ai,true,true);
+      SparseMatrix * A = Ac.GetSystemMatrix();
+
+      UMFPackSolver umf(*A);
+      umf.Mult(B,X);
+
+      delete A;
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+      double residual = residuals.Norml2();
+
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ComplexGridFunction E(E_fes);
+      E.real().MakeRef(E_fes,x.GetData());
+      E.imag().MakeRef(E_fes,&x.GetData()[offsets.Last()]);
+
+      VectorFunctionCoefficient E_ex_r(dim,E_exact_r);
+      VectorFunctionCoefficient E_ex_i(dim,E_exact_i);
+
+      ComplexGridFunction H(H_fes);
+      H.real().MakeRef(H_fes,&x.GetData()[offsets[1]]);
+      H.imag().MakeRef(H_fes,&x.GetData()[offsets.Last()+offsets[1]]);
+
+      VectorFunctionCoefficient H_ex_r(dimc,H_exact_r);
+      VectorFunctionCoefficient H_ex_i(dimc,H_exact_i);
+      
+
+      int dofs = X.Size()/2;
+
+      double E_err_r = E.real().ComputeL2Error(E_ex_r);
+      double E_err_i = E.imag().ComputeL2Error(E_ex_i);
+      double H_err_r = H.real().ComputeL2Error(H_ex_r);
+      double H_err_i = H.imag().ComputeL2Error(H_ex_i);
+
+      double L2Error = sqrt(  E_err_r*E_err_r + E_err_i*E_err_i 
+                            + H_err_r*H_err_r + H_err_i*H_err_i );
+
+      ComplexGridFunction Egf_ex(E_fes);
+      ComplexGridFunction Hgf_ex(H_fes);
+      Egf_ex.ProjectCoefficient(E_ex_r, E_ex_i);
+      Hgf_ex.ProjectCoefficient(H_ex_r, H_ex_i);
+
+   
+
+      double E_norm_r = Egf_ex.real().ComputeL2Error(E_zero);
+      double E_norm_i = Egf_ex.imag().ComputeL2Error(E_zero);
+      double H_norm_r = Hgf_ex.real().ComputeL2Error(H_zero);
+      double H_norm_i = Hgf_ex.imag().ComputeL2Error(H_zero);
+
+      double L2norm = sqrt(  E_norm_r*E_norm_r + E_norm_i*E_norm_i 
+                           + H_norm_r*H_norm_r + H_norm_i*H_norm_i );
+
+      double rel_err = L2Error/L2norm;
+
+
+      double rate_err = (i) ? dim*log(err0/rel_err)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = rel_err;
+      res0 = residual;
+      dof0 = dofs;
+
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_err*100 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::setw(10) << std::scientific 
+                << std::endl;
+
+      if (visualization)
+      {
+         E_out_r.precision(8);
+         E_out_r << "solution\n" << mesh << E.real() <<
+                  "window_title 'Real Numerical Electric field' "
+                  << flush;
+
+         // E_out_i.precision(8);
+         // E_out_i << "solution\n" << mesh << E.imag() <<
+         //          "window_title 'Imag Numerical Electric field' "
+         //          << flush;         
+
+
+
+
+         Eex_out_r.precision(8);
+         Eex_out_r << "solution\n" << mesh << Egf_ex.real()  
+                  << "window_title 'Real Exact Electric field' " 
+                  << flush;
+         // socketstream E_i_sock(vishost, visport);
+         // E_i_sock.precision(8);
+         // E_i_sock << "solution\n" << mesh << Egf_ex.imag()  
+         //          << "window_title 'Imag Exact Electric field' " 
+         //          << flush;
+
+
+      }
+
+      if (i == ref-1)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete F_fec;
+   delete G_fec;
+   delete hatH_fes;
+   delete hatH_fec;
+   delete hatE_fes;
+   delete hatE_fec;
+   delete H_fec;
+   delete E_fec;
+   delete H_fes;
+   delete E_fes;
+
+   return 0;
+}                                       
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   Vector curlE_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void E_exact_i(const Vector &x, Vector & E_i)
+{
+   Vector curlE_i;
+   Vector curlcurlE_i;
+
+   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   Vector E_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void curlE_exact_i(const Vector &x, Vector &curlE_i)
+{
+   Vector E_i;
+   Vector curlcurlE_i;
+
+   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
+}
+
+void curlcurlE_exact_r(const Vector &x, Vector & curlcurlE_r)
+{
+   Vector E_r;
+   Vector curlE_r;
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void curlcurlE_exact_i(const Vector &x, Vector & curlcurlE_i)
+{
+   Vector E_i;
+   Vector curlE_i;
+   maxwell_solution_i(x,E_i,curlE_i,curlcurlE_i);
+}
+
+
+void H_exact_r(const Vector &x, Vector & H_r)
+{
+   // H = i ∇ × E / ω μ  
+   // H_r = - ∇ × E_i / ω μ  
+   Vector curlE_i;
+   curlE_exact_i(x,curlE_i);
+   H_r.SetSize(dimc);
+   for (int i = 0; i<dimc; i++)
+   {
+      H_r(i) = - curlE_i(i) / (omega * mu);
+   }
+}
+
+void H_exact_i(const Vector &x, Vector & H_i)
+{
+   // H = i ∇ × E / ω μ  
+   // H_i =  ∇ × E_r / ω μ  
+   Vector curlE_r;
+   curlE_exact_r(x,curlE_r);
+   H_i.SetSize(dimc);
+   for (int i = 0; i<dimc; i++)
+   {
+      H_i(i) = curlE_r(i) / (omega * mu);
+   }
+}
+
+void curlH_exact_r(const Vector &x,Vector &curlH_r)
+{
+   // ∇ × H_r = - ∇ × ∇ × E_i / ω μ  
+   Vector curlcurlE_i;
+   curlcurlE_exact_i(x,curlcurlE_i);
+   curlH_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      curlH_r(i) = -curlcurlE_i(i) / (omega * mu);
+   }
+}
+
+void curlH_exact_i(const Vector &x,Vector &curlH_i)
+{
+   // ∇ × H_i = ∇ × ∇ × E_r / ω μ  
+   Vector curlcurlE_r;
+   curlcurlE_exact_r(x,curlcurlE_r);
+   curlH_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      curlH_i(i) = curlcurlE_r(i) / (omega * mu);
+   }
+}
+
+
+void hatE_exact_r(const Vector & x, Vector & hatE_r)
+{
+   if (dim == 3)
+   {
+      E_exact_r(x,hatE_r);
+   }
+   else
+   {
+      Vector E_r;
+      E_exact_r(x,E_r);
+      hatE_r.SetSize(hatE_r.Size());
+      // rotate E_hat
+      hatE_r[0] = E_r[1];
+      hatE_r[1] = -E_r[0];
+   }
+}
+
+void hatE_exact_i(const Vector & x, Vector & hatE_i)
+{
+   if (dim == 3)
+   {
+      E_exact_i(x,hatE_i);
+   }
+   else
+   {
+      Vector E_i;
+      E_exact_i(x,E_i);
+      hatE_i.SetSize(hatE_i.Size());
+      // rotate E_hat
+      hatE_i[0] = E_i[1];
+      hatE_i[1] = -E_i[0];
+   }
+}
+
+
+void hatH_exact_r(const Vector & x, Vector & hatH_r)
+{
+   H_exact_r(x,hatH_r);
+}
+
+void hatH_exact_i(const Vector & x, Vector & hatH_i)
+{
+   H_exact_i(x,hatH_i);
+}
+
+double hatH_exact_scalar_r(const Vector & x)
+{
+   Vector hatH_r;
+   H_exact_r(x,hatH_r);
+   return hatH_r[0];
+}
+
+double hatH_exact_scalar_i(const Vector & x)
+{
+   Vector hatH_i;
+   H_exact_i(x,hatH_i);
+   return hatH_i[0];
+}
+
+// J = -i ω ϵ E + ∇ × H 
+// J_r + iJ_i = -i ω ϵ (E_r + i E_i) + ∇ × (H_r + i H_i) 
+void  rhs_func_r(const Vector &x, Vector & J_r)
+{
+   // J_r = ω ϵ E_i + ∇ × H_r
+   Vector E_i, curlH_r;
+   E_exact_i(x,E_i);
+   curlH_exact_r(x,curlH_r);
+   J_r.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_r(i) = omega * epsilon * E_i(i) + curlH_r(i);
+   }
+}
+
+void  rhs_func_i(const Vector &x, Vector & J_i)
+{
+   // J_i = - ω ϵ E_r + ∇ × H_i
+   Vector E_r, curlH_i;
+   E_exact_r(x,E_r);
+   curlH_exact_i(x,curlH_i);
+   J_i.SetSize(dim);
+   for (int i = 0; i<dim; i++)
+   {
+      J_i(i) = -omega * epsilon * E_r(i) + curlH_i(i);
+   }
+}
+
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE)
+{
+   double x = X(0);
+   double y = X(1);
+   double z;
+   if (dim == 3) z = X(2);
+
+   E.resize(dim);
+   curlE.resize(dimc);
+   curlcurlE.resize(dim);
+   switch (prob)
+   {
+   case prob_type::polynomial:
+   {
+      if (dim == 3)
+      {
+         E[0] = y * z * (1.0 - y) * (1.0 - z);
+         E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+         E[2] = x * y * (1.0 - x) * (1.0 - y);
+         curlE[0] = (1.0 - x) * x * (y*(2.0*z-3.0)+1.0);
+         curlE[1] = 2.0*(1.0 - y)*y*(x-z);
+         curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+         curlcurlE[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+         curlcurlE[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+         curlcurlE[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+      }
+      else
+      {
+         E[0] = y * (1.0 - y);
+         E[1] = x * y * (1.0 - x);
+         curlE[0] = y*(3.0 - 2*x) - 1.0;
+         curlcurlE[0] = 3.0 - 2*x;
+         curlcurlE[1] = 2.0*y;
+      }
+   }
+      break;
+
+   case prob_type::plane_wave:
+   {
+      std::complex<double> zi(0,1);
+      std::complex<double> pw = exp(-zi * omega * (X.Sum()));
+      E[0] = pw;
+      E[1] = 0.0;
+      if (dim == 3)
+      {
+         E[2] = 0.0;
+         curlE[0] = 0.0;
+         curlE[1] = -zi * omega * pw;
+         curlE[2] =  zi * omega * pw;
+
+         curlcurlE[0] = 2.0 * omega * omega * pw;
+         curlcurlE[1] = - omega * omega * pw;
+         curlcurlE[2] = - omega * omega * pw;
+      }
+      else
+      {
+         curlE[0] = zi * omega * pw;
+         curlcurlE[0] =   omega * omega * pw;
+         curlcurlE[1] = - omega * omega * pw ;
+      }
+   }
+      break;
+
+   default:
+      MFEM_ABORT("Fichera 'oven' problem not implemented yet");
+      break;
+   }
+   
+}
+
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                        Vector &curlE_r, 
+                        Vector &curlcurlE_r)
+{
+   E_r.SetSize(dim);
+   curlE_r.SetSize(dimc);
+   curlcurlE_r.SetSize(dim);
+
+   std::vector<complex<double>> E;
+   std::vector<complex<double>> curlE;
+   std::vector<complex<double>> curlcurlE;
+
+   maxwell_solution(X,E,curlE,curlcurlE);
+   for (int i = 0; i<dim ; i++)
+   {
+      E_r(i) = E[i].real();
+      curlcurlE_r(i) = curlcurlE[i].real();
+   }
+   for (int i = 0; i<dimc; i++)
+   {
+      curlE_r(i) = curlE[i].real();
+   }
+}
+
+
+void maxwell_solution_i(const Vector & X, Vector &E_i, 
+                      Vector &curlE_i, 
+                      Vector &curlcurlE_i)
+{
+   E_i.SetSize(dim);
+   curlE_i.SetSize(dimc);
+   curlcurlE_i.SetSize(dim);
+
+   std::vector<complex<double>> E;
+   std::vector<complex<double>> curlE;
+   std::vector<complex<double>> curlcurlE;
+
+   maxwell_solution(X,E,curlE,curlcurlE);
+   for (int i = 0; i<dim; i++)
+   {
+      E_i(i) = E[i].imag();
+      curlcurlE_i(i) = curlcurlE[i].imag();
+   }
+   for (int i = 0; i<dimc; i++)
+   {
+      curlE_i(i) = curlE[i].imag();
+   }
+}  
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = complex_uw_dpg complex_uw_dpg_2D
+PAR_EXAMPLES = pcomplex_uw_dpg
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
@@ -0,0 +1,253 @@
+// Test integrator
+// (∇ × E, F) 
+
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void E_exact_r(const Vector &x, Vector & E_r);
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r);
+
+void maxwell_solution(const Vector & X, 
+                      std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE);
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                      Vector &curlE_r, 
+                      Vector &curlcurlE_r);
+
+int dim;
+int dimc;
+double omega;
+
+enum prob_type
+{
+   polynomial,
+   plane_wave,
+   fichera_oven  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-hex.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   bool static_cond = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: plane wave, 2: Gaussian beam");                     
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+   omega = 2.*M_PI*rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   dimc = (dim == 3) ? 3 : 1;
+
+   // Define spaces
+   // L2 space for E
+   FiniteElementCollection *E_fec = new ND_FECollection(order,dim);
+   FiniteElementSpace *E_fes = new FiniteElementSpace(&mesh,E_fec);
+
+
+   FiniteElementCollection *curlE_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *curlE_fes = new FiniteElementSpace(&mesh,curlE_fec,dimc);
+
+   mfem::out << "E_fes space true dofs = " << E_fes->GetTrueVSize() << endl;
+   mfem::out << "curlE_fes space true dofs = " << curlE_fes->GetTrueVSize() << endl;
+
+
+   GridFunction E_gf(E_fes);
+   VectorFunctionCoefficient E_cf(dim,E_exact_r);
+   E_gf.ProjectCoefficient(E_cf);
+
+   GridFunction curlE_gf(curlE_fes);
+   VectorFunctionCoefficient curlE_cf(dimc,curlE_exact_r);
+   curlE_gf.ProjectCoefficient(curlE_cf);
+
+
+   char vishost[] = "localhost";
+   int visport = 19916;
+   socketstream E_sock(vishost, visport);
+   E_sock.precision(8);
+   E_sock << "solution\n"
+               << mesh << E_gf 
+               << "window_title 'Exact E'" << flush;
+
+   socketstream curlE_sock(vishost, visport);
+   curlE_sock.precision(8);
+   curlE_sock << "solution\n"
+               << mesh << curlE_gf 
+               << "window_title 'Exact curlE'" << flush;
+
+
+
+   MixedBilinearForm a(E_fes,curlE_fes);
+   a.AddDomainIntegrator(new CurlIntegrator());
+   a.Assemble();
+   Array<int> empty;
+   SparseMatrix A;
+   a.FormRectangularSystemMatrix(empty,empty,A);
+
+
+   Vector curl_load(A.Height());
+   A.Mult(E_gf,curl_load);
+   BilinearForm m(curlE_fes);
+   m.AddDomainIntegrator(new VectorMassIntegrator);
+   m.Assemble();
+   SparseMatrix M;
+   m.FormSystemMatrix(empty, M);
+
+
+   GSSmoother prec(M);
+   PCG(M, prec, curl_load, curlE_gf, 1, 200, 1e-12, 0.0);
+
+
+   socketstream curlE2_sock(vishost, visport);
+   curlE2_sock.precision(8);
+   curlE2_sock << "solution\n"
+               << mesh << curlE_gf 
+               << "window_title 'Numerical curlE'" << flush;
+
+
+   delete E_fec;
+   delete E_fes;
+   delete curlE_fec;
+   delete curlE_fes;
+
+   return 0;
+}                                       
+
+void E_exact_r(const Vector &x, Vector & E_r)
+{
+   Vector curlE_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+void curlE_exact_r(const Vector &x, Vector &curlE_r)
+{
+   Vector E_r;
+   Vector curlcurlE_r;
+
+   maxwell_solution_r(x,E_r,curlE_r,curlcurlE_r);
+}
+
+
+
+void maxwell_solution(const Vector & X, std::vector<complex<double>> &E, 
+                      std::vector<complex<double>> &curlE, 
+                      std::vector<complex<double>> &curlcurlE)
+{
+   double x = X(0);
+   double y = X(1);
+   double z;
+   if (dim == 3)
+   {
+      z = X(2);
+   } 
+
+   E.resize(dim);
+   curlE.resize(dimc);
+   curlcurlE.resize(dim);
+
+   if (dim == 3)
+   {
+      E[0] = y * z * (1.0 - y) * (1.0 - z);
+      E[1] = x * y * z * (1.0 - x) * (1.0 - z);
+      E[2] = x * y * (1.0 - x) * (1.0 - y);
+      
+      curlE[0] = (1.0 - x) * x * (y*(2.0*z-3.0)+1.0);
+      curlE[1] = 2.0*(1.0 - y)*y*(x-z);
+      curlE[2] = (z-1)*z*(1.0+y*(2.0*x-3.0));
+      
+      curlcurlE[0] = 2.0 * y * (1.0 - y) - (2.0 * x - 3.0) * z * (1 - z);
+      curlcurlE[1] = 2.0 * y * (x * (1.0 - x) + (1.0 - z) * z);
+      curlcurlE[2] = 2.0 * y * (1.0 - y) + x * (3.0 - 2.0 * z) * (1.0 - x);
+   }
+   else if (dim == 2)
+   {
+      double c = 2.0*M_PI;
+      E[0] = sin(c * y);
+      E[1] = sin(c * x);
+      curlE[0] = c * (cos(c*x) - cos(c*y));
+      curlcurlE[0] = c*c * sin(c*y);
+      curlcurlE[1] = c*c * sin(c*x);
+   }
+   else
+   {
+      MFEM_ABORT("Dimension cannot be 1");
+   }
+
+}
+
+void maxwell_solution_r(const Vector & X, Vector &E_r, 
+                        Vector &curlE_r, 
+                        Vector &curlcurlE_r)
+{
+   E_r.SetSize(dim);
+   curlE_r.SetSize(dimc);
+   curlcurlE_r.SetSize(dim);
+
+   std::vector<complex<double>> E;
+   std::vector<complex<double>> curlE;
+   std::vector<complex<double>> curlcurlE;
+
+   maxwell_solution(X,E,curlE,curlcurlE);
+   for (int i = 0; i<dim; i++)
+   {
+      E_r(i) = E[i].real();
+      curlcurlE_r(i) = curlcurlE[i].real();
+   }
+   for (int i = 0; i<dimc; i++)
+   {
+      curlE_r(i) = curlE[i].real();
+   }
+}
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
-//             * ex1 -fa -d cuda
+//               ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -30,7 +30,7 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
-//             * mpirun -np 4 ex1p -fa -d cuda
+//               mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
@@ -195,7 +195,7 @@ int main(int argc, char *argv[])
   Array<int> ess_tdof_list(0);
   if (h1 && pmesh.bdr_attributes.Size())
   {
-      // For a continuous basis the linear system must be modified to enforce an
+      // For a continuous basis the linear system must be modifed to enforce an
      // essential (Dirichlet) boundary condition. In the DG case this is not
      // necessary as the boundary condition will only be enforced weakly.
      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
      SparseMatrix &M(mVarf->SpMat());
      SparseMatrix &B(bVarf->SpMat());
      B *= -1.;
+      B.EnsureMultTranspose();
      Bt = new TransposeOperator(&B);

      darcyOp.SetBlock(0,0, &M);
@@ -72,8 +72,8 @@ int main(int argc, char *argv[])
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
   {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = 1;
+      // (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
@@ -147,6 +147,8 @@ int main(int argc, char *argv[])
   F.AddDomainIntegrator(new DomainLFIntegrator(one));
   F.Assemble();

+
+
   // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
   //    the mixed bilinear form for the interfacial unknowns, Bhat,
   //    the inverse stiffness matrix on the discontinuous test space, Sinv,
@@ -187,10 +189,17 @@ int main(int argc, char *argv[])
   // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
   //    the normal equation operator, A = B^t Sinv B, and
   //    the normal equation right-hand-size, b = B^t Sinv F.
-   BlockOperator B(offsets_test, offsets);
+   // BlockOperator B(offsets_test, offsets);
+
+   BlockMatrix B(offsets_test, offsets);
   B.SetBlock(0,0,&matB0);
   B.SetBlock(0,1,&matBhat);
-   RAPOperator A(B, matSinv, B);
+   SparseMatrix * Bh = B.CreateMonolithic();
+
+   SparseMatrix * A = RAP(*Bh, matSinv, *Bh);
+
+
+   // RAPOperator A(B, matSinv, B);
   {
      Vector SinvF(s_test);
      matSinv.Mult(F,SinvF);
@@ -234,7 +243,20 @@ int main(int argc, char *argv[])
   // 10. Solve the normal equation system using the PCG iterative solver.
   //     Check the weighted norm of residual for the DPG least square problem.
   //     Wrap the primal variable in a GridFunction for visualization purposes.
-   PCG(A, P, b, x, 1, 200, 1e-12, 0.0);
+
+
+   // PCG(*A, P, b, x, 1, 200, 1e-12, 0.0);
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(b, x);
+

   {
      Vector LSres(s_test);
@@ -187,6 +187,7 @@ int main(int argc, char *argv[])
 {
   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();

@@ -248,10 +248,7 @@ int main(int argc, char *argv[])
   //     constraints for non-conforming AMR, static condensation, etc.
   if (myid == 0) { cout << "matrix ... " << flush; }
   if (static_cond) { a->EnableStaticCondensation(); }
-   // Here we want to try out block-size aware AMG solver in PETSc.
-   // For that to work properly, we need a fully-compliant block-size
-   // structure and we do not skip zeros when assembling.
-   a->Assemble(use_petsc ? 0 : 1);
+   a->Assemble();

   Vector B, X;
   if (!use_petsc)
@@ -297,14 +294,13 @@ int main(int argc, char *argv[])
         cout << "done." << endl;
         cout << "Size of linear system: " << A.M() << endl;
      }
-      // Tell PETSc the matrix has a block structure
-      A.SetBlockSize(dim);
-
-      // The preconditioner for the PCG solver can be specified in the
-      // PETSc config file
      PetscPCGSolver *pcg = new PetscPCGSolver(A);
+
+      // The preconditioner for the PCG solver defined below is specified in the
+      // PETSc config file, rc_ex2p, since a Krylov solver in PETSc can also
+      // customize its preconditioner.
      PetscPreconditioner *prec = NULL;
-      if (use_nonoverlapping) // Specialized BDDC construction
+      if (use_nonoverlapping)
      {
         // Compute dofs belonging to the natural boundary
         Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());
@@ -450,7 +450,7 @@ int main(int argc, char *argv[])
      for (int ti = 0; !done; )
      {
         // We cannot match exactly the time history of the Run method
-         // since we are explicitly telling PETSc to use a time step
+         // since we are explictly telling PETSc to use a time step
         double dt_real = min(dt, t_final - t);
         ode_solver->Step(*U, t, dt_real);
         ti++;
@@ -78,7 +78,6 @@ EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly -
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
-EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -110,7 +109,6 @@ endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_ASM))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p
@@ -1,7 +1,8 @@
 -ksp_converged_reason

 # GAMG is still not used at its best,
-# since we are not exploiting the RBMs
+# since we are not exploiting the
+# block size (Ordering::byVDIM) and the RBMs

 -ksp_view
 -pc_type gamg
@@ -1,10 +0,0 @@
-# Additive Schwarz with Overlap
-# This is not a good solver for elasticity
-# These options are here only to describe
-# the setup of the solver
-ksp_converged_reason
-ksp_view
-ksp_max_it 10
-pc_type asm
-pc_asm_overlap 1
-sub_pc_type icc
@@ -210,9 +210,6 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,

 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
   // 1. Parse command-line options.
   const char *mesh_file = "../../data/beam-quad.mesh";
   int ref_levels = 2;
@@ -215,11 +215,10 @@ void visualize(ostream &os, ParMesh *mesh, ParGridFunction *deformed_nodes,

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
   int myid = Mpi::WorldRank();
   Hypre::Init();
-   Sundials::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/beam-quad.mesh";
@@ -109,9 +109,6 @@ double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
   // 1. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
   int ref_levels = 2;
@@ -293,10 +290,7 @@ int main(int argc, char *argv[])
         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
         ode_solver = arkode; break;
      case 12:
         arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);
@@ -101,12 +101,11 @@ double InitialTemperature(const Vector &x);

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();
-   Sundials::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../../data/star.mesh";
@@ -328,10 +327,7 @@ int main(int argc, char *argv[])
         arkode->Init(oper);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
         ode_solver = arkode; break;
      case 12:
         arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);
@@ -140,9 +140,6 @@ public:

 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
   // 1. Parse command-line options.
   problem = 0;
   const char *mesh_file = "../../data/periodic-hexagon.mesh";
@@ -411,7 +408,7 @@ int main(int argc, char *argv[])
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         arkode->SetERKTableNum(FEHLBERG_13_7_8);
         ode_solver = arkode; break;
   }

@@ -152,12 +152,11 @@ public:

 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
   Mpi::Init(argc, argv);
   int num_procs = Mpi::WorldSize();
   int myid = Mpi::WorldRank();
   Hypre::Init();
-   Sundials::Init();

   // 2. Parse command-line options.
   problem = 0;
@@ -488,10 +487,7 @@ int main(int argc, char *argv[])
         arkode->Init(adv);
         arkode->SetSStolerances(reltol, abstol);
         arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
         ode_solver = arkode; break;
   }

@@ -35,7 +35,7 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 if (MFEM_ENABLE_TESTING)
  # Command line options for the tests.
  # Example 1: Test SuperLU on the simple Poisson problem
-  set(EX1_COMMON_OPTS -m ../../data/star.mesh)
+  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
  set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})

  # Add the tests: one test per source file.
@@ -39,7 +39,6 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
-  dgmassinv.cpp
  doftrans.cpp
  eltrans.cpp
  estimators.cpp
@@ -73,7 +72,6 @@ set(SRCS
  linearform.cpp
  linearform_ext.cpp
  lininteg.cpp
-  lininteg_boundary.cpp
  lininteg_domain.cpp
  lininteg_domain_grad.cpp
  lor/lor.cpp
@@ -90,7 +88,6 @@ set(SRCS
  fespacehierarchy.cpp
  nonlininteg_vectorconvection.cpp
  nonlininteg_vectorconvection_mf.cpp
-  qfunction.cpp
  qinterp/det.cpp
  qinterp/eval_by_nodes.cpp
  qinterp/eval_by_vdim.cpp
@@ -98,7 +95,6 @@ set(SRCS
  qinterp/grad_by_vdim.cpp
  qinterp/grad_phys_by_nodes.cpp
  qinterp/grad_phys_by_vdim.cpp
-  qspace.cpp
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
@@ -140,13 +136,10 @@ set(HDRS
  bilinearform.hpp
  bilinearform_ext.hpp
  bilininteg.hpp
-  bilininteg_mass_pa.hpp
  coefficient.hpp
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
-  dgmassinv.hpp
-  dgmassinv_kernels.hpp
  doftrans.hpp
  eltrans.hpp
  estimators.hpp
@@ -196,11 +189,9 @@ set(HDRS
  nonlinearform.hpp
  nonlinearform_ext.hpp
  nonlininteg.hpp
-  qfunction.hpp
  qinterp/dispatch.hpp
  qinterp/eval.hpp
  qinterp/grad.hpp
-  qspace.hpp
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
@@ -136,7 +136,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
         ext = new MFBilinearFormExtension(this);
         break;
      default:
-         MFEM_ABORT("BilinearForm: unknown assembly level");
+         mfem_error("Unknown assembly level");
   }
 }

@@ -992,7 +992,6 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
      mat_e = new SparseMatrix(height);
   }

-   vdofs_.HostRead();
   for (int i = 0; i < vdofs_.Size(); i++)
   {
      int vdof = vdofs_[i];
@@ -26,8 +26,7 @@ namespace mfem
 {

 /** @brief Enumeration defining the assembly level for bilinear and nonlinear
-    form classes derived from Operator. For more details, see
-    https://mfem.org/howto/assembly_levels */
+    form classes derived from Operator. */
 enum class AssemblyLevel
 {
   /// In the case of a BilinearForm LEGACY corresponds to a fully assembled
@@ -178,7 +177,7 @@ public:
       - AssemblyLevel::ELEMENT
       - AssemblyLevel::NONE

-       If used, this method must be called before assembly. */
+       This method must be called before assembly. */
   void SetAssemblyLevel(AssemblyLevel assembly_level);

   /// Returns the assembly level
@@ -334,7 +333,7 @@ public:


   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transferring ownership. */
+        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
@@ -775,7 +774,7 @@ public:
   SparseMatrix &SpMat() { return *mat; }

   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transferring ownership. */
+        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /// Adds a domain integrator. Assumes ownership of @a bfi.
@@ -18,8 +18,6 @@
 #include "pgridfunc.hpp"
 #include "ceed/interface/util.hpp"

-#include "../general/nvtx.hpp"
-
 namespace mfem
 {

@@ -162,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultMF(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
      }
   }

@@ -178,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultMF(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
      }
   }
 }
@@ -219,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposeMF(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
      }
   }

@@ -235,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposeMF(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
      }
   }
 }
@@ -291,10 +289,6 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)

 void PABilinearFormExtension::Assemble()
 {
-#undef MFEM_NVTX_COLOR
-#define MFEM_NVTX_COLOR NavyBlue
-   NVTX("HO Assemble");
-
   SetupRestrictionOperators(L2FaceValues::DoubleValued);

   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
@@ -389,10 +383,6 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,

 void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
 {
-#undef MFEM_NVTX_COLOR
-#define MFEM_NVTX_COLOR MediumSpringGreen
-   NVTX("HO Apply");
-
   Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();

   const int iSz = integrators.Size();
@@ -428,7 +418,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
      }
   }

@@ -444,7 +434,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
      }
   }
 }
@@ -485,7 +475,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposePA(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
      }
   }

@@ -501,7 +491,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
      }
   }
 }
@@ -678,7 +668,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
      }
   }

@@ -709,7 +699,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
      }
   }
 }
@@ -806,7 +796,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
      }
   }

@@ -837,7 +827,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
      }
   }
 }
@@ -985,9 +975,6 @@ void FABilinearFormExtension::RAP(OperatorHandle &A)
 void FABilinearFormExtension::EliminateBC(const Array<int> &ess_dofs,
                                          OperatorHandle &A)
 {
-   MFEM_VERIFY(a->diag_policy == DiagonalPolicy::DIAG_ONE,
-               "Only DiagonalPolicy::DIAG_ONE supported with"
-               " FABilinearFormExtension.");
 #ifdef MFEM_USE_MPI
   if ( dynamic_cast<ParBilinearForm*>(a) )
   {
@@ -144,6 +144,14 @@ void BilinearFormIntegrator::AssembleFaceMatrix (
               "   is not implemented for this class.");
 }

+void BilinearFormIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                     const FiniteElement &trial_face_fe,  const FiniteElement &test_fe,
+                                                     FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   mfem_error ("BilinearFormIntegrator::AssembleTraceFaceMatrix(...)\n"
+               "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssembleFaceMatrix(
   const FiniteElement &trial_face_fe, const FiniteElement &test_fe1,
   const FiniteElement &test_fe2, FaceElementTransformations &Trans,
@@ -793,6 +801,84 @@ const IntegrationRule &GradientIntegrator::GetRule(const FiniteElement
 }


+void CurlIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans,  DenseMatrix &elmat)
+{
+   int dim = trial_fe.GetDim();
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dimc = (dim == 3) ? 3 : 1;
+
+   MFEM_ASSERT(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
+               dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE,
+               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
+   MFEM_ASSERT(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
+               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
+               "Test finite element must be in H1/L2");
+
+   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
+
+   if (spaceH1)
+   {
+      dshape.SetSize(trial_dof,dim);
+      curlshape.SetSize(dim*trial_dof,1);
+      dimc = dim;
+   }
+   else
+   {
+      curlshape.SetSize(trial_dof,dimc);
+      elmat_comp.SetSize(test_dof, trial_dof);
+   }
+   elmat.SetSize(dimc * test_dof, trial_dof);
+   shape.SetSize(test_dof);
+   elmat = 0.0;
+
+   double c;
+   Vector d_col;
+   const IntegrationRule *ir = IntRule;
+
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+      if (spaceH1)
+      {
+         trial_fe.CalcPhysDShape(Trans, dshape);
+         dshape.GradToCurl(curlshape);
+      }
+      else
+      {
+         trial_fe.CalcPhysCurlShape(Trans, curlshape);
+      }
+      test_fe.CalcPhysShape(Trans, shape);
+      c = ip.weight*Trans.Weight();
+      if (Q)
+      {
+         c *= Q->Eval(Trans, ip);
+      }
+      shape *= c;
+
+      for (int d = 0; d < dimc; ++d)
+      {
+         double * curldata = &(curlshape.GetData())[d*trial_dof];
+         for (int jj = 0; jj < trial_dof; ++jj)
+         {
+            for (int ii = 0; ii < test_dof; ++ii)
+            {
+               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+            }
+         }
+      }
+   }
+}
+
 void DiffusionIntegrator::AssembleElementMatrix
 ( const FiniteElement &el, ElementTransformation &Trans,
  DenseMatrix &elmat )
@@ -2003,6 +2089,7 @@ void CurlCurlIntegrator::AssembleElementMatrix
   }
 }

+
 void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
                                                const FiniteElement &test_fe,
                                                ElementTransformation &Trans,
@@ -2017,7 +2104,7 @@ void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
 #ifdef MFEM_THREAD_SAFE
   Vector D;
   DenseMatrix curlshape(tr_nd,dimc), curlshape_dFt(tr_nd,dimc), M;
-   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc);
+   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc), M;
 #else
   curlshape.SetSize(tr_nd,dimc);
   curlshape_dFt.SetSize(tr_nd,dimc);
@@ -2317,84 +2404,6 @@ double VectorCurlCurlIntegrator::GetElementEnergy(
   return 0.5 * energy;
 }

-void MixedCurlIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe,
-   ElementTransformation &Trans, DenseMatrix &elmat)
-{
-   int dim = trial_fe.GetDim();
-   int trial_dof = trial_fe.GetDof();
-   int test_dof = test_fe.GetDof();
-   int dimc = (dim == 3) ? 3 : 1;
-
-   MFEM_VERIFY(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
-               (dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE),
-               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
-   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
-               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
-               "Test finite element must be in H1/L2");
-
-   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
-
-   if (spaceH1)
-   {
-      dshape.SetSize(trial_dof,dim);
-      curlshape.SetSize(dim*trial_dof,1);
-      dimc = dim;
-   }
-   else
-   {
-      curlshape.SetSize(trial_dof,dimc);
-      elmat_comp.SetSize(test_dof, trial_dof);
-   }
-   elmat.SetSize(dimc * test_dof, trial_dof);
-   shape.SetSize(test_dof);
-   elmat = 0.0;
-
-   double c;
-   Vector d_col;
-   const IntegrationRule *ir = IntRule;
-
-   if (ir == NULL)
-   {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-   }
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint(&ip);
-      if (spaceH1)
-      {
-         trial_fe.CalcPhysDShape(Trans, dshape);
-         dshape.GradToCurl(curlshape);
-      }
-      else
-      {
-         trial_fe.CalcPhysCurlShape(Trans, curlshape);
-      }
-      test_fe.CalcPhysShape(Trans, shape);
-      c = ip.weight*Trans.Weight();
-      if (Q)
-      {
-         c *= Q->Eval(Trans, ip);
-      }
-      shape *= c;
-
-      for (int d = 0; d < dimc; ++d)
-      {
-         double * curldata = &(curlshape.GetData())[d*trial_dof];
-         for (int jj = 0; jj < trial_dof; ++jj)
-         {
-            for (int ii = 0; ii < test_dof; ++ii)
-            {
-               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
-            }
-         }
-      }
-   }
-}
-

 void VectorFEMassIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -2760,6 +2769,7 @@ void DivDivIntegrator::AssembleElementMatrix2(
 #endif
   elmat.SetSize(te_nd,tr_nd);

+
   const IntegrationRule *ir = IntRule;
   if (ir == NULL)
   {
@@ -3997,6 +4007,214 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
   }
 }

+void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                              const FiniteElement &trial_face_fe,
+                                              const FiniteElement &test_fe,
+                                              FaceElementTransformations & Trans,
+                                              DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof;
+   int order;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
+      {
+         order += Trans.OrderW();
+      }
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight;
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += scale * shape(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations &Trans,
+                                                    DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof, dim;
+   int order;
+
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE, "");
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+   dim = test_fe.GetDim();
+
+   face_shape.SetSize(face_ndof);
+   normal.SetSize(dim);
+   shape.SetSize(ndof,dim);
+   shape_n.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      trial_face_fe.CalcPhysShape(Trans, face_shape);
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*etrans, shape);
+      shape.Mult(normal, shape_n);
+      face_shape *= ip.weight;
+
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += scale * shape_n(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void TangentTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                     const FiniteElement &trial_face_fe,
+                                                     const FiniteElement &test_fe,
+                                                     FaceElementTransformations & Trans,
+                                                     DenseMatrix &elmat)
+{
+
+   MFEM_VERIFY(test_fe.GetMapType() == FiniteElement::H_CURL, "");
+
+   int face_ndof, ndof, dim;
+   int order;
+   dim = test_fe.GetDim();
+   if (dim == 3)
+   {
+      std::string msg =
+         "Trial space should be ND face trace and test space should be a ND vector field in 3D ";
+      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::H_CURL &&
+                  trial_face_fe.GetDim() == 2 && test_fe.GetDim() == 3, msg);
+   }
+   else
+   {
+      std::string msg =
+         "Trial space should be H1 edge trace and test space should be a ND vector field in 2D";
+      MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE &&
+                  trial_face_fe.GetDim() == 1 && test_fe.GetDim() == 2, msg);
+   }
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   int dimc = (dim == 3) ? 3 : 1;
+
+   face_shape.SetSize(face_ndof,dimc);
+   shape_n.SetSize(ndof,dimc);
+   shape.SetSize(ndof,dim);
+   normal.SetSize(dim);
+   DenseMatrix face_shape_n(face_ndof,dimc);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      if (dim == 3)
+      {
+         trial_face_fe.CalcVShape(Trans,face_shape);
+      }
+      else
+      {
+         face_shape.GetColumnReference(0,temp);
+         trial_face_fe.CalcPhysShape(Trans,temp);
+      }
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*eltrans, shape);
+
+      // rotate
+      cross_product(normal, shape, shape_n);
+
+      const double w = scale*ip.weight;
+      AddMult_a_ABt(w,shape_n, face_shape, elmat);
+   }
+}
+
+
+

 void NormalInterpolator::AssembleElementMatrix2(
   const FiniteElement &dom_fe, const FiniteElement &ran_fe,
@@ -151,6 +151,12 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

+   virtual void AssembleTraceFaceMatrix(int elem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+
   /** Abstract method used for assembling TraceFaceIntegrators in a
       MixedBilinearForm. */
   virtual void AssembleFaceMatrix(const FiniteElement &trial_face_fe,
@@ -215,10 +221,10 @@ public:
                             function by any coefficients describing the
                             integrator.
       @param[in] ir  If passed (the default value is NULL), the implementation
-                      of the method will ignore the integration rule provided
-                      by the @a fluxelem parameter and, instead, compute the
-                      discrete flux at the points specified by the integration
-                      rule @a ir.
+                        of the method will ignore the integration rule provided
+                        by the @a fluxelem parameter and, instead, compute the
+                        discrete flux at the points specified by the integration
+                        rule @a ir.
    */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
@@ -2069,6 +2075,31 @@ public:
                                         ElementTransformation &Trans);
 };

+/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is a
+    scalar coefficient, and v is a vector with components v_i in the L2 or H1 space.
+    u can be in H(curl) (2D or 3D) or it can be a scalar H1.
+    Note: If u is scalar H1 then curl u = [0 1; -1 0] grad u */
+class CurlIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+
+private:
+   Vector shape;
+   DenseMatrix dshape;
+   DenseMatrix curlshape;
+   DenseMatrix elmat_comp;
+public:
+   CurlIntegrator() : Q{NULL} { }
+   CurlIntegrator(Coefficient *q_) :  Q{q_} { }
+   CurlIntegrator(Coefficient &q) :  Q{&q} { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+};
+
 /** Class for integrating the bilinear form a(u,v) := (Q grad u, grad v) where Q
    can be a scalar or a matrix coefficient. */
 class DiffusionIntegrator: public BilinearFormIntegrator
@@ -2174,7 +2205,6 @@ public:
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
 class MassIntegrator: public BilinearFormIntegrator
 {
-   friend class DGMassInverse;
 protected:
 #ifndef MFEM_THREAD_SAFE
   Vector shape, te_shape;
@@ -2609,35 +2639,6 @@ public:
                                   const Vector &elfun);
 };

-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
-    an optional scalar coefficient, and v is a vector with components v_i in
-    the L2 or H1 space. This integrator handles 3 cases:
-    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
-    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
-    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
-        2D vector field with components v_i in L^2 or H^1 space.
-    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
-class MixedCurlIntegrator : public BilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-
-private:
-   Vector shape;
-   DenseMatrix dshape;
-   DenseMatrix curlshape;
-   DenseMatrix elmat_comp;
-public:
-   MixedCurlIntegrator() : Q{NULL} { }
-   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
-   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-};
-
 /** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
@@ -2779,12 +2780,10 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
-
   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
                                       const FiniteElement &test_fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-
   const Coefficient *GetCoefficient() const { return Q; }
 };

@@ -3301,6 +3300,88 @@ public:
                                   DenseMatrix &elmat);
 };

+
+/** Integrator for the DPG form: < v, w > over a face (the interface) where
+    the trial variable v is defined on the interface
+    (H^-1/2 i.e., v:=u⋅n normal trace of H(div))
+    and the test variable w is in an H1-conforming space. */
+class TraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+public:
+   TraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
+/** Integrator for the form: < v, w.n > over a face (the interface) where
+    the trial variable v is defined on the interface (H^1/2, i.e., trace of H1)
+    and the test variable w is in an H(div)-conforming space. */
+class NormalTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, normal, shape_n;
+   DenseMatrix shape;
+
+public:
+   NormalTraceIntegrator() { }
+   virtual void AssembleTraceFaceMatrix(int ielem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+};
+
+
+/** Integrator for the form: < v, w × n > over a face (the interface)
+ *  In 3D the trial variable v is defined on the interface (H^-1/2(curl), trace of H(curl))
+ *  In 2D it's defined on the interface (H^1/2, trace of H1)
+ *  The test variable w is in an H(curl)-conforming space. */
+class TangentTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   DenseMatrix face_shape, shape, shape_n;
+   Vector normal;
+   Vector temp;
+
+   void cross_product(const Vector & x, const DenseMatrix & Y, DenseMatrix & Z)
+   {
+      int dim = x.Size();
+      MFEM_VERIFY(Y.Width() == dim, "Size missmatch");
+      int dimc = dim == 3 ? dim : 1;
+      int h = Y.Height();
+      Z.SetSize(h,dimc);
+      if (dim == 3)
+      {
+         for (int i = 0; i<h; i++)
+         {
+            Z(i,0) = x(2) * Y(i,1) - x(1) * Y(i,2);
+            Z(i,1) = x(0) * Y(i,2) - x(2) * Y(i,0);
+            Z(i,2) = x(1) * Y(i,0) - x(0) * Y(i,1);
+         }
+      }
+      else
+      {
+         for (int i = 0; i<h; i++)
+         {
+            Z(i,0) = x(1) * Y(i,0) - x(0) * Y(i,1);
+         }
+      }
+   }
+
+public:
+   TangentTraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
 /** Abstract class to serve as a base for local interpolators to be used in the
    DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"

@@ -1409,10 +1408,66 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * ne, mt);
+   Vector vel;
+   if (VectorConstantCoefficient *cQ =
+          dynamic_cast<VectorConstantCoefficient*>(Q))
+   {
+      vel = cQ->GetVec();
+   }
+   else if (VectorGridFunctionCoefficient *vgfQ =
+               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
+   {
+      vel.SetSize(dim * nq * ne, mt);

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);
+      const GridFunction *gf = vgfQ->GetGridFunction();
+      const FiniteElementSpace &gf_fes = *gf->FESpace();
+      const QuadratureInterpolator *qi(gf_fes.GetQuadratureInterpolator(*ir));
+      const bool use_tensor_products = UsesTensorBasis(gf_fes);
+      const ElementDofOrdering ordering = use_tensor_products ?
+                                          ElementDofOrdering::LEXICOGRAPHIC :
+                                          ElementDofOrdering::NATIVE;
+      const Operator *R = gf_fes.GetElementRestriction(ordering);

+      Vector xe(R->Height(), mt);
+      xe.UseDevice(true);
+
+      R->Mult(*gf, xe);
+      qi->SetOutputLayout(QVectorLayout::byVDIM);
+      qi->DisableTensorProducts(!use_tensor_products);
+      qi->Values(xe,vel);
+   }
+   else if (VectorQuadratureFunctionCoefficient* vqfQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+
+      qFun.Read();
+      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      vel.SetSize(dim * nq * ne);
+      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
+      DenseMatrix MQ_ir;
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         Q->Eval(MQ_ir, T, *ir);
+         for (int q = 0; q < nq; ++q)
+         {
+            for (int i = 0; i < dim; ++i)
+            {
+               C(i,q,e) = MQ_ir(i,q);
+            }
+         }
+      }
+   }
   PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
                     vel, alpha, pa_data);
 }
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "restriction.hpp"

 using namespace std;
@@ -162,24 +161,88 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
-
-   FaceQuadratureSpace qs(*mesh, *ir, type);
-   CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);
-
-   CoefficientVector r(qs, CoefficientStorage::COMPRESSED);
-   if (rho == nullptr)
+   Vector vel;
+   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
+                                        (u))
   {
-      r.SetConstant(1.0);
+      vel = c_u->GetVec();
   }
-   else if (ConstantCoefficient *const_rho = dynamic_cast<ConstantCoefficient*>
-                                             (rho))
+   else if (VectorQuadratureFunctionCoefficient* qf_u =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
   {
-      r.SetConstant(const_rho->constant);
+      // Assumed to be in lexicographical ordering
+      const QuadratureFunction &qFun = qf_u->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      vel.SetSize(dim * nq * nf);
+      auto C = Reshape(vel.HostWrite(), dim, nq, nf);
+      Vector Vq(dim);
+      int f_ind = 0;
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
+      {
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (face.IsNonconformingCoarse())
+         {
+            // We skip nonconforming coarse faces as they are treated
+            // by the corresponding nonconforming fine faces.
+            continue;
+         }
+         else if ( face.IsOfFaceType(type) )
+         {
+            const int mask = FaceElementTransformations::HAVE_ELEM1 |
+                             FaceElementTransformations::HAVE_LOC1;
+            FaceElementTransformations &T =
+               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
+            for (int q = 0; q < nq; ++q)
+            {
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
+               T.SetAllIntPoints(&ir->IntPoint(q));
+               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+               u->Eval(Vq, *T.Elem1, eip1);
+               for (int i = 0; i < dim; ++i)
+               {
+                  C(i,iq,f_ind) = Vq(i);
+               }
+            }
+            f_ind++;
+         }
+      }
+      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+   }
+   Vector r;
+   if (rho==nullptr)
+   {
+      r.SetSize(1);
+      r(0) = 1.0;
+   }
+   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
+   {
+      r.SetSize(1);
+      r(0) = c_rho->constant;
   }
   else if (QuadratureFunctionCoefficient* qf_rho =
               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
   {
-      r.MakeRef(qf_rho->GetQuadFunction());
+      const QuadratureFunction &qFun = qf_rho->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
   }
   else
   {
@@ -191,42 +254,45 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
      {
         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse() || !face.IsOfFaceType(type))
+         if (face.IsNonconformingCoarse())
         {
            // We skip nonconforming coarse faces as they are treated
            // by the corresponding nonconforming fine faces.
            continue;
         }
-         FaceElementTransformations &T =
-            *fes.GetMesh()->GetFaceElementTransformations(f);
-         for (int q = 0; q < nq; ++q)
+         else if ( face.IsOfFaceType(type) )
         {
-            // Convert to lexicographic ordering
-            int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                   quad1D, q);
-
-            T.SetAllIntPoints(&ir->IntPoint(q));
-            const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-            const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-            double rq;
-
-            if (face.IsBoundary())
+            FaceElementTransformations &T =
+               *fes.GetMesh()->GetFaceElementTransformations(f);
+            for (int q = 0; q < nq; ++q)
            {
-               rq = rho->Eval(*T.Elem1, eip1);
-            }
-            else
-            {
-               double udotn = 0.0;
-               for (int d=0; d<dim; ++d)
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
+
+               T.SetAllIntPoints(&ir->IntPoint(q));
+               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
+               double rq;
+
+               if ( face.IsBoundary() )
               {
-                  udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+                  rq = rho->Eval(*T.Elem1, eip1);
               }
-               if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
-               else { rq = rho->Eval(*T.Elem1, eip1); }
+               else
+               {
+                  double udotn = 0.0;
+                  for (int d=0; d<dim; ++d)
+                  {
+                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+                  }
+                  if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
+                  else { rq = rho->Eval(*T.Elem1, eip1); }
+               }
+               C(iq,f_ind) = rq;
            }
-            C(iq,f_ind) = rq;
+            f_ind++;
         }
-         f_ind++;
      }
      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
   }
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;
@@ -391,21 +390,120 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
+   int coeffDim = 1;
+   Vector coeff;
+   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
+   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
+   {
+      MFEM_VERIFY(SMQ->GetSize() == dim, "");
+      coeffDim = symmDims;
+      coeff.SetSize(symmDims * nq * ne);

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
+      DenseSymmetricMatrix sym_mat;
+      sym_mat.SetSize(dim);

-   if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (VQ) { coeff.Project(*VQ); }
-   else if (Q) { coeff.Project(*Q); }
-   else { coeff.SetConstant(1.0); }
+      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);

-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dims*dims);
-   const int pa_size = symmetric ? symmDims : dims*dims;
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
+            int cnt = 0;
+            for (int i=0; i<dim; ++i)
+               for (int j=i; j<dim; ++j, ++cnt)
+               {
+                  C(cnt, p, e) = sym_mat(i,j);
+               }
+         }
+      }
+   }
+   else if (MQ)
+   {
+      symmetric = false;
+      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");

-   pa_data.SetSize(pa_size * nq * ne, mt);
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne, ir->GetWeights(),
+      coeffDim = MQfullDim;
+
+      coeff.SetSize(MQfullDim * nq * ne);
+
+      DenseMatrix mat;
+      mat.SetSize(dim);
+
+      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            MQ->Eval(mat, *tr, ir->IntPoint(p));
+            for (int i=0; i<dim; ++i)
+               for (int j=0; j<dim; ++j)
+               {
+                  C(j+(i*dim), p, e) = mat(i,j);
+               }
+         }
+      }
+   }
+   else if (VQ)
+   {
+      MFEM_VERIFY(VQ->GetVDim() == dim, "");
+      coeffDim = VQ->GetVDim();
+      coeff.SetSize(coeffDim * nq * ne);
+      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+      Vector DM(coeffDim);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            VQ->Eval(DM, *tr, ir->IntPoint(p));
+            for (int i=0; i<coeffDim; ++i)
+            {
+               C(i, p, e) = DM[i];
+            }
+         }
+      }
+   }
+   else if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
+   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                    geom->J, coeff, pa_data);
 }

@@ -1686,7 +1784,7 @@ static void PADiffusionApply(const int dim,
         case 0x77: return SmemPADiffusionApply2D<7,7,4>(NE,symm,B,G,D,X,Y);
         case 0x88: return SmemPADiffusionApply2D<8,8,2>(NE,symm,B,G,D,X,Y);
         case 0x99: return SmemPADiffusionApply2D<9,9,2>(NE,symm,B,G,D,X,Y);
-            // default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         default:   return PADiffusionApply2D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }

@@ -1704,14 +1802,7 @@ static void PADiffusionApply(const int dim,
         case 0x67: return SmemPADiffusionApply3D<6,7>(NE,symm,B,G,D,X,Y);
         case 0x78: return SmemPADiffusionApply3D<7,8>(NE,symm,B,G,D,X,Y);
         case 0x89: return SmemPADiffusionApply3D<8,9>(NE,symm,B,G,D,X,Y);
-         case 0x33: return SmemPADiffusionApply3D<3,3>(NE,symm,B,G,D,X,Y);
-         case 0x44: return SmemPADiffusionApply3D<4,4>(NE,symm,B,G,D,X,Y);
-         case 0x55: return SmemPADiffusionApply3D<5,5>(NE,symm,B,G,D,X,Y);
-         case 0x66: return SmemPADiffusionApply3D<6,6>(NE,symm,B,G,D,X,Y);
-         case 0x77: return SmemPADiffusionApply3D<7,7>(NE,symm,B,G,D,X,Y);
-         case 0x88: return SmemPADiffusionApply3D<8,8>(NE,symm,B,G,D,X,Y);
-         case 0x99: return SmemPADiffusionApply3D<9,9>(NE,symm,B,G,D,X,Y);
-            // default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
+         default:   return PADiffusionApply3D(NE,symm,B,G,Bt,Gt,D,X,Y,D1D,Q1D);
      }
   }
   MFEM_ABORT("Unknown kernel: 0x"<<std::hex << id << std::dec);
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"

 using namespace std;

@@ -210,8 +209,44 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
               "PA requires test and trial space to have same number of quadrature points!");
   pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   Vector coeff;
+
+   if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *trial_fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }

   PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
                   ne, ir->GetWeights(), geom->J, coeff, pa_data);
@@ -830,3 +865,4 @@ void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 }

 } // namespace mfem
+
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qspace.hpp"

 using namespace std;

@@ -968,6 +967,8 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dim = mesh->Dimension();
   MFEM_VERIFY(dim == 2 || dim == 3, "");

+   const int dimc = (dim == 3) ? 3 : 1;
+
   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -977,19 +978,88 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
-   if (Q) { coeff.Project(*Q); }
-   else if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
+   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);

-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dim*dim);
-   const int sym_dims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int ndata = (dim == 2) ? 1 : (symmetric ? sym_dims : dim*dim);
+   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+
+   symmetric = (SMQ || MQ == NULL);
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

+   Vector coeff(coeffDim * ne * nq);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ || MQ)
+   {
+      Vector DM(DQ ? coeffDim : 0);
+      DenseMatrix GM;
+      DenseSymmetricMatrix SM;
+
+      if (DQ)
+      {
+         MFEM_VERIFY(coeffDim == dimc, "");
+      }
+      if (SMQ)
+      {
+         SM.SetSize(dimc);
+         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
+      }
+      else if (MQ)
+      {
+         GM.SetSize(dimc);
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
+      }
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            if (SMQ)
+            {
+               SMQ->Eval(SM, *tr, ir->IntPoint(p));
+
+               int cnt = 0;
+               for (int i=0; i<dimc; ++i)
+                  for (int j=i; j<dimc; ++j, ++cnt)
+                  {
+                     coeffh(cnt, p, e) = SM(i,j);
+                  }
+
+            }
+            else if (MQ)
+            {
+               MQ->Eval(GM, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dimc; ++i)
+                  for (int j=0; j<dimc; ++j)
+                  {
+                     coeffh(j+(i*dimc), p, e) = GM(i,j);
+                  }
+
+            }
+            else if (DQ)
+            {
+               DQ->Eval(DM, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = DM[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }
+
   if (el->GetDerivType() != mfem::FiniteElement::CURL)
   {
      MFEM_ABORT("Unknown kernel.");
@@ -997,7 +1067,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   if (dim == 3)
   {
-      PACurlCurlSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J, coeff,
+      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
                        pa_data);
   }
   else
@@ -2710,7 +2780,7 @@ void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
   }
 }

-// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply3D(const int D1D,
                      const int Q1D,
@@ -2900,7 +2970,7 @@ void PAHcurlH1Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOFs in H(curl), integrated
+// Apply to x corresponding to DOF's in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose3D(const int D1D,
                               const int Q1D,
@@ -3099,7 +3169,7 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply2D(const int D1D,
                      const int Q1D,
@@ -3223,7 +3293,7 @@ void PAHcurlH1Apply2D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOFs in H(curl), integrated
+// Apply to x corresponding to DOF's in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose2D(const int D1D,
                               const int Q1D,
@@ -3419,8 +3489,20 @@ void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), nq, ne);
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeffh(p, e) = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }

   if (dim == 2)
   {
@@ -3511,11 +3593,38 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   const int ndata = curlSpaces ? (coeffDim == 1 ? 1 : 9) : symmDims;
   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::FULL);
-   if (Q) { coeff.Project(*Q); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
+   Vector coeff(coeffDim * nq * ne);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ)
+   {
+      Vector V(coeffDim);
+      if (DQ)
+      {
+         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
+      }
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+
+         for (int p=0; p<nq; ++p)
+         {
+            if (DQ)
+            {
+               DQ->Eval(V, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = V[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }

   if (testType == mfem::FiniteElement::CURL &&
       trialType == mfem::FiniteElement::CURL && dim == 3)
@@ -3543,7 +3652,7 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
 }

-// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3D(const int D1D,
@@ -3906,7 +4015,7 @@ static void PAHcurlL2Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void SmemPAHcurlL2Apply3D(const int D1D,
@@ -4216,7 +4325,7 @@ static void SmemPAHcurlL2Apply3D(const int D1D,
   ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }

-// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3D(const int D1D,
@@ -4572,7 +4681,7 @@ static void PAHcurlHdivApply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOFs in H(div) (test), integrated against the
+// Apply to x corresponding to DOF's in H(div) (test), integrated against the
 // curl of H(curl) trial functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3DTranspose(const int D1D,
@@ -5037,11 +5146,38 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace

   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::FULL);
-   if (Q) { coeff.Project(*Q); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
+   Vector coeff(coeffDim * nq * ne);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ)
+   {
+      Vector V(coeffDim);
+      if (DQ)
+      {
+         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
+      }
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+
+         for (int p=0; p<nq; ++p)
+         {
+            if (DQ)
+            {
+               DQ->Eval(V, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = V[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }

   if (trialType == mfem::FiniteElement::CURL && dim == 3)
   {
@@ -5067,7 +5203,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
   }
 }

-// Apply to x corresponding to DOFs in H(curl) (trial), integrated against curl
+// Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3DTranspose(const int D1D,
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qspace.hpp"

 using namespace std;

@@ -1514,8 +1513,19 @@ void DivDivIntegrator::AssemblePA(const FiniteElementSpace &fes)

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }

   if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
   {
@@ -1773,8 +1783,19 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }

   if (test_el->GetMapType() == FiniteElement::INTEGRAL)
   {
@@ -1797,7 +1818,7 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
 }

-// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply3D(const int D1D,
                            const int Q1D,
@@ -1960,7 +1981,7 @@ static void PAHdivL2Apply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply2D(const int D1D,
                            const int Q1D,
@@ -12,9 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/mass/mass.hpp"
-#include "bilininteg_mass_pa.hpp"

 using namespace std;

@@ -62,10 +60,43 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(ne*nq, mt);
+   Vector coeff;
+   if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
   if (dim==2)
   {
@@ -559,18 +590,85 @@ static void PAMassApply2D(const int NE,
                          const int d1d = 0,
                          const int q1d = 0)
 {
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b_.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto X = Reshape(x_.Read(), D1D, D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL(e, NE,
   {
-      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double sol_xy[max_Q1D][max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double sol_x[max_Q1D];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            sol_x[qy] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s = X(dx,dy,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] += B(qx,dx)* s;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double d2q = B(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] += d2q * sol_x[qx];
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] *= D(qx,qy,e);
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double sol_x[max_D1D];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] = 0.0;
+         }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double s = sol_xy[qy][qx];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] += Bt(dx,qx) * s;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            const double q2d = Bt(dy,qy);
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx,dy,e) += q2d * sol_x[dx];
+            }
+         }
+      }
   });
 }

@@ -592,13 +690,108 @@ static void SmemPAMassApply2D(const int NE,
   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
   MFEM_VERIFY(D1D <= MD1, "");
   MFEM_VERIFY(Q1D <= MQ1, "");
-   const auto b = b_.Read();
-   const auto D = d_.Read();
-   const auto x = x_.Read();
-   auto Y = y_.ReadWrite();
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
-      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      MFEM_SHARED double BBt[MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) BBt;
+      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
+      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
+      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
+      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
+      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
+      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
+      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            X[dy][dx] = x(dx,dy,e);
+         }
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][dy] = b(q,dy);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double dq = 0.0;
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               dq += X[dy][dx] * B[qx][dx];
+            }
+            DQ[dy][qx] = dq;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double qq = 0.0;
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               qq += DQ[dy][qx] * B[qy][dy];
+            }
+            QQ[qy][qx] = qq * D(qx, qy, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bt[dy][q] = b(q,dy);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double dq = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dq += QQ[qy][qx] * Bt[dx][qx];
+            }
+            QD[qy][dx] = dq;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double dd = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               dd += (QD[qy][dx] * Bt[dy][qy]);
+            }
+            Y(dx, dy, e) += dd;
+         }
+      }
   });
 }

@@ -612,18 +805,134 @@ static void PAMassApply3D(const int NE,
                          const int d1d = 0,
                          const int q1d = 0)
 {
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b_.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL(e, NE,
   {
-      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         double sol_xy[max_Q1D][max_Q1D];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            double sol_x[max_Q1D];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] = 0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double s = X(dx,dy,dz,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_x[qx] += B(qx,dx) * s;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy = B(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xy[qy][qx] += wy * sol_x[qx];
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double wz = B(qz,dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double sol_xy[max_D1D][max_D1D];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_xy[dy][dx] = 0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double sol_x[max_D1D];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] = 0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double s = sol_xyz[qz][qy][qx];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_x[dx] += Bt(dx,qx) * s;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double wy = Bt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_xy[dy][dx] += wy * sol_x[dx];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            const double wz = Bt(dz,qz);
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
+               }
+            }
+         }
+      }
   });
 }

@@ -644,13 +953,213 @@ static void SmemPAMassApply3D(const int NE,
   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
   MFEM_VERIFY(D1D <= M1D, "");
   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = b_.Read();
-   auto d = d_.Read();
-   auto x = x_.Read();
-   auto y = y_.ReadWrite();
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
   {
-      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      MFEM_SHARED double sDQ[MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) sDQ;
+      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
+      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
+      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
+      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
+      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
+      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
+      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
+      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               X[dz][dy][dx] = x(dx,dy,dz,e);
+            }
+         }
+         MFEM_FOREACH_THREAD(dx,x,Q1D)
+         {
+            B[dx][dy] = b(dx,dy);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += X[dz][dy][dx] * B[qx][dx];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ[dz][dy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; dz++)
+               {
+                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ[dz][qy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; qz++)
+               {
+                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[d][q] = b(q,d);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD[qz][qy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD[qz][dy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               y(dx,dy,dz,e) += u[dz];
+            }
+         }
+      }
   });
 }

@@ -1,632 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_MASS_PA_HPP
-#define MFEM_BILININTEG_MASS_PA_HPP
-
-#include "../config/config.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-
-namespace mfem
-{
-
-namespace internal
-{
-
-template <bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void PAMassApply2D_Element(const int e,
-                           const int NE,
-                           const double *b_,
-                           const double *bt_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_,
-                           const int d1d = 0,
-                           const int q1d = 0)
-{
-   const int D1D = d1d;
-   const int Q1D = q1d;
-   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
-   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
-   auto X = ConstDeviceCube(x_, D1D, D1D, NE);
-   auto Y = DeviceCube(y_, D1D, D1D, NE);
-
-   if (!ACCUMULATE)
-   {
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            Y(dx, dy, e) = 0.0;
-         }
-      }
-   }
-
-   constexpr int max_D1D = MAX_D1D;
-   constexpr int max_Q1D = MAX_Q1D;
-   double sol_xy[max_Q1D][max_Q1D];
-   for (int qy = 0; qy < Q1D; ++qy)
-   {
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         sol_xy[qy][qx] = 0.0;
-      }
-   }
-   for (int dy = 0; dy < D1D; ++dy)
-   {
-      double sol_x[max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         sol_x[qy] = 0.0;
-      }
-      for (int dx = 0; dx < D1D; ++dx)
-      {
-         const double s = X(dx,dy,e);
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_x[qx] += B(qx,dx)* s;
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         const double d2q = B(qy,dy);
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] += d2q * sol_x[qx];
-         }
-      }
-   }
-   for (int qy = 0; qy < Q1D; ++qy)
-   {
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         sol_xy[qy][qx] *= D(qx,qy,e);
-      }
-   }
-   for (int qy = 0; qy < Q1D; ++qy)
-   {
-      double sol_x[max_D1D];
-      for (int dx = 0; dx < D1D; ++dx)
-      {
-         sol_x[dx] = 0.0;
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         const double s = sol_xy[qy][qx];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] += Bt(dx,qx) * s;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         const double q2d = Bt(dy,qy);
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            Y(dx,dy,e) += q2d * sol_x[dx];
-         }
-      }
-   }
-}
-
-template<int T_D1D, int T_Q1D, int T_NBZ, bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void SmemPAMassApply2D_Element(const int e,
-                               const int NE,
-                               const double *b_,
-                               const double *d_,
-                               const double *x_,
-                               double *y_,
-                               int d1d = 0,
-                               int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
-   auto x = ConstDeviceCube(x_, D1D, D1D, NE);
-   auto Y = DeviceCube(y_, D1D, D1D, NE);
-
-   const int tidz = MFEM_THREAD_ID(z);
-
-   MFEM_SHARED double BBt[MQ1*MD1];
-   double (*B)[MD1] = (double (*)[MD1]) BBt;
-   double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-   MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-   MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-   double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-   double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-   double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-   double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-
-
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         X[dy][dx] = x(dx,dy,e);
-      }
-   }
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            B[q][dy] = b(q,dy);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double dq = 0.0;
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            dq += X[dy][dx] * B[qx][dx];
-         }
-         DQ[dy][qx] = dq;
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double qq = 0.0;
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            qq += DQ[dy][qx] * B[qy][dy];
-         }
-         QQ[qy][qx] = qq * D(qx, qy, e);
-      }
-   }
-   MFEM_SYNC_THREAD;
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[dy][q] = b(q,dy);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double dq = 0.0;
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            dq += QQ[qy][qx] * Bt[dx][qx];
-         }
-         QD[qy][dx] = dq;
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double dd = 0.0;
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            dd += (QD[qy][dx] * Bt[dy][qy]);
-         }
-         if (ACCUMULATE)
-         {
-            Y(dx, dy, e) += dd;
-         }
-         else
-         {
-            Y(dx, dy, e) = dd;
-         }
-      }
-   }
-}
-
-template <bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void PAMassApply3D_Element(const int e,
-                           const int NE,
-                           const double *b_,
-                           const double *bt_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_,
-                           const int d1d,
-                           const int q1d)
-{
-   const int D1D = d1d;
-   const int Q1D = q1d;
-   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
-   auto D = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
-   auto X = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
-   auto Y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
-
-   if (!ACCUMULATE)
-   {
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx, dy, dz, e) = 0.0;
-            }
-         }
-      }
-   }
-
-   constexpr int max_D1D = MAX_D1D;
-   constexpr int max_Q1D = MAX_Q1D;
-   double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-   for (int qz = 0; qz < Q1D; ++qz)
-   {
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xyz[qz][qy][qx] = 0.0;
-         }
-      }
-   }
-   for (int dz = 0; dz < D1D; ++dz)
-   {
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_x[qx] = 0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = X(dx,dy,dz,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx) * s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double wy = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += wy * sol_x[qx];
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         const double wz = B(qz,dz);
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-            }
-         }
-      }
-   }
-   for (int qz = 0; qz < Q1D; ++qz)
-   {
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
-         }
-      }
-   }
-   for (int qz = 0; qz < Q1D; ++qz)
-   {
-      double sol_xy[max_D1D][max_D1D];
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_xy[dy][dx] = 0;
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xyz[qz][qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double wy = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] += wy * sol_x[dx];
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         const double wz = Bt(dz,qz);
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-            }
-         }
-      }
-   }
-}
-
-template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void SmemPAMassApply3D_Element(const int e,
-                               const int NE,
-                               const double *b_,
-                               const double *d_,
-                               const double *x_,
-                               double *y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
-{
-   constexpr int D1D = T_D1D ? T_D1D : d1d;
-   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto d = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
-   auto x = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
-   auto y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
-
-   MFEM_SHARED double sDQ[MQ1*MD1];
-   double (*B)[MD1] = (double (*)[MD1]) sDQ;
-   double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-   MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-   MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-   double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-   double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-   double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-   double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-   double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-   double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            X[dz][dy][dx] = x(dx,dy,dz,e);
-         }
-      }
-      MFEM_FOREACH_THREAD(dx,x,Q1D)
-      {
-         B[dx][dy] = b(dx,dy);
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double u[D1D];
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; dz++)
-         {
-            u[dz] = 0;
-         }
-         MFEM_UNROLL(MD1)
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] += X[dz][dy][dx] * B[qx][dx];
-            }
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            DDQ[dz][dy][qx] = u[dz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double u[D1D];
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; dz++)
-         {
-            u[dz] = 0;
-         }
-         MFEM_UNROLL(MD1)
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-            }
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; dz++)
-         {
-            DQQ[dz][qy][qx] = u[dz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double u[Q1D];
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; qz++)
-         {
-            u[qz] = 0;
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-            }
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; qz++)
-         {
-            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(di,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(q,x,Q1D)
-      {
-         Bt[di][q] = b(q,di);
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u[Q1D];
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            u[qz] = 0;
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-            }
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            QQD[qz][qy][dx] = u[qz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u[Q1D];
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            u[qz] = 0;
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-            }
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            QDD[qz][dy][dx] = u[qz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u[D1D];
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            u[dz] = 0;
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-            }
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            if (ACCUMULATE)
-            {
-               y(dx,dy,dz,e) += u[dz];
-            }
-            else
-            {
-               y(dx,dy,dz,e) = u[dz];
-            }
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-} // namespace internal
-
-} // namespace mfem
-
-#endif
@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;
@@ -176,9 +175,43 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(!VQ && !MQ,
               "Only scalar coefficient supported for partial assembly for VectorDiffusionIntegrator");
+   Vector coeff;
+   if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto Co = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            Co(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }

   const Array<double> &w = ir->GetWeights();
   const Vector &j = geom->J;
@@ -11,7 +11,6 @@

 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
-#include "qspace.hpp"
 #include "gridfunc.hpp"

 namespace mfem
@@ -794,63 +793,140 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   trial_fetype = trial_el->GetDerivType();
   test_fetype = test_el->GetDerivType();

+   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+
+   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+
+   symmetric = (SMQ || MQ == NULL);
+
   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
-   if (Q) { coeff.Project(*Q); }
-   else if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
-
-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dim*dim);
-
   if ((trial_curl && test_div) || (trial_div && test_curl))
-      pa_data.SetSize((coeff_dim == 1 ? 1 : dim*dim) * nq * ne,
+      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
                      Device::GetMemoryType());
   else
-      pa_data.SetSize((symmetric ? symmDims : dims*dims) * nq * ne,
+      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
                      Device::GetMemoryType());

+   Vector coeff;
+
+   auto *qf_c = dynamic_cast<QuadratureFunctionCoefficient*>(Q);
+   if (qf_c)
+   {
+      const QuadratureFunction &qf = qf_c->GetQuadFunction();
+      qf.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qf), 0);
+   }
+   else
+   {
+      coeff.SetSize(coeffDim * ne * nq);
+      coeff = 1.0;
+      auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+      if (Q || DQ || MQ)
+      {
+         Vector DM(DQ ? coeffDim : 0);
+         DenseMatrix M;
+         DenseSymmetricMatrix SM;
+
+         if (DQ)
+         {
+            MFEM_VERIFY(coeffDim == dim, "");
+         }
+         if (SMQ)
+         {
+            MFEM_VERIFY(SMQ->GetSize() == dim, "");
+            SM.SetSize(dim);
+         }
+         else if (MQ)
+         {
+            MFEM_VERIFY(coeffDim == MQdim, "");
+            MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+            M.SetSize(dim);
+         }
+
+         for (int e=0; e<ne; ++e)
+         {
+            ElementTransformation *tr = mesh->GetElementTransformation(e);
+            for (int p=0; p<nq; ++p)
+            {
+               if (SMQ)
+               {
+                  SMQ->Eval(SM, *tr, ir->IntPoint(p));
+                  int cnt = 0;
+                  for (int i=0; i<dim; ++i)
+                     for (int j=i; j<dim; ++j, ++cnt)
+                     {
+                        coeffh(cnt, p, e) = SM(i,j);
+                     }
+               }
+               else if (MQ)
+               {
+                  MQ->Eval(M, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<dim; ++i)
+                     for (int j=0; j<dim; ++j)
+                     {
+                        coeffh(j+(i*dim), p, e) = M(i,j);
+                     }
+               }
+               else if (DQ)
+               {
+                  DQ->Eval(DM, *tr, ir->IntPoint(p));
+                  for (int i=0; i<coeffDim; ++i)
+                  {
+                     coeffh(i, p, e) = DM[i];
+                  }
+               }
+               else
+               {
+                  coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+               }
+            }
+         }
+      }
+   }
+
   if (trial_curl && test_curl && dim == 3)
   {
-      PADiffusionSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                         coeff, pa_data);
   }
   else if (trial_curl && test_curl && dim == 2)
   {
-      PADiffusionSetup2D<2>(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                            coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 3)
   {
-      PAHdivSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 2)
   {
-      PAHdivSetup2D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
            test_fel->GetOrder() == trial_fel->GetOrder())
   {
-      if (coeff_dim == 1)
+      if (coeffDim == 1)
      {
-         PAHcurlL2Setup(nq, coeff_dim, ne, ir->GetWeights(), coeff, pa_data);
+         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
      }
      else
      {
         const bool tr = (trial_div && test_curl);
         if (dim == 3)
-            PAHcurlHdivSetup3D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
                               geom->J, coeff, pa_data);
         else
-            PAHcurlHdivSetup2D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
                               geom->J, coeff, pa_data);
      }
   }
@@ -1092,8 +1168,19 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace

   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());

-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }

   // Use the same setup functions as VectorFEMassIntegrator.
   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
@@ -0,0 +1,512 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+BlockBilinearForm::BlockBilinearForm(Array<FiniteElementSpace *> & fespaces_) :
+   Matrix(0), fespaces(fespaces_)
+{
+   height = 0;
+   nblocks = fespaces.Size();
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fespaces[i]->GetVSize();
+      tdof_offsets[i+1] = fespaces[i]->GetTrueVSize();
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+   height = dof_offsets[nblocks];
+   width = height;
+   mat = mat_e = NULL;
+   extern_bfs = 0;
+   element_matrices = NULL;
+   diag_policy = DIAG_KEEP;
+}
+
+
+// Allocate appropriate SparseMatrix and assign it to mat
+void BlockBilinearForm::AllocMat()
+{
+   mat = new SparseMatrix(height);
+}
+
+void BlockBilinearForm::BuildProlongation()
+{
+   P = new BlockMatrix(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   for (int i = 0; i<nblocks; i++)
+   {
+      const SparseMatrix *P_ = fespaces[i]->GetConformingProlongation();
+      const SparseMatrix *R_ = fespaces[i]->GetRestrictionMatrix();
+      P->SetBlock(i,i,const_cast<SparseMatrix*>(P_));
+      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void BlockBilinearForm::ConformingAssemble()
+{
+   Finalize(0);
+   MFEM_ASSERT(mat, "the BilinearForm is not assembled");
+
+   if (!P) { BuildProlongation(); }
+
+   SparseMatrix * Pm = P->CreateMonolithic();
+
+   SparseMatrix *Pt = Transpose(*Pm);
+
+   SparseMatrix *PtA = mfem::Mult(*Pt, *mat);
+   delete mat;
+   if (mat_e)
+   {
+      SparseMatrix *PtAe = mfem::Mult(*Pt, *mat_e);
+      delete mat_e;
+      mat_e = PtAe;
+   }
+   delete Pt;
+   mat = mfem::Mult(*PtA, *Pm);
+   delete PtA;
+   if (mat_e)
+   {
+      SparseMatrix *PtAeP = mfem::Mult(*mat_e, *Pm);
+      delete mat_e;
+      mat_e = PtAeP;
+   }
+   delete Pm;
+   height = mat->Height();
+   width = mat->Width();
+}
+
+void BlockBilinearForm::Mult(const Vector &x, Vector &y) const
+{
+   // TODO
+}
+
+
+double& BlockBilinearForm::Elem (int i, int j)
+{
+   return mat -> Elem(i,j);
+}
+
+const double& BlockBilinearForm::Elem (int i, int j) const
+{
+   return mat -> Elem(i,j);
+}
+
+MatrixInverse * BlockBilinearForm::Inverse() const
+{
+   return mat -> Inverse();
+}
+
+void BlockBilinearForm::Finalize(int skip_zeros)
+{
+   mat->Finalize(skip_zeros);
+   if (mat_e) { mat_e->Finalize(skip_zeros); }
+}
+
+/// Adds new Block Domain Integrator. Assumes ownership of @a bfi.
+void BlockBilinearForm::AddDomainIntegrator(BlockBilinearFormIntegrator *bfi)
+{
+   domain_integs.Append(bfi);
+}
+
+/// Assembles the form i.e. sums over all domain integrators.
+void BlockBilinearForm::Assemble(int skip_zeros)
+{
+   ElementTransformation *eltrans;
+   DofTransformation * doftrans_j, *doftrans_k;
+   Mesh *mesh = fespaces[0] -> GetMesh();
+   DenseMatrix elmat, *elmat_p;
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   Array<int> vdofs_j, vdofs_k;
+   Array<int> offsetvdofs_j;
+   Array<int> elementblockoffsets(nblocks+1);
+   elementblockoffsets[0] = 0;
+   Array<int> blockoffsets(nblocks+1);
+   blockoffsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      blockoffsets[i+1] = fespaces[i]->GetVSize();
+   }
+   blockoffsets.PartialSum();
+   // mfem::out << "blockoffsets = " ; blockoffsets.Print();
+
+   if (mat == NULL)
+   {
+      AllocMat();
+   }
+
+   if (domain_integs.Size())
+   {
+      // loop through elements
+      for (int i = 0; i < mesh -> GetNE(); i++)
+      {
+         if (element_matrices)
+         {
+            elmat_p = &(*element_matrices)(i);
+         }
+         else
+         {
+            elmat.SetSize(0);
+            for (int k = 0; k < domain_integs.Size(); k++)
+            {
+               for (int j = 0; j<nblocks; j++)
+               {
+                  fe[j] = fespaces[j]->GetFE(i);
+                  elementblockoffsets[j+1] = fe[j]->GetDof();
+               }
+               elementblockoffsets.PartialSum();
+               eltrans = mesh->GetElementTransformation(i);
+               domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+               if (elmat.Size() == 0)
+               {
+                  elmat = elemmat;
+               }
+               else
+               {
+                  elmat += elemmat;
+               }
+            }
+         }
+         if (elmat.Size() == 0)
+         {
+            continue;
+         }
+         else
+         {
+            elmat_p = &elmat;
+         }
+         vdofs.SetSize(0);
+         for (int j = 0; j<nblocks; j++)
+         {
+            doftrans_j = fespaces[j]->GetElementVDofs(i, vdofs_j);
+            int jbeg = elementblockoffsets[j];
+            int jend = elementblockoffsets[j+1]-1;
+            int offset_j = blockoffsets[j];
+            offsetvdofs_j.SetSize(vdofs_j.Size());
+
+            for (int l = 0; l<vdofs_j.Size(); l++)
+            {
+               offsetvdofs_j[l] = vdofs_j[l]<0 ? -offset_j + vdofs_j[l]
+                                  :  offset_j + vdofs_j[l];
+            }
+            vdofs.Append(offsetvdofs_j);
+            for (int k = 0; k<nblocks; k++)
+            {
+               doftrans_k = fespaces[k]->GetElementVDofs(i, vdofs_k);
+               if (doftrans_k || doftrans_j)
+               {
+                  int kbeg = elementblockoffsets[k];
+                  int kend = elementblockoffsets[k+1]-1;
+                  DenseMatrix A;
+                  elmat_p->GetSubMatrix(jbeg,jend,kbeg, kend, A);
+                  TransformDual(doftrans_j, doftrans_k, A);
+                  elmat_p->SetSubMatrix(jbeg,kbeg,A);
+               }
+            }
+         }
+         mat->AddSubMatrix(vdofs,vdofs,*elmat_p, skip_zeros);
+      }
+   }
+
+}
+
+
+
+void BlockBilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                         Vector &x,
+                                         Vector &b, OperatorHandle &A, Vector &X,
+                                         Vector &B, int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   if (!P)
+   {
+      EliminateVDofsInRHS(ess_tdof_list, x, b);
+      X.MakeRef(x, 0, x.Size());
+      B.MakeRef(b, 0, b.Size());
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+   else // non conforming space
+   {
+      B.SetSize(P->Width());
+      P->MultTranspose(b, B);
+      X.SetSize(R->Height());
+
+      mfem::out << "R height, width  = " << R->Height() <<" x "<< R->Width() <<
+                std::endl;
+
+      R->Mult(x, X);
+      EliminateVDofsInRHS(ess_tdof_list, X, B);
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+
+}
+
+void BlockBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                         OperatorHandle &A)
+{
+   if (!mat_e)
+   {
+      const SparseMatrix *P_ = fespaces[0]->GetConformingProlongation();
+      if (P_) { ConformingAssemble(); }
+      EliminateVDofs(ess_tdof_list, diag_policy);
+      const int remove_zeros = 0;
+      Finalize(remove_zeros);
+   }
+   A.Reset(mat, false);
+}
+
+void BlockBilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
+                                           Vector &x)
+{
+   if (!P)
+   {
+      x.SyncMemory(X);
+   }
+   else
+   {
+      // Apply conforming prolongation
+      x.SetSize(P->Height());
+      P->Mult(X, x);
+   }
+}
+
+
+
+void BlockBilinearForm::ComputeElementMatrices()
+{
+   MFEM_ABORT("BlockBilinearForm::ComputeElementMatrices:not implemented yet")
+}
+
+void BlockBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
+{
+   if (element_matrices)
+   {
+      elmat.SetSize(element_matrices->SizeI(), element_matrices->SizeJ());
+      elmat = element_matrices->GetData(i);
+      return;
+   }
+
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   ElementTransformation *eltrans;
+
+   elmat.SetSize(0);
+   if (domain_integs.Size())
+   {
+      for (int j = 0; j<nblocks; j++)
+      {
+         fe[j] = fespaces[j]->GetFE(i);
+      }
+      eltrans = fespaces[0]->GetElementTransformation(i);
+      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
+      {
+         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         elmat += elemmat;
+      }
+   }
+   else
+   {
+      int matsize = 0;
+      for (int j = 0; j<nblocks; j++)
+      {
+         matsize += fespaces[j]->GetFE(i)->GetDof();
+      }
+      elmat.SetSize(matsize);
+      elmat = 0.0;
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                                             const Vector &sol, Vector &rhs,
+                                             DiagonalPolicy dpolicy)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofs(ess_dofs, sol, rhs, dpolicy);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofs(conf_ess_dofs, sol, rhs, dpolicy);
+   // }
+}
+
+void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                                             DiagonalPolicy dpolicy)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofs(ess_dofs, dpolicy);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofs(conf_ess_dofs, dpolicy);
+   // }
+}
+
+void BlockBilinearForm::EliminateEssentialBCDiag (const Array<int>
+                                                  &bdr_attr_is_ess,
+                                                  double value)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBCDiag: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofsDiag(ess_dofs, value);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofsDiag(conf_ess_dofs, value);
+   // }
+}
+
+void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
+                                       const Vector &sol, Vector &rhs,
+                                       DiagonalPolicy dpolicy)
+{
+   vdofs.HostRead();
+   for (int i = 0; i < vdofs.Size(); i++)
+   {
+      int vdof = vdofs[i];
+      if ( vdof >= 0 )
+      {
+         mat -> EliminateRowCol (vdof, sol(vdof), rhs, dpolicy);
+      }
+      else
+      {
+         mat -> EliminateRowCol (-1-vdof, sol(-1-vdof), rhs, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
+                                       DiagonalPolicy dpolicy)
+{
+   if (mat_e == NULL)
+   {
+      mat_e = new SparseMatrix(height);
+   }
+
+   // mat -> EliminateCols(vdofs, *mat_e,)
+
+   for (int i = 0; i < vdofs.Size(); i++)
+   {
+      int vdof = vdofs[i];
+      if ( vdof >= 0 )
+      {
+         mat -> EliminateRowCol (vdof, *mat_e, dpolicy);
+      }
+      else
+      {
+         mat -> EliminateRowCol (-1-vdof, *mat_e, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofs(
+   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
+   DiagonalPolicy dpolicy)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
+   MFEM_ASSERT(rhs.Size() == height, "incorrect rhs Vector size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowCol (i, sol(i), rhs, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofs (const Array<int>
+                                                      &ess_dofs,
+                                                      DiagonalPolicy dpolicy)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowCol (i, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofsDiag (
+   const Array<int> &ess_dofs,
+   double value)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowColDiag (i, value);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x, Vector &b)
+{
+   mat_e->AddMult(x, b, -1.);
+   mat->PartMult(vdofs, x, b);
+}
+
+
+
+BlockBilinearForm::~BlockBilinearForm()
+{
+   delete mat_e;
+   delete mat;
+   delete element_matrices;
+
+   for (int k=0; k < domain_integs.Size(); k++)
+   {
+      delete domain_integs[k];
+   }
+   for (int k=0; k < trace_integs.Size(); k++)
+   {
+      delete trace_integs[k];
+   }
+   delete P;
+   delete R;
+}
+
+
+
+} // namespace mfem
@@ -0,0 +1,288 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKBILINEARFORM
+#define MFEM_BLOCKBILINEARFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+
+namespace mfem
+{
+
+/** @brief A "square matrix" operator for the associated FE space and
+    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
+    M.  */
+class BlockBilinearForm : public Matrix
+{
+
+protected:
+
+   int nblocks;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
+   SparseMatrix *mat;
+
+   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
+        from the b.c.  Owned.
+       \f$ M + M_e = M_{original} \f$ */
+   SparseMatrix *mat_e;
+
+   /// FE spaces on which the block form lives. Not owned.
+   Array<FiniteElementSpace * > fespaces;
+
+   /** @brief Indicates the Mesh::sequence corresponding to the current state of
+      the BilinearForm. */
+   long sequence;
+
+   /** @brief Indicates the BlockBilinearFormIntegrator%s stored in #domain_integs,
+     are owned by another BlockBilinearForm. */
+   int extern_bfs;
+
+   /// Set of Domain Integrators to be applied.
+   Array<BlockBilinearFormIntegrator * > domain_integs;
+
+   /// Trace integrators.
+   Array<BlockBilinearFormIntegrator * > trace_integs;
+
+   DenseMatrix elemmat;
+   Array<int>  vdofs;
+
+   DenseTensor *element_matrices; ///< Owned.
+
+   BlockMatrix * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+   /** This data member allows one to specify what should be done to the
+    diagonal matrix entries and corresponding RHS values upon elimination of
+    the constrained DoFs. */
+   DiagonalPolicy diag_policy;
+
+   // Allocate appropriate SparseMatrix and assign it to mat
+   void AllocMat();
+
+   void ConformingAssemble();
+
+   void BuildProlongation();
+
+
+private:
+
+public:
+
+   /// Creates bilinear form associated with FE spaces @a *fespaces.
+   BlockBilinearForm(Array<FiniteElementSpace * > & fespaces_);
+
+   /// Get the size of the BilinearForm as a square matrix.
+   int Size() const { return height; }
+
+
+   /// Pre-allocate the internal SparseMatrix before assembly.
+   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
+
+   /// Returns a reference to: \f$ M_{ij} \f$
+   const double &operator()(int i, int j) { return (*mat)(i,j); }
+
+
+   /// Matrix vector multiplication:  \f$ y = M x \f$
+   virtual void Mult(const Vector &x, Vector &y) const;
+
+   /** @brief Matrix vector multiplication with the original uneliminated
+       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
+       \f$ y = M x + M_e x \f$ */
+   void FullMult(const Vector &x, Vector &y) const
+   { mat->Mult(x, y); mat_e->AddMult(x, y); }
+
+   virtual double &Elem(int i, int j);
+   virtual const double &Elem(int i, int j) const;
+   virtual MatrixInverse *Inverse() const;
+
+   /// Finalizes the matrix initialization.
+   virtual void Finalize(int skip_zeros = 1);
+
+   /// Returns a const reference to the sparse matrix.
+   const SparseMatrix &SpMat() const
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
+   SparseMatrix &SpMat()
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   const SparseMatrix &SpMatElim() const
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   SparseMatrix &SpMatElim()
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
+   void AddDomainIntegrator(BlockBilinearFormIntegrator *bfi);
+
+   /// Adds new Trace Integrator. Assumes ownership of @a bfi.
+   void AddTraceIntegrator(BlockBilinearFormIntegrator *bfi);
+
+   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
+   void operator=(const double a)
+   {
+      if (mat != NULL) { *mat = a; }
+      if (mat_e != NULL) { *mat_e = a; }
+   }
+
+   /// Assembles the form i.e. sums over all domain integrators.
+   void Assemble(int skip_zeros = 1);
+
+
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                                 Vector &b, OperatorHandle &A, Vector &X,
+                                 Vector &B, int copy_interior = 0);
+
+   /** @brief Form the linear system A X = B, corresponding to this bilinear
+       form and the linear form @a b(.). */
+   /** Version of the method FormLinearSystem() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OpType &A, Vector &X, Vector &B,
+                         int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
+   /// Form the linear system matrix A, see FormLinearSystem() for details.
+   /** Version of the method FormSystemMatrix() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
+
+
+   void ComputeElementMatrices();
+
+   /// Free the memory used by the element matrices.
+   void FreeElementMatrices()
+   { delete element_matrices; element_matrices = NULL; }
+
+   /// Compute the element matrix of the given element
+   /** The element matrix is computed by calling the domain integrators
+       or the one stored internally by a prior call of ComputeElementMatrices()
+       is returned when available.
+   */
+   void ComputeElementMatrix(int i, DenseMatrix &elmat);
+
+   /// Eliminate essential boundary DOFs from the system.
+   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
+       the essential part of the boundary. By default, the diagonal at the
+       essential DOFs is set to 1.0. This behavior is controlled by the argument
+       @a dpolicy. */
+   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                             const Vector &sol, Vector &rhs,
+                             DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /// Eliminate essential boundary DOFs from the system matrix.
+   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                             DiagonalPolicy dpolicy = DIAG_ONE);
+   /// Perform elimination and set the diagonal entry to the given value
+   void EliminateEssentialBCDiag(const Array<int> &bdr_attr_is_ess,
+                                 double value);
+
+   /// Eliminate the given @a vdofs.
+   /** NOTE: here, @a vdofs is a list of DOFs from all the fespaces
+       In this case the eliminations are applied to the internal \f$ M \f$
+       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
+   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
+                       DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /// Eliminate the given @a vdofs (all the fespaces), storing the eliminated part internally in \f$ M_e \f$.
+   /** This method works in conjunction with EliminateVDofsInRHS() and allows
+       elimination of boundary conditions in multiple right-hand sides. In this
+       method, @a vdofs is a list of DOFs. */
+   void EliminateVDofs(const Array<int> &vdofs,
+                       DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /** @brief Similar to
+       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
+       but here @a ess_dofs is a marker (boolean) array on all vector-dofs
+       (@a ess_dofs[i] < 0 is true). */
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
+                                     Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
+       here @a ess_dofs is a marker (boolean) array on all vector-dofs
+       (@a ess_dofs[i] < 0 is true). */
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs,
+                                     DiagonalPolicy dpolicy = DIAG_ONE);
+   /// Perform elimination and set the diagonal entry to the given value
+   void EliminateEssentialBCFromDofsDiag(const Array<int> &ess_dofs,
+                                         double value);
+
+   /** @brief Use the stored eliminated part of the matrix (see
+       EliminateVDofs(const Array<int> &, DiagonalPolicy)) to modify the r.h.s.
+       @a b; @a vdofs is a list of DOFs (non-directional, i.e. >= 0). */
+   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
+                            Vector &b);
+
+
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
+   void SetDiagonalPolicy(DiagonalPolicy policy)
+   {
+      diag_policy = policy;
+   }
+
+   /// Destroys bilinear form.
+   virtual ~BlockBilinearForm();
+
+};
+
+} // namespace mfem
+
+#endif
@@ -0,0 +1,136 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void BlockBilinearFormIntegrator::AssembleElementMatrix(
+   const Array<const FiniteElement *> &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   mfem_error ("BlockBilinearFormIntegrator::AssembleElementMatrix\n"
+               "   is not implemented for this class.");
+}
+
+void BlockLinearFormIntegrator::AssembleRHSElementVect(
+   const Array<const FiniteElement *> &el,
+   ElementTransformation &Trans,
+   Vector &elvect)
+{
+   mfem_error ("BlockLinearFormIntegrator::AssembleElementVector\n"
+               "   is not implemented for this class.");
+}
+
+/** Given a particular Finite Element computes the element vector */
+void TestBlockBilinearFormIntegrator::AssembleElementMatrix
+(const Array<const FiniteElement *> &el,
+ ElementTransformation &Trans,
+ DenseMatrix &elmat)
+{
+   int nd = 0;
+   int nblocks = el.Size();
+   Array<int> offsets(nblocks+1);
+   offsets[0] = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      nd += el[i]->GetDof();
+      offsets[i+1] = el[i]->GetDof();
+   }
+   offsets.PartialSum();
+   elmat.SetSize(nd);
+   elmat = 0.0;
+   DenseMatrix dmat;
+
+   if (blfis.NumRows())
+   {
+      // Get the matrices directly from the existing BilinearFormIntegrators
+      for (int i = 0; i<nblocks; i++)
+      {
+         // mfem::out << "i = " << i << std::endl;
+         int offset_i = offsets[i];
+         const FiniteElement * fe_i = el[i];
+         for (int j = 0; j<nblocks; j++)
+         {
+            // mfem::out << "j = " << j << std::endl;
+            BilinearFormIntegrator * blfi = blfis(i,j);
+            if (!blfi) { continue; }
+            if (j == i)
+            {
+               blfi->AssembleElementMatrix(*fe_i,Trans,dmat);
+               // mfem::out << "j 1 = " << j << std::endl;
+               elmat.SetSubMatrix(offset_i,dmat);
+            }
+            else
+            {
+               const FiniteElement * fe_j = el[j];
+               blfi->AssembleElementMatrix2(*fe_j,*fe_i,Trans,dmat);
+               // mfem::out << "j 2 = " << j << std::endl;
+               int offset_j = offsets[j];
+               elmat.SetSubMatrix(offset_i,offset_j,dmat);
+            }
+         }
+      }
+      return;
+   }
+
+   // else compute the matrices
+   elmat = 25.0;
+   // TODO
+
+}
+
+/** Given a particular Finite Element computes the element vector */
+void TestBlockLinearFormIntegrator::AssembleRHSElementVect
+(const Array<const FiniteElement *> &el,
+ ElementTransformation &Trans,
+ Vector &elvector)
+{
+   int nd = 0;
+   int nblocks = el.Size();
+   Array<int> offsets(nblocks+1);
+   offsets[0] = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      nd += el[i]->GetDof();
+      offsets[i+1] = el[i]->GetDof();
+   }
+   offsets.PartialSum();
+   elvector.SetSize(nd);
+   elvector = 0.0;
+   Vector subvector;
+
+   if (lfis.Size())
+   {
+      // Get the matrices directly from the existing BilinearFormIntegrators
+      for (int i = 0; i<nblocks; i++)
+      {
+         int offset = offsets[i];
+         const FiniteElement * fe_i = el[i];
+         LinearFormIntegrator * lfi = lfis[i];
+         if (!lfi)
+         {
+            continue;
+         }
+         lfi->AssembleRHSElementVect(*fe_i,Trans,subvector);
+         elvector.SetVector(subvector,offset);
+      }
+      return;
+   }
+
+   // else, compute the block linear form integrator
+   // elvector = 1.0;
+   // TODO
+}
+
+} // namespace mfem
@@ -0,0 +1,147 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKINTEG
+#define MFEM_BLOCKINTEG
+
+#include "../config/config.hpp"
+#include "fe.hpp"
+#include "coefficient.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+/** The abstract base class BlockBilinearFormIntegrator is
+ a generalization of the BilinearFormIntegrator class suitable
+ for block formulations. */
+class BlockBilinearFormIntegrator
+{
+protected:
+   const IntegrationRule *IntRule;
+   BlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : IntRule(ir) { }
+public:
+
+
+   /// Given a particular Finite Element computes the element matrix elmat.
+   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+
+   virtual ~BlockBilinearFormIntegrator() { }
+};
+
+/** The abstract base class BlockBilinearFormIntegrator is
+ a generalization of the BilinearFormIntegrator class suitable
+ for block formulations. */
+class BlockLinearFormIntegrator
+{
+protected:
+   const IntegrationRule *IntRule;
+   BlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : IntRule(ir) { }
+
+public:
+   /// Given a particular Finite Element computes the element matrix elmat.
+   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
+                                       ElementTransformation &Trans,
+                                       Vector &elvect);
+
+   virtual ~BlockLinearFormIntegrator() { }
+};
+
+
+
+
+class TestBlockBilinearFormIntegrator: public BlockBilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   Array<const FiniteElementSpace * > fespaces;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+
+   Array2D<BilinearFormIntegrator *> blfis;
+
+
+public:
+
+   TestBlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : BlockBilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a mass integrator with coefficient q
+   TestBlockBilinearFormIntegrator(Coefficient &q,
+                                   const IntegrationRule *ir = NULL)
+      : BlockBilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL)   { }
+
+
+   TestBlockBilinearFormIntegrator(Array2D<BilinearFormIntegrator *> blfis_)
+      : BlockBilinearFormIntegrator(NULL), blfis(blfis_) { }
+
+   void SetIntegrators(Array2D<BilinearFormIntegrator *> blfis_)
+   {
+      blfis = blfis_;
+   }
+
+
+   /** Given a particular Finite Element computes the element matrix
+       elmat. */
+   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+
+   virtual ~TestBlockBilinearFormIntegrator() { }
+
+};
+
+/** Class for local vector assembly */
+class TestBlockLinearFormIntegrator: public BlockLinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   Array<const FiniteElementSpace * > fespaces;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+   Array<LinearFormIntegrator *> lfis;
+
+public:
+
+   TestBlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : BlockLinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a test linear integrator with coefficient q
+   TestBlockLinearFormIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
+      : BlockLinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL) { }
+
+
+   TestBlockLinearFormIntegrator(Array<LinearFormIntegrator *> lfis_)
+      : BlockLinearFormIntegrator(NULL), lfis(lfis_) { }
+
+   void SetIntegrators(Array<LinearFormIntegrator *> lfis_)
+   {
+      lfis = lfis_;
+   }
+
+   /** Given a particular Finite Element computes the element vector */
+   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
+                                       ElementTransformation &Trans,
+                                       Vector &elvector);
+
+
+};
+
+
+} // namespace mfem
+
+
+#endif
@@ -0,0 +1,123 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+BlockLinearForm::BlockLinearForm(Array<FiniteElementSpace * > & fespaces_) :
+   Vector(0), fespaces(fespaces_)
+{
+   int s = 0;
+   int nblocks = fespaces.Size();
+   for (int i =0; i<nblocks; i++)
+   {
+      s += fespaces[i]->GetVSize();
+   }
+   // mfem::out << "size = " << size << std::endl;
+
+   SetSize(s);
+
+}
+
+
+void BlockLinearForm::AddDomainIntegrator(BlockLinearFormIntegrator *lfi)
+{
+   domain_integs.Append(lfi);
+}
+
+void BlockLinearForm::Assemble()
+{
+   ElementTransformation *eltrans;
+   DofTransformation *doftrans;
+   Mesh *mesh = fespaces[0] -> GetMesh();
+   Vector subvect,elvect, *elvect_p;
+
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   Array<int> offsetvdofs;
+   Array<int> elementblockoffsets(nblocks+1);
+   elementblockoffsets[0] = 0;
+   Array<int> blockoffsets(nblocks+1);
+   blockoffsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      blockoffsets[i+1] = fespaces[i]->GetVSize();
+   }
+   blockoffsets.PartialSum();
+
+   Vector::operator=(0.0);
+
+   if (domain_integs.Size())
+   {
+      // loop through elements
+      for (int i = 0; i < mesh -> GetNE(); i++)
+      {
+         elvect.SetSize(0);
+         for (int k = 0; k < domain_integs.Size(); k++)
+         {
+            for (int j = 0; j<nblocks; j++)
+            {
+               fe[j] = fespaces[j]->GetFE(i);
+               elementblockoffsets[j+1] = fe[j]->GetDof();
+            }
+            elementblockoffsets.PartialSum();
+            eltrans = mesh->GetElementTransformation(i);
+
+            domain_integs[k]->AssembleRHSElementVect(fe, *eltrans, elemvect);
+            if (elvect.Size() == 0)
+            {
+               elvect = elemvect;
+            }
+            else
+            {
+               elvect += elemvect;
+            }
+         }
+         if (elvect.Size() == 0)
+         {
+            continue;
+         }
+         else
+         {
+            elvect_p = &elvect;
+         }
+
+         double *data = elvect_p->GetData();
+
+         for (int j = 0; j<nblocks; j++)
+         {
+            doftrans = fespaces[j]->GetElementVDofs(i, vdofs);
+            int offset = blockoffsets[j];
+            offsetvdofs.SetSize(vdofs.Size());
+            for (int l = 0; l<vdofs.Size(); l++)
+            {
+               offsetvdofs[l] = vdofs[l]<0 ? -offset + vdofs[l]
+                                :  offset + vdofs[l];
+            }
+            int jbeg = elementblockoffsets[j];
+            int jend = elementblockoffsets[j+1]-1;
+            subvect.SetSize(jend-jbeg+1);
+            subvect.SetData(&data[jbeg]);
+
+            if (doftrans)
+            {
+               doftrans->TransformDual(subvect);
+            }
+            AddElementVector(offsetvdofs,subvect);
+         }
+      }
+   }
+}
+
+
+} // name space mfem
@@ -0,0 +1,49 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKLINEARFORM
+#define MFEM_BLOCKLINEARFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+
+namespace mfem
+{
+
+
+class BlockLinearForm : public Vector
+{
+protected:
+   /// FE spaces on which the LinearForm lives. Not owned.
+   Array<FiniteElementSpace * > fespaces;
+
+   /// Set of Domain Integrators to be applied.
+   Array<BlockLinearFormIntegrator*> domain_integs;
+
+   Vector elemvect;
+   Array<int> vdofs;
+
+public:
+   BlockLinearForm(Array<FiniteElementSpace * > & fespaces_);
+
+   /// Adds new Domain Integrator. Assumes ownership of @a lfi.
+   void AddDomainIntegrator(BlockLinearFormIntegrator *lfi);
+
+   /// Assembles the block linear form i.e. sums over all domain integrators.
+   void Assemble();
+
+
+};
+
+} // namespace mfem
+
+
+#endif
@@ -0,0 +1,969 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "blockstaticcond.hpp"
+
+namespace mfem
+{
+
+
+BlockStaticCondensation::BlockStaticCondensation(Array<FiniteElementSpace *> &
+                                                 fes_)
+{
+   SetSpaces(fes_);
+
+   Array<int> rvdofs;
+   Array<int> vdofs;
+   Array<int> rdof_edof0;
+   for (int k = 0; k<nblocks; k++)
+   {
+      if (!tr_fes[k]) { continue; }
+      rdof_edof0.SetSize(tr_fes[k]->GetVSize());
+      for (int i = 0; i < mesh->GetNE(); i++)
+      {
+         fes[k]->GetElementVDofs(i, vdofs);
+         tr_fes[k]->GetElementVDofs(i, rvdofs);
+         const int vdim = fes[k]->GetVDim();
+         const int nsd = vdofs.Size()/vdim;
+         const int nsrd = rvdofs.Size()/vdim;
+         for (int vd = 0; vd < vdim; vd++)
+         {
+            for (int j = 0; j < nsrd; j++)
+            {
+               int rvdof = rvdofs[j+nsrd*vd];
+               int vdof = vdofs[j+nsd*vd];
+               if (rvdof < 0)
+               {
+                  rvdof = -1-rvdof;
+                  vdof = -1-vdof;
+               }
+               MFEM_ASSERT(vdof >= 0, "incompatible volume and trace FE spaces");
+               rdof_edof0[rvdof] = vdof + dof_offsets[k];
+            }
+         }
+      }
+      rdof_edof.Append(rdof_edof0);
+   }
+}
+
+void BlockStaticCondensation::SetSpaces(Array<FiniteElementSpace*> & fes_)
+{
+#ifdef MFEM_USE_MPI
+   ParMesh *pmesh = nullptr;
+   parallel = false;
+   if (dynamic_cast<ParFiniteElementSpace *>(fes_[0]))
+   {
+      parallel = true;
+   }
+#else
+   parallel = false;
+#endif
+   fes=fes_;
+   nblocks = fes.Size();
+   rblocks = 0;
+   tr_fes.SetSize(nblocks);
+   mesh = fes[0]->GetMesh();
+
+   IsTraceSpace.SetSize(nblocks);
+   const FiniteElementCollection * fec;
+   for (int i = 0; i < nblocks; i++)
+   {
+      fec = fes[i]->FEColl();
+      IsTraceSpace[i] =
+         (dynamic_cast<const H1_Trace_FECollection*>(fec) ||
+          dynamic_cast<const ND_Trace_FECollection*>(fec) ||
+          dynamic_cast<const RT_Trace_FECollection*>(fec));
+#ifdef MFEM_USE_MPI
+      if (parallel)
+      {
+         pmesh = dynamic_cast<ParMesh *>(mesh);
+         tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
+                     nullptr : (IsTraceSpace[i]) ? fes[i] :
+                     new ParFiniteElementSpace(pmesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                                               fes[i]->GetOrdering());
+      }
+      else
+      {
+         tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
+                     nullptr : (IsTraceSpace[i]) ? fes[i] :
+                     new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                                            fes[i]->GetOrdering());
+      }
+#else
+      // skip if it's an L2 space (no trace space to construct)
+      tr_fes[i] = (fec->GetContType() == FiniteElementCollection::DISCONTINUOUS) ?
+                  nullptr : (IsTraceSpace[i]) ? fes[i] :
+                  new FiniteElementSpace(mesh, fec->GetTraceCollection(), fes[i]->GetVDim(),
+                                         fes[i]->GetOrdering());
+#endif
+      if (tr_fes[i]) { rblocks++; }
+   }
+   if (parallel)
+   {
+      ess_tdofs.SetSize(rblocks);
+      for (int i = 0; i<rblocks; i++)
+      {
+         ess_tdofs[i] = new Array<int>();
+      }
+   }
+   Init();
+}
+
+void BlockStaticCondensation::ComputeOffsets()
+{
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+
+   rdof_offsets.SetSize(rblocks+1);
+   rtdof_offsets.SetSize(rblocks+1);
+   rdof_offsets[0] = 0;
+   rtdof_offsets[0] = 0;
+
+   int j=0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fes[i]->GetVSize();
+      tdof_offsets[i+1] = fes[i]->GetTrueVSize();
+      if (tr_fes[i])
+      {
+         rdof_offsets[j+1] = tr_fes[i]->GetVSize();
+         rtdof_offsets[j+1] = tr_fes[i]->GetTrueVSize();
+         j++;
+      }
+   }
+   rdof_offsets.PartialSum();
+   rtdof_offsets.PartialSum();
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+}
+
+
+void BlockStaticCondensation::Init()
+{
+   lmat.SetSize(mesh->GetNE());
+   lvec.SetSize(mesh->GetNE());
+   for (int i = 0; i < mesh->GetNE(); i++)
+   {
+      lmat[i] = nullptr;
+      lvec[i] = nullptr;
+   }
+
+   ComputeOffsets();
+
+   S = new BlockMatrix(rdof_offsets);
+   S->owns_blocks = 1;
+
+   for (int i = 0; i<S->NumRowBlocks(); i++)
+   {
+      int h = rdof_offsets[i+1] - rdof_offsets[i];
+      for (int j = 0; j<S->NumColBlocks(); j++)
+      {
+         int w = rdof_offsets[j+1] - rdof_offsets[j];
+         S->SetBlock(i,j,new SparseMatrix(h, w));
+      }
+   }
+   y = new BlockVector(rdof_offsets);
+   *y = 0.;
+}
+
+void BlockStaticCondensation::GetReduceElementIndicesAndOffsets(int el,
+                                                                Array<int> & trace_ldofs,
+                                                                Array<int> & interior_ldofs,
+                                                                Array<int> & offsets) const
+{
+   int dim = mesh->Dimension();
+   offsets.SetSize(tr_fes.Size()+1); offsets = 0;
+   Array<int> dofs;
+   Array<int> faces, ori;
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+
+   trace_ldofs.SetSize(0);
+   interior_ldofs.SetSize(0);
+   // construct Array of bubble dofs to be extracted
+   int skip=0;
+   Array<int> tr_dofs;
+   Array<int> int_dofs;
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      int td = 0;
+      int ndof;
+      // if it's an L2 space (bubbles)
+      if (!tr_fes[i])
+      {
+         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         td = 0;
+      }
+      else if (IsTraceSpace[i])
+      {
+         for (int iface = 0; iface < numfaces; iface++)
+         {
+            td += fes[i]->GetVDim()*fes[i]->GetFaceElement(faces[iface])->GetDof();
+         }
+         ndof = td;
+      }
+      else
+      {
+         Array<int> trace_dofs;
+         ndof = fes[i]->GetVDim()*fes[i]->GetFE(el)->GetDof();
+         tr_fes[i]->GetElementVDofs(el, trace_dofs);
+         td = trace_dofs.Size(); // number of trace dofs
+      }
+      offsets[i+1] = td;
+      tr_dofs.SetSize(td);
+      int_dofs.SetSize(ndof - td);
+      for (int j = 0; j<td; j++)
+      {
+         tr_dofs[j] = skip + j;
+      }
+      for (int j = 0; j<ndof-td; j++)
+      {
+         int_dofs[j] = skip + td + j;
+      }
+      skip+=ndof;
+
+      trace_ldofs.Append(tr_dofs);
+      interior_ldofs.Append(int_dofs);
+   }
+   offsets.PartialSum();
+}
+
+
+void BlockStaticCondensation::GetReduceElementVDofs(int el,
+                                                    Array<int> & rdofs) const
+{
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+   rdofs.SetSize(0);
+   int skip = 0;
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      Array<int> vdofs;
+      if (IsTraceSpace[i])
+      {
+         Array<int> face_vdofs;
+         for (int k = 0; k < numfaces; k++)
+         {
+            int iface = faces[k];
+            tr_fes[i]->GetFaceVDofs(iface, face_vdofs);
+            vdofs.Append(face_vdofs);
+         }
+      }
+      else
+      {
+         tr_fes[i]->GetElementVDofs(el, vdofs);
+      }
+      for (int j=0; j<vdofs.Size(); j++)
+      {
+         vdofs[j] = (vdofs[j]>=0) ? vdofs[j]+rdof_offsets[skip] :
+                    vdofs[j]-rdof_offsets[skip];
+      }
+      skip++;
+      rdofs.Append(vdofs);
+   }
+}
+void BlockStaticCondensation::GetElementVDofs(int el, Array<int> & vdofs) const
+{
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+   vdofs.SetSize(0);
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      Array<int> dofs;
+      if (IsTraceSpace[i])
+      {
+         Array<int> face_vdofs;
+         for (int k = 0; k < numfaces; k++)
+         {
+            int iface = faces[k];
+            fes[i]->GetFaceVDofs(iface, face_vdofs);
+            dofs.Append(face_vdofs);
+         }
+      }
+      else
+      {
+         fes[i]->GetElementVDofs(el, dofs);
+      }
+      for (int j=0; j<dofs.Size(); j++)
+      {
+         dofs[j] = (dofs[j]>=0) ? dofs[j]+dof_offsets[i] :
+                   dofs[j]-dof_offsets[i];
+      }
+      vdofs.Append(dofs);
+   }
+}
+
+
+void BlockStaticCondensation::GetLocalShurComplement(int el,
+                                                     const Array<int> & tr_idx, const Array<int> & int_idx,
+                                                     const DenseMatrix & elmat, const Vector & elvect,
+                                                     DenseMatrix & rmat, Vector & rvect)
+{
+   int rdofs = tr_idx.Size();
+   int idofs = int_idx.Size();
+   MFEM_VERIFY(idofs != 0, "Number of interior dofs is zero");
+   MFEM_VERIFY(rdofs != 0, "Number of interface dofs is zero");
+
+   rmat.SetSize(rdofs);
+   rvect.SetSize(rdofs);
+
+   DenseMatrix A_tt, A_ti, A_it, A_ii;
+   Vector y_t, y_i;
+
+   elmat.GetSubMatrix(tr_idx,A_tt);
+   elmat.GetSubMatrix(tr_idx,int_idx, A_ti);
+   elmat.GetSubMatrix(int_idx, tr_idx, A_it);
+   elmat.GetSubMatrix(int_idx, A_ii);
+
+   elvect.GetSubVector(tr_idx, y_t);
+   elvect.GetSubVector(int_idx, y_i);
+
+   DenseMatrixInverse lu(A_ii);
+   lu.Factor();
+   lmat[el] = new DenseMatrix(idofs,rdofs);
+   lvec[el] = new Vector(idofs);
+
+   lu.Mult(A_it,*lmat[el]);
+   lu.Mult(y_i,*lvec[el]);
+
+   // LHS
+   mfem::Mult(A_ti,*lmat[el],rmat);
+
+   rmat.Neg();
+   rmat.Add(1., A_tt);
+
+   // RHS
+   A_ti.Mult(*lvec[el], rvect);
+   rvect.Neg();
+   rvect.Add(1., y_t);
+}
+
+
+void BlockStaticCondensation::AssembleReducedSystem(int el,
+                                                    DenseMatrix &elmat,
+                                                    Vector & elvect)
+{
+   // Get Shur Complement
+   Array<int> tr_idx, int_idx;
+   Array<int> offsets;
+   // Get local element idx and offsets for global assembly
+   GetReduceElementIndicesAndOffsets(el, tr_idx,int_idx, offsets);
+
+   DenseMatrix rmat, *rmatptr;
+   Vector rvec, *rvecptr;
+   // Extract the reduced matrices based on tr_idx and int_idx
+   if (int_idx.Size()!=0)
+   {
+      GetLocalShurComplement(el,tr_idx,int_idx, elmat, elvect, rmat, rvec);
+      rmatptr = &rmat;
+      rvecptr = &rvec;
+   }
+   else
+   {
+      rmatptr = &elmat;
+      rvecptr = &elvect;
+   }
+
+   // Assemble global mat and rhs
+   DofTransformation * doftrans_i, *doftrans_j;
+
+
+   Array<int> faces, ori;
+   int dim = mesh->Dimension();
+   if (dim == 1)
+   {
+      mesh->GetElementVertices(el, faces);
+   }
+   if (dim == 2)
+   {
+      mesh->GetElementEdges(el, faces, ori);
+   }
+   else //dim = 3
+   {
+      mesh->GetElementFaces(el,faces,ori);
+   }
+   int numfaces = faces.Size();
+
+   int skip_i=0;
+   for (int i = 0; i<tr_fes.Size(); i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      Array<int> vdofs_i;
+      doftrans_i = nullptr;
+      if (IsTraceSpace[i])
+      {
+         Array<int> face_vdofs;
+         for (int k = 0; k < numfaces; k++)
+         {
+            int iface = faces[k];
+            tr_fes[i]->GetFaceVDofs(iface, face_vdofs);
+            vdofs_i.Append(face_vdofs);
+         }
+      }
+      else
+      {
+         doftrans_i = tr_fes[i]->GetElementVDofs(el, vdofs_i);
+      }
+      int skip_j=0;
+      for (int j = 0; j<tr_fes.Size(); j++)
+      {
+         if (!tr_fes[j]) { continue; }
+         Array<int> vdofs_j;
+         doftrans_j = nullptr;
+
+         if (IsTraceSpace[j])
+         {
+            Array<int> face_vdofs;
+            for (int k = 0; k < numfaces; k++)
+            {
+               int iface = faces[k];
+               tr_fes[j]->GetFaceVDofs(iface, face_vdofs);
+               vdofs_j.Append(face_vdofs);
+            }
+         }
+         else
+         {
+            doftrans_j = tr_fes[j]->GetElementVDofs(el, vdofs_j);
+         }
+
+         DenseMatrix Ae;
+         rmatptr->GetSubMatrix(offsets[i],offsets[i+1],
+                               offsets[j],offsets[j+1], Ae);
+         if (doftrans_i || doftrans_j)
+         {
+            TransformDual(doftrans_i, doftrans_j, Ae);
+         }
+         S->GetBlock(skip_i,skip_j).AddSubMatrix(vdofs_i,vdofs_j, Ae);
+         skip_j++;
+      }
+
+      // assemble rhs
+      double * data = rvecptr->GetData();
+      Vector vec1;
+      // ref subvector
+      vec1.SetDataAndSize(&data[offsets[i]],
+                          offsets[i+1]-offsets[i]);
+      if (doftrans_i)
+      {
+         doftrans_i->TransformDual(vec1);
+      }
+      y->GetBlock(skip_i).AddElementVector(vdofs_i,vec1);
+      skip_i++;
+   }
+}
+
+void BlockStaticCondensation::BuildProlongation()
+{
+   P = new BlockMatrix(rdof_offsets, rtdof_offsets);
+   R = new BlockMatrix(rtdof_offsets, rdof_offsets);
+   P->owns_blocks = 0;
+   R->owns_blocks = 0;
+   int skip = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      const SparseMatrix *P_ = tr_fes[i]->GetConformingProlongation();
+      if (P_)
+      {
+         const SparseMatrix *R_ = tr_fes[i]->GetRestrictionMatrix();
+         P->SetBlock(skip,skip,const_cast<SparseMatrix*>(P_));
+         R->SetBlock(skip,skip,const_cast<SparseMatrix*>(R_));
+      }
+      skip++;
+   }
+}
+
+#ifdef MFEM_USE_MPI
+void BlockStaticCondensation::BuildParallelProlongation()
+{
+   MFEM_VERIFY(parallel, "BuildParallelProlongation: wrong code path");
+   pP = new BlockOperator(rdof_offsets, rtdof_offsets);
+   R = new BlockMatrix(rtdof_offsets, rdof_offsets);
+   pP->owns_blocks = 0;
+   R->owns_blocks = 0;
+   int skip = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      const HypreParMatrix *P_ =
+         dynamic_cast<ParFiniteElementSpace *>(tr_fes[i])->Dof_TrueDof_Matrix();
+      if (P_)
+      {
+         const SparseMatrix *R_ = tr_fes[i]->GetRestrictionMatrix();
+         pP->SetBlock(skip,skip,const_cast<HypreParMatrix*>(P_));
+         R->SetBlock(skip,skip,const_cast<SparseMatrix*>(R_));
+      }
+      skip++;
+   }
+}
+
+void BlockStaticCondensation::ParallelAssemble(BlockMatrix *m)
+{
+   if (!pP) { BuildParallelProlongation(); }
+
+   pS = new BlockOperator(rtdof_offsets);
+   pS_e = new BlockOperator(rtdof_offsets);
+   pS->owns_blocks = 1;
+   pS_e->owns_blocks = 1;
+   HypreParMatrix * A = nullptr;
+   HypreParMatrix * PtAP = nullptr;
+   int skip_i=0;
+   ParFiniteElementSpace * pfes_i = nullptr;
+   ParFiniteElementSpace * pfes_j = nullptr;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      pfes_i = dynamic_cast<ParFiniteElementSpace*>(fes[i]);
+      HypreParMatrix * Pi = (HypreParMatrix*)(&pP->GetBlock(skip_i,skip_i));
+      int skip_j=0;
+      for (int j = 0; j<nblocks; j++)
+      {
+         if (!tr_fes[j]) { continue; }
+         if (m->IsZeroBlock(skip_i,skip_j)) { continue; }
+         if (skip_i == skip_j)
+         {
+            // Make block diagonal square hypre matrix
+            A = new HypreParMatrix(pfes_i->GetComm(), pfes_i->GlobalVSize(),
+                                   pfes_i->GetDofOffsets(),&m->GetBlock(skip_i,skip_i));
+            PtAP = RAP(A,Pi);
+            delete A;
+            pS_e->SetBlock(skip_i,skip_i,PtAP->EliminateRowsCols(*ess_tdofs[skip_i]));
+         }
+         else
+         {
+            pfes_j = dynamic_cast<ParFiniteElementSpace*>(fes[j]);
+            HypreParMatrix * Pj = (HypreParMatrix*)(&pP->GetBlock(skip_j,skip_j));
+            A = new HypreParMatrix(pfes_i->GetComm(), pfes_i->GlobalVSize(),
+                                   pfes_j->GlobalVSize(), pfes_i->GetDofOffsets(),
+                                   pfes_j->GetDofOffsets(), &m->GetBlock(skip_i,skip_j));
+            PtAP = RAP(Pi,A,Pj);
+            delete A;
+            pS_e->SetBlock(skip_i,skip_j,PtAP->EliminateCols(*ess_tdofs[skip_j]));
+            PtAP->EliminateRows(*ess_tdofs[skip_i]);
+         }
+         pS->SetBlock(skip_i,skip_j,PtAP);
+         skip_j++;
+      }
+      skip_i++;
+   }
+}
+
+#endif
+
+
+void BlockStaticCondensation::ConformingAssemble(int skip_zeros)
+{
+   Finalize(0);
+   if (!P) { BuildProlongation(); }
+
+   BlockMatrix * Pt = Transpose(*P);
+   BlockMatrix * PtA = mfem::Mult(*Pt, *S);
+   delete S;
+   if (S_e)
+   {
+      BlockMatrix *PtAe = mfem::Mult(*Pt, *S_e);
+      delete S_e;
+      S_e = PtAe;
+   }
+   delete Pt;
+   S = mfem::Mult(*PtA, *P);
+   delete PtA;
+
+   if (S_e)
+   {
+      BlockMatrix *PtAeP = mfem::Mult(*S_e, *P);
+      S_e = PtAeP;
+   }
+   height = S->Height();
+   width = S->Width();
+}
+
+void BlockStaticCondensation::Finalize(int skip_zeros)
+{
+   if (S) { S->Finalize(skip_zeros); }
+   if (S_e) { S_e->Finalize(skip_zeros); }
+}
+
+void BlockStaticCondensation::FormSystemMatrix(Operator::DiagonalPolicy
+                                               diag_policy)
+{
+   if (parallel)
+   {
+      FillEssTdofLists(ess_rtdof_list);
+      if (S)
+      {
+         const int remove_zeros = 0;
+         Finalize(remove_zeros);
+         ParallelAssemble(S);
+         delete S;
+         S=nullptr;
+         delete S_e;
+         S_e = nullptr;
+      }
+   }
+   else
+   {
+      if (!S_e)
+      {
+         bool conforming = true;
+         for (int i = 0; i<nblocks; i++)
+         {
+            if (!tr_fes[i]) { continue; }
+            const SparseMatrix *P_ = tr_fes[i]->GetConformingProlongation();
+            if (P_)
+            {
+               conforming = false;
+               break;
+            }
+         }
+         if (!conforming) { ConformingAssemble(0); }
+         const int remove_zeros = 0;
+         EliminateReducedTrueDofs(ess_rtdof_list, diag_policy);
+         Finalize(remove_zeros);
+      }
+   }
+}
+
+
+void BlockStaticCondensation::ConvertMarkerToReducedTrueDofs(
+   Array<int> & tdof_marker,
+   Array<int> & rtdof_marker)
+{
+   // convert tdof_marker to dof_marker
+   rtdof_marker.SetSize(0);
+   Array<int> tdof_marker0;
+   Array<int> dof_marker0;
+   Array<int> dof_marker;
+   int * data = tdof_marker.GetData();
+   for (int i = 0; i<nblocks; i++)
+   {
+      tdof_marker0.MakeRef(&data[tdof_offsets[i]],tdof_offsets[i+1]-tdof_offsets[i]);
+      const SparseMatrix * R = fes[i]->GetRestrictionMatrix();
+      if (!R)
+      {
+         dof_marker0.MakeRef(tdof_marker0);
+      }
+      else
+      {
+         dof_marker0.SetSize(fes[i]->GetVSize());
+         R->BooleanMultTranspose(tdof_marker0, dof_marker0);
+      }
+      dof_marker.Append(dof_marker0);
+   }
+
+   int rdofs = rdof_edof.Size();
+   Array<int> rdof_marker(rdofs);
+
+   for (int i = 0; i < rdofs; i++)
+   {
+      rdof_marker[i] = dof_marker[rdof_edof[i]];
+   }
+
+   // convert rdof_marker to rtdof_marker
+   Array<int> rtdof_marker0;
+   Array<int> rdof_marker0;
+   int * rdata = rdof_marker.GetData();
+   int k=0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      if (!tr_fes[i]) { continue; }
+      rdof_marker0.MakeRef(&rdata[rdof_offsets[k]],rdof_offsets[k+1]-rdof_offsets[k]);
+      const SparseMatrix *tr_R = tr_fes[i]->GetRestrictionMatrix();
+      if (!tr_R)
+      {
+         rtdof_marker0.MakeRef(rdof_marker0);
+      }
+      else
+      {
+         rtdof_marker0.SetSize(tr_fes[i]->GetTrueVSize());
+         tr_R->BooleanMult(rdof_marker0, rtdof_marker0);
+      }
+      rtdof_marker.Append(rtdof_marker0);
+      k++;
+   }
+}
+
+void BlockStaticCondensation::FillEssTdofLists(const Array<int> & ess_tdof_list)
+{
+   int j;
+   for (int i = 0; i<ess_tdof_list.Size(); i++)
+   {
+      int tdof = ess_tdof_list[i];
+      for (j = 0; j < rblocks; j++)
+      {
+         if (rtdof_offsets[j+1] > tdof) { break; }
+      }
+      ess_tdofs[j]->Append(tdof-rtdof_offsets[j]);
+   }
+}
+
+void BlockStaticCondensation::SetEssentialTrueDofs(const Array<int>
+                                                   &ess_tdof_list)
+{
+   Array<int> tdof_marker;
+   Array<int> rtdof_marker;
+   FiniteElementSpace::ListToMarker(ess_tdof_list,tdof_offsets.Last(),tdof_marker);
+   ConvertMarkerToReducedTrueDofs(tdof_marker, rtdof_marker);
+   FiniteElementSpace::MarkerToList(rtdof_marker,ess_rtdof_list);
+}
+
+void BlockStaticCondensation::EliminateReducedTrueDofs(const Array<int>
+                                                       &ess_rtdof_list,
+                                                       Matrix::DiagonalPolicy dpolicy)
+{
+
+   MFEM_VERIFY(!parallel, "EliminateReducedTrueDofs::Wrong Code path");
+
+   if (S_e == NULL)
+   {
+      Array<int> offsets;
+
+      offsets.MakeRef( (P) ? rtdof_offsets : rdof_offsets);
+
+      S_e = new BlockMatrix(offsets);
+      S_e->owns_blocks = 1;
+      for (int i = 0; i<S_e->NumRowBlocks(); i++)
+      {
+         int h = offsets[i+1] - offsets[i];
+         for (int j = 0; j<S_e->NumColBlocks(); j++)
+         {
+            int w = offsets[j+1] - offsets[j];
+            S_e->SetBlock(i,j,new SparseMatrix(h, w));
+         }
+      }
+   }
+   S->EliminateRowCols(ess_rtdof_list,S_e,dpolicy);
+}
+
+void BlockStaticCondensation::EliminateReducedTrueDofs(Matrix::DiagonalPolicy
+                                                       dpolicy)
+{
+   EliminateReducedTrueDofs(ess_rtdof_list, dpolicy);
+}
+
+void BlockStaticCondensation::ReduceSolution(const Vector &sol,
+                                             Vector &sc_sol) const
+{
+   MFEM_ASSERT(sol.Size() == dof_offsets.Last(), "'sol' has incorrect size");
+   const int nrdofs = rdof_offsets.Last();
+   Vector sol_r;
+   if (!R)
+   {
+      sc_sol.SetSize(nrdofs);
+      sol_r.SetDataAndSize(sc_sol.GetData(), sc_sol.Size());
+   }
+   else
+   {
+      sol_r.SetSize(nrdofs);
+   }
+   for (int i = 0; i < nrdofs; i++)
+   {
+      sol_r(i) = sol(rdof_edof[i]);
+   }
+   if (R)
+   {
+      // wrap vector into a block vector
+      BlockVector blsol_r(sol_r,rdof_offsets);
+      sc_sol.SetSize(R->Height());
+      R->Mult(blsol_r, sc_sol);
+   }
+}
+
+void BlockStaticCondensation::ReduceSystem(Vector &x, Vector &X,
+                                           Vector &B,
+                                           int copy_interior) const
+{
+   ReduceSolution(x, X);
+   if (parallel)
+   {
+      B.SetSize(pP->Width());
+      pP->MultTranspose(*y,B);
+
+      Vector tmp(B.Size());
+      pS_e->Mult(X,tmp);
+      B-=tmp;
+      for (int j = 0; j<rblocks; j++)
+      {
+         if (!ess_tdofs[j]->Size()) { continue; }
+         HypreParMatrix *Ah = (HypreParMatrix *)(&pS->GetBlock(j,j));
+         Vector diag;
+         Ah->GetDiag(diag);
+         for (int i = 0; i < ess_tdofs[j]->Size(); i++)
+         {
+            int tdof = (*ess_tdofs[j])[i];
+            int gdof = tdof + rtdof_offsets[j];
+            B(gdof) = diag(tdof)*X(gdof);
+         }
+      }
+   }
+   else
+   {
+      if (!P)
+      {
+         S_e->AddMult(X,*y,-1.);
+         S->PartMult(ess_rtdof_list,X,*y);
+         B.MakeRef(*y, 0, y->Size());
+      }
+      else
+      {
+         B.SetSize(P->Width());
+         P->MultTranspose(*y, B);
+         S_e->AddMult(X,B,-1.);
+         S->PartMult(ess_rtdof_list,X,B);
+      }
+   }
+   if (!copy_interior) { X.SetSubVectorComplement(ess_rtdof_list, 0.0); }
+}
+
+
+void BlockStaticCondensation::ComputeSolution(const Vector &sc_sol,
+                                              Vector &sol) const
+{
+
+   const int nrdofs = rdof_offsets.Last();
+   const int nrtdofs = rtdof_offsets.Last();
+   MFEM_VERIFY(sc_sol.Size() == nrtdofs, "'sc_sol' has incorrect size");
+
+   Vector sol_r;
+   if (parallel)
+   {
+      sol_r.SetSize(nrdofs);
+      pP->Mult(sc_sol, sol_r);
+   }
+   else
+   {
+      if (!P)
+      {
+         sol_r.SetDataAndSize(sc_sol.GetData(), sc_sol.Size());
+      }
+      else
+      {
+         sol_r.SetSize(nrdofs);
+         P->Mult(sc_sol, sol_r);
+      }
+   }
+
+
+   if (rdof_offsets.Last() == dof_offsets.Last())
+   {
+      sol = sol_r;
+      return;
+   }
+   else
+   {
+      sol.SetSize(dof_offsets.Last());
+   }
+
+   Vector lsr; // element (local) sc solution vector
+   Vector lsi; // element (local) interior solution vector
+   const int NE = mesh->GetNE();
+
+   Array<int> trace_vdofs;
+   Array<int> vdofs;
+   Array<int> tr_offsets;
+   Vector lsol;
+   for (int iel = 0; iel < NE; iel++)
+   {
+      lsol.SetSize(lmat[iel]->Width() + lmat[iel]->Height());
+      // GetReduceElementIndicesAndOffsets(iel, trace_ldofs, interior_ldofs, tr_offsets);
+      GetReduceElementVDofs(iel, trace_vdofs);
+
+      lsr.SetSize(trace_vdofs.Size());
+      sol_r.GetSubVector(trace_vdofs, lsr);
+      // complete the interior dofs
+
+      lsi.SetSize(lmat[iel]->Height());
+      lmat[iel]->Mult(lsr,lsi);
+      lsi.Neg();
+      lsi+=*lvec[iel];
+
+      Array<int> tr_idx,int_idx,idx_offs;
+      GetReduceElementIndicesAndOffsets(iel,tr_idx, int_idx, idx_offs);
+      lsol.SetSubVector(tr_idx,lsr);
+
+      lsol.SetSubVector(int_idx,lsi);
+
+      GetElementVDofs(iel, vdofs);
+      sol.SetSubVector(vdofs,lsol);
+
+   }
+
+}
+
+BlockStaticCondensation::~BlockStaticCondensation()
+{
+   delete S_e; S_e = nullptr;
+   delete S; S=nullptr;
+   delete y; y=nullptr;
+
+   if (P) { delete P; } P=nullptr;
+   if (R) { delete R; } R=nullptr;
+
+   if (parallel)
+   {
+      delete pS; pS=nullptr;
+      delete pS_e; pS_e=nullptr;
+      for (int i = 0; i<rblocks; i++)
+      {
+         delete ess_tdofs[i];
+      }
+      delete pP; pP=nullptr;
+   }
+
+   for (int i=0; i<lmat.Size(); i++)
+   {
+      delete lmat[i]; lmat[i] = nullptr;
+      delete lvec[i]; lvec[i] = nullptr;
+   }
+}
+
+}
+
@@ -0,0 +1,191 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCK_STATIC_CONDENSATION
+#define MFEM_BLOCK_STATIC_CONDENSATION
+
+#include "../config/config.hpp"
+#include "fespace.hpp"
+
+#ifdef MFEM_USE_MPI
+#include "pfespace.hpp"
+#endif
+
+namespace mfem
+{
+
+
+class BlockStaticCondensation
+{
+   int height, width;
+   int nblocks; // original number of blocks
+   int rblocks; // reduces number of blocks
+   Mesh * mesh = nullptr;
+   bool parallel = false;
+   // original set of Finite Element Spaces
+   Array<FiniteElementSpace *> fes;
+   // indicates if the original space is already a trace space
+   Array<bool> IsTraceSpace;
+
+   // New set of "reduced" Finite Element Spaces
+   // (after static condensation)
+   Array<FiniteElementSpace *> tr_fes;
+
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   Array<int> rdof_offsets;
+   Array<int> rtdof_offsets;
+
+   // Schur complement matrix
+   // S = A_ii - A_ib (A_bb)^{-1} A_bi.
+   BlockMatrix * S = nullptr;
+   BlockMatrix * S_e = nullptr;
+
+   BlockVector * y = nullptr;
+
+   Array<DenseMatrix * > lmat;
+   Array<Vector * > lvec;
+
+   Array<int> rdof_edof;      // Map from reduced dofs to exposed dofs
+   Array<int> ess_rtdof_list;
+
+   BlockMatrix * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+#ifdef MFEM_USE_MPI
+   BlockOperator * pS = nullptr;
+   BlockOperator * pS_e = nullptr;
+   // Block HypreParMatrix for Prolongation
+   BlockOperator * pP = nullptr;
+#endif
+
+   bool Parallel() const { return parallel; }
+
+
+   // tr_idx (trace dofs indices)
+   // int_idx (interior dof indices)
+   void GetReduceElementIndicesAndOffsets(int el, Array<int> & tr_idx,
+                                          Array<int> & int_idx,
+                                          Array<int> & offsets) const;
+
+   void GetReduceElementVDofs(int el, Array<int> & rdofs) const;
+   void GetElementVDofs(int el, Array<int> & vdofs) const;
+
+
+   //  S = A_ii - A_ib (A_bb)^{-1} A_bi.
+   //  y = y_i - A_ib (A_bb)^{-1} y_b
+   void GetLocalShurComplement(int el, const Array<int> & tr_idx,
+                               const Array<int> & int_idx,
+                               const DenseMatrix & elmat, const Vector & elvect,
+                               DenseMatrix & rmat, Vector & rvect);
+
+   void ComputeOffsets();
+
+   void BuildProlongation();
+#ifdef MFEM_USE_MPI
+   void BuildParallelProlongation();
+#endif
+
+   // ess_tdof list for each space
+   Array<Array<int> *> ess_tdofs;
+   void FillEssTdofLists(const Array<int> & ess_tdof_list);
+
+   void ConformingAssemble(int skip_zeros);
+
+   /** Restrict a marker Array on the true FE spaces dofs to a marker Array on
+    the reduced/trace true FE spaces dofs. */
+   void ConvertMarkerToReducedTrueDofs(Array<int> & tdof_marker,
+                                       Array<int> & rtdof_marker);
+public:
+
+   BlockStaticCondensation(Array<FiniteElementSpace *> & fes_);
+
+   ~BlockStaticCondensation();
+
+   void SetSpaces(Array<FiniteElementSpace*> & fes_);
+
+   void Init();
+
+   /** Assemble the contribution to the Schur complement from the given
+       element matrix 'elmat'; save the other blocks internally: A_bb_inv, A_bi,
+       and A_bi. */
+
+   void AssembleReducedSystem(int el, DenseMatrix &elmat,
+                              Vector & elvect);
+
+   /// Finalize the construction of the Schur complement matrix.
+   void Finalize(int skip_zeros = 0);
+
+   /// Determine and save internally essential reduced true dofs.
+   void SetEssentialTrueDofs(const Array<int> &ess_tdof_list);
+
+   /// Eliminate the given reduced true dofs from the Schur complement matrix S.
+   void EliminateReducedTrueDofs(const Array<int> &ess_rtdof_list,
+                                 Matrix::DiagonalPolicy dpolicy);
+
+   void EliminateReducedTrueDofs(Matrix::DiagonalPolicy dpolicy);
+
+   bool HasEliminatedBC() const
+   {
+#ifndef MFEM_USE_MPI
+      return S_e;
+#else
+      return S_e || pS_e;
+#endif
+
+   }
+
+   /// Return the serial Schur complement matrix.
+   BlockMatrix &GetMatrix() { return *S; }
+
+   /// Return the eliminated part of the serial Schur complement matrix.
+   BlockMatrix &GetMatrixElim() { return *S_e; }
+
+#ifdef MFEM_USE_MPI
+   /// Return the parallel Schur complement matrix.
+   BlockOperator &GetParallelMatrix() { return *pS; }
+
+   /// Return the eliminated part of the parallel Schur complement matrix.
+   BlockOperator &GetParallelMatrixElim() { return *pS_e; }
+
+   void ParallelAssemble(BlockMatrix *m);
+#endif
+
+   void FormSystemMatrix(Operator::DiagonalPolicy diag_policy);
+
+   /** Restrict a solution vector on the full FE space dofs to a vector on the
+       reduced/trace true FE space dofs. */
+   void ReduceSolution(const Vector &sol, Vector &sc_sol) const;
+
+   /** @brief Set the reduced solution `X` and r.h.s `B` vectors from the full
+    linear system solution `x` and r.h.s. `b` vectors.
+
+      This method should be called after the internal reduced essential dofs
+      have been set using SetEssentialTrueDofs() and both the Schur complement
+      and its eliminated part have been finalized. */
+   void ReduceSystem(Vector &x, Vector &X, Vector &B,
+                     int copy_interior = 0) const;
+
+   /** Restrict a list of true FE space dofs to a list of reduced/trace true FE
+    space dofs. */
+   void ConvertListToReducedTrueDofs(const Array<int> &ess_tdof_list,
+                                     Array<int> &ess_rtdof_list) const;
+
+   /** Given a solution of the reduced system 'sc_sol' and the RHS 'b' for the
+       full linear system, compute the solution of the full system 'sol'. */
+   void ComputeSolution(const Vector &sc_sol, Vector &sol) const;
+
+};
+
+}
+
+#endif
@@ -112,7 +112,7 @@ static void InitBasisImpl(const FiniteElementSpace &fes,
   const bool tensor = dynamic_cast<const mfem::TensorBasisElement *>
                       (&fe) != nullptr;

-   // Init or retrieve key values
+   // Init or retreive key values
   if (basis_itr == mfem::internal::ceed_basis_map.end())
   {
      if ( tensor )
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Socratis Petrides	6744b7f895	fix path issue	2022-08-16 14:30:09 -07:00
psocratis	51d1876427	Merge branch 'dpg-complex-dev' of github.com:mfem/mfem into dpg-complex-dev	2022-08-15 17:12:29 -07:00
psocratis	cf18624920	small print edits	2022-08-15 17:12:11 -07:00
Socratis Petrides	ac5d8d5993	merge master	2022-08-15 16:27:04 -07:00
psocratis	5f38694913	several edits for comparison with DST runs	2022-07-27 06:51:45 -07:00
psocratis	967a024dbe	setting up the dpg pml proble for comparison with DST	2022-07-27 01:45:43 -07:00
psocratis	c8242b8e98	added pml terms to adjoint graph norm	2022-07-26 17:49:23 +03:00
psocratis	c91875c5a9	fix pml coeff bug in uu_dpg maxwell	2022-07-25 17:08:10 +03:00
psocratis	94112d7a7a	debugging uw pml maxwell	2022-07-25 15:33:57 +03:00
psocratis	c275856ced	dpg pml maxwell formulation	2022-07-25 13:14:51 +03:00
psocratis	ad58472eb5	acoustics preconditioner edits	2022-07-22 13:00:46 +03:00
psocratis	041ffbd32f	format print	2022-07-21 06:39:30 -07:00
psocratis	72683f55ca	minor-sample run edit	2022-07-21 02:04:38 -07:00
Socratis Petrides	0de5f1f70b	fichera 'oven' problem for maxwell	2022-05-20 18:40:09 -07:00
Socratis Petrides	18e7b9552c	fix preconditioner	2022-05-19 14:12:18 -07:00
Socratis Petrides	ee5cc572ce	adding parallel maxwell example and pcg solver	2022-05-18 18:07:36 -07:00
Socratis Petrides	1314d0b7d3	use ComplexCholeskyFactors for G	2022-05-18 15:34:11 -07:00
Socratis Petrides	95be3d9c12	resolve conflict with master	2022-05-18 15:33:24 -07:00
Socratis Petrides	3d6f1694b3	fix integration order in CurlIntegrator	2022-05-18 15:33:05 -07:00
Socratis Petrides	c454ddeb1c	merge master	2022-05-18 12:23:35 -07:00
Socratis Petrides	878bfef86b	minor fix in error calculation	2022-05-12 14:31:10 -07:00
Socratis Petrides	bb0b0132b1	fixing graph norm in 2D	2022-05-12 12:28:50 -07:00
Socratis Petrides	623fc516f0	merging 2D and 3D uw maxwell	2022-05-06 19:57:59 -07:00
Socratis Petrides	24e142b6b4	fixing BC for E in 2D maxwell	2022-05-06 19:57:32 -07:00
Socratis Petrides	4e75435e20	merging ScalarCurlintegrator to CurlIntegrator	2022-05-06 19:57:11 -07:00
Socratis Petrides	364bc0faa4	2D maxwell uw_dpg with essential BC on H	2022-05-05 20:10:27 -07:00
Socratis Petrides	03f1f5e252	fix sign bug in densmat::GradToCurl	2022-05-05 20:08:28 -07:00
Socratis Petrides	6b69120856	adding ScalarCurlIntegrator (rotated gradient) for scalar H1	2022-05-05 20:07:48 -07:00
Socratis Petrides	e591dbe8b0	fix sign bug	2022-05-04 20:35:58 -07:00
Socratis Petrides	edd4bae030	uw_maxwell for linear solution	2022-05-04 19:15:34 -07:00
Socratis Petrides	6ea11d6e65	testing new CurlIntegrator, starting UW_maxwell	2022-05-03 19:37:34 -07:00
Socratis Petrides	6e2d4bd6b8	varying options for acoustics	2022-05-03 19:36:28 -07:00
Socratis Petrides	6b40a55988	adding CurlIntegrator for H(curl) vs vector L2	2022-05-03 19:35:54 -07:00
Socratis Petrides	7931e85395	EM-diffusion in 2D with primal DPG merging 2D and 3D integrators to TangentTraceIntegrator	2022-04-22 17:45:43 -07:00
Socratis Petrides	1190685292	adding LS maxwell in 2D	2022-04-21 17:42:23 -07:00
Socratis Petrides	b0828aa597	fixing VectorFETraceTangentIntegrator, Adding AssempleMatrix2 to CurlCurlIntegrator	2022-04-21 17:42:00 -07:00
Socratis Petrides	caa6085362	primal dpg EM diffusion	2022-04-20 17:33:29 -07:00
Socratis Petrides	cef5e71448	starting primal dpg EM-diffusion example	2022-04-20 16:05:38 -07:00
Socratis Petrides	078603885e	adding tangential trace integrator	2022-04-20 16:03:05 -07:00
Socratis Petrides	8123a2e211	solving complex-acoustics with CG	2022-04-20 16:02:36 -07:00
Socratis Petrides	63e3496131	block preconditioner for indefinite helmholtz (complex case)	2022-04-15 19:23:13 -07:00
Socratis Petrides	dddef6be52	testing with block diagonal preconditioner, real indefinite	2022-04-15 17:59:00 -07:00
Socratis Petrides	8bdfcd231e	finalizing unit tests for ComplexFactors	2022-03-25 17:54:58 -07:00
Socratis Petrides	5af0b1cf8a	fixing some bugs in ComplexFactors	2022-03-25 17:54:34 -07:00
Socratis Petrides	4564b6dd5d	unit tests ref matrices	2022-03-24 15:37:45 -07:00
Socratis Petrides	7f7394c991	adding cholesky and lu factors for complex systems. Started adding unit tests	2022-03-24 15:22:34 -07:00
psocratis	e4bd8dd615	fix valgrind complaints for the parallel complex case with sc	2022-03-23 20:36:32 -07:00
psocratis	00f23ff6c7	minor valgrind issue on real example	2022-03-23 18:48:05 -07:00
psocratis	9c6553ce2e	fixing valgrind issues for the serial case	2022-03-23 13:09:24 -07:00
Socratis Petrides	a008739b3d	minor	2022-03-22 16:01:13 -07:00
Socratis Petrides	673bb91447	merge with dpg-dev	2022-03-22 14:38:39 -07:00
Socratis Petrides	d322228504	testing dpg residual with cholesky factors	2022-03-22 14:13:56 -07:00
Socratis Petrides	77ef3a2e92	Merge branch 'dpg-dev' into dpg-complex-dev	2022-03-22 12:09:23 -07:00
Socratis Petrides	2cc8c18250	fix repo check	2022-03-22 12:08:56 -07:00
Socratis Petrides	48c54c0bbc	fix repo-check	2022-03-22 12:05:22 -07:00
Socratis Petrides	57c99323e7	fix repo check	2022-03-22 12:04:26 -07:00
Socratis Petrides	1b69590800	2022	2022-03-22 10:29:58 -07:00
Socratis Petrides	fec64571ac	Merge branch 'dpg-dev' into dpg-complex-dev	2022-03-22 10:27:47 -07:00
Socratis Petrides	aa9a4887f7	change to 2022	2022-03-22 10:24:38 -07:00
Socratis Petrides	757369beb3	Merge branch 'master' into dpg-dev	2022-03-22 10:19:35 -07:00
Socratis Petrides	463b57205d	Gaussian beam - Acoustics AMR parallel example	2022-03-07 20:35:41 -08:00
Socratis Petrides	0e9e25f893	adaptive complex example with sc, and parallel sc	2022-03-07 14:50:45 -08:00
Socratis Petrides	6598a8d90b	adding parallel sc in the complex case	2022-03-07 12:34:02 -08:00
Socratis Petrides	91fabccd8b	static cond for the complex case	2022-03-05 14:38:08 -08:00
Socratis Petrides	950fb937c7	Simplifying ComplexNormalEquations interface	2022-03-04 17:56:47 -08:00
Socratis Petrides	ac9b14e8d7	Using OperatorHandle (for ComplexOperator) in FormSystemMatrix and FormLinearSystem	2022-03-04 15:53:23 -08:00
Socratis Petrides	b58ac482d3	complex valued acoustics works	2022-03-03 19:58:36 -08:00
Socratis Petrides	1ebd603e62	fix signature in ComplexNormalEquations::EliminateVDofsInRHS	2022-03-03 19:57:52 -08:00
Socratis Petrides	6967342813	minor	2022-03-03 13:06:29 -08:00
Socratis Petrides	fe16207d6a	Merge branch 'dpg-dev' into dpg-complex-dev	2022-03-01 19:24:18 -08:00
Socratis Petrides	ba94547c4d	fixing residual computation	2022-03-01 19:20:44 -08:00
Socratis Petrides	02ef0f7f04	debugging complex-dpg	2022-02-28 11:01:11 -08:00
Socratis Petrides	fa3a262608	starting complex acoustics/helmholtz examples for uw_dpg	2022-02-25 20:19:00 -08:00
Socratis Petrides	0f5b0eb57e	place holder for par complex normal equations and sc	2022-02-25 20:15:53 -08:00
Socratis Petrides	8abd6992d8	DPG normal equations for complex case	2022-02-25 20:14:39 -08:00
Socratis Petrides	7349ece185	Merge branch 'dpg-dev' into dpg-complex-dev	2022-02-25 17:47:42 -08:00
Socratis Petrides	58f7aacce9	fix acoustics graph norm	2022-02-25 17:45:49 -08:00
Socratis Petrides	af871ffa7b	merge complex-linalg	2022-02-25 13:57:50 -08:00
Socratis Petrides	369d55c7b2	Merge branch 'dpg-dev' into dpg-complex-dev	2022-02-25 13:56:25 -08:00
Socratis Petrides	3876d111cf	unit tests from complex dense matrix: Mult, MultAtB, Inverse, SystemMatrix	2022-02-23 18:17:38 -08:00
Socratis Petrides	e3085a1182	adding ComplexDenseMatrix class	2022-02-23 18:16:44 -08:00
psocratis	66e0c79b3f	small valgrind fix in the parallel case	2022-02-11 09:50:52 -08:00
psocratis	8b5f9f7720	resolving conflicts	2022-02-09 19:50:00 -08:00
psocratis	98739f8c09	fix valgrind complaints in the serial case	2022-02-09 19:41:48 -08:00
Socratis Petrides	84129a1f83	finished Block-sc in the parallel case, uw_dpgp passes initial tests	2022-02-09 17:04:01 -08:00
Socratis Petrides	b6ea025795	cleanup, started on par sc case	2022-02-02 20:18:19 -08:00
Socratis Petrides	d528e9d63e	filling blockstaticcond destructor	2022-02-01 10:19:19 -08:00
Socratis Petrides	d18f11e6c7	blockstatic-cond passes tests for primal-dpg	2022-01-31 17:56:37 -08:00
Socratis Petrides	efeb400b27	Block sc FormSystemMatrix. Started on ReduceSystem	2022-01-28 20:01:07 -08:00
Socratis Petrides	1451c3c483	small bug-fix in assembly if doftrans is not null	2022-01-27 17:33:04 -08:00
Socratis Petrides	f12d46bdf9	Shur complement assembly and conforming assembly	2022-01-27 17:31:55 -08:00
Socratis Petrides	6dfb809f97	changing GetSubMatrix to const	2022-01-27 17:30:38 -08:00
Socratis Petrides	ef8785b852	computing indices for interior/interface local dofs for block-static-cond	2022-01-26 18:24:56 -08:00
Socratis Petrides	efd182ec4c	visualize mesh in amr for the l-shape problem	2022-01-26 18:23:12 -08:00
Socratis Petrides	e7856b982a	style	2022-01-26 10:59:38 -08:00
Socratis Petrides	4f4d1f9d2d	adding function signatures in blockstaticcond	2022-01-26 10:59:19 -08:00
Socratis Petrides	1a89d5af02	started on static condensation for DPG (block) systems	2022-01-25 19:45:40 -08:00
Socratis Petrides	c71b02a915	fix print format	2022-01-21 14:37:21 -08:00
Socratis Petrides	5576f3653e	style	2022-01-21 13:45:29 -08:00
Socratis Petrides	7f316dd07b	changing strong_dpg to use vdim>1 test space	2022-01-21 13:45:00 -08:00
Socratis Petrides	19b5ae6b9a	parallel examples acoustics uw_dpg	2022-01-21 13:44:17 -08:00
Socratis Petrides	94cd8859b2	adding adjoint graphnorm in uw_dpg for acoustics	2022-01-21 13:43:56 -08:00
Socratis Petrides	8ea8b97fc8	adding vdim>1 for test spaces in normalequations	2022-01-21 13:43:00 -08:00
Socratis Petrides	f9abc77026	clean up	2022-01-20 16:45:43 -08:00
Socratis Petrides	68e3f9aeb9	adding traditional fosls formulation for acoustics (real)	2022-01-20 16:34:51 -08:00
Socratis Petrides	08f018c299	adding uw dpg formulation for acoustics (real)	2022-01-20 16:33:58 -08:00
Socratis Petrides	69e57546d4	adding 'strong' dpg formulation for acoustics	2022-01-20 16:33:35 -08:00
Socratis Petrides	433a29c9e3	fixing comment in diffusion fosls	2022-01-20 16:32:47 -08:00
Socratis Petrides	92ee7099a4	reorganizing examples	2022-01-19 11:05:16 -08:00
Socratis Petrides	808c9cd2c4	Merge branch 'master' into dpg-dev	2022-01-19 10:10:05 -08:00
Socratis Petrides	f3cd0a10e7	minor	2022-01-17 11:30:23 -08:00
Socratis Petrides	e6bb8c8976	small bugfix in updating mesh dependent coefficient	2022-01-05 18:09:49 -08:00
Socratis Petrides	4718577810	bug-fix serial AMR in normalequations P/R Mult in RHS and Sol vector	2021-12-30 19:59:14 -08:00
Socratis Petrides	9e520183c8	Merge branch 'master' into dpg-dev	2021-12-30 09:26:52 -08:00
Socratis Petrides	95a40d7377	clean up convection-diffusion	2021-12-22 14:40:11 -08:00
Socratis Petrides	036846bcd6	adding parallel convection-diffusion with AMR example	2021-12-21 17:59:56 -08:00
Socratis Petrides	6c2968156f	Merge branch 'master' into dpg-dev	2021-12-21 10:01:43 -08:00
Socratis Petrides	9ba650f2fd	adding Erikson Johnson problem for convection-diffusion UW-DPG	2021-12-17 16:34:22 -08:00
Socratis Petrides	ee6cb39ee0	Merge branch 'pncmesh-getessvdof-fix' into dpg-dev	2021-12-16 09:46:14 -08:00
Socratis Petrides	1701a6e84c	Merge branch 'master' into dpg-dev	2021-12-16 09:45:02 -08:00
Socratis Petrides	f500e5220a	Parallel AMR for lshape	2021-12-15 18:28:31 -08:00
Socratis Petrides	e078288ecb	title fix	2021-12-15 12:40:42 -08:00
Socratis Petrides	e46c7ac0e7	fix sample run and title in the example	2021-12-15 12:39:33 -08:00
Socratis Petrides	c2b133f3c8	minor visualization edits	2021-12-15 12:37:46 -08:00
Socratis Petrides	373379fbdd	lshape mesh	2021-12-15 12:20:41 -08:00
Socratis Petrides	bfc3835a1d	adding AMR l-shape example in diffusion_uwdpg	2021-12-15 12:20:08 -08:00
Socratis Petrides	ab8578a728	add element residual computation to NormalEquations	2021-12-15 12:19:30 -08:00
Socratis Petrides	fdaaacfdf3	Started residual based error estimator. Done some refactoring	2021-12-13 17:36:52 -08:00
Socratis Petrides	5d7ebbb51a	fixed small size bug in DenseMatrix::GetSubMatrix	2021-12-13 17:35:26 -08:00
Socratis Petrides	fe0821413a	adding parallel test for uw diffusion	2021-12-09 15:16:49 -08:00
Socratis Petrides	695a5adcb2	ParNormalEquations cleanup	2021-12-09 15:16:15 -08:00
Socratis Petrides	b2dfc76f1a	minor changes in serial example	2021-12-09 15:13:54 -08:00
Socratis Petrides	76c700b41e	adding BlockOperator and BlockMatrix in OpType	2021-12-09 15:13:05 -08:00
Socratis Petrides	dc3f10fa63	Passing first tests in parallel for ParNormalEquations	2021-12-08 17:36:34 -08:00
Socratis Petrides	420bb73d6c	minor change in normal equations constructor	2021-12-08 17:36:11 -08:00
Socratis Petrides	13648768ae	removing unused code	2021-12-08 17:34:00 -08:00
Socratis Petrides	7bfda36bca	Starting ParNormalEquations -> adding class methods signatures	2021-12-07 18:48:24 -08:00
Socratis Petrides	46fc3ce9c2	remove not used code	2021-12-07 18:47:20 -08:00
Socratis Petrides	b3086ce754	Merge branch 'master' into dpg-dev	2021-12-07 13:05:01 -08:00
Socratis Petrides	d1312d354a	minor bug-fix in the adjoint graph norm	2021-12-03 20:04:49 -08:00
Socratis Petrides	16fcc3f970	Fixing P and R null diagonal blocks to act as Identity for L2 Space and AMR	2021-12-03 20:04:23 -08:00
Socratis Petrides	a6caab3bed	PartMult and AddPartMult for BlockMatrix	2021-12-03 19:53:27 -08:00
Socratis Petrides	172b5bdcc9	Modifying DPG examples to use block-diagonal preconditioners	2021-12-02 17:25:03 -08:00
Socratis Petrides	72f83c132d	Changing normalequations to use only BlockMatrix	2021-12-02 17:24:25 -08:00
Socratis Petrides	911e394d80	adding EliminateRowCols with saving Ae to BlockMatrix class	2021-12-02 17:22:36 -08:00
Socratis Petrides	4ff29e40c9	Refactoring NormalEquations assemble to use BlockMatrix	2021-12-01 19:00:07 -08:00
Socratis Petrides	dac7808875	style	2021-11-30 18:23:31 -08:00
Socratis Petrides	246fd8e6fe	started element residual calculation for AMR	2021-11-30 18:23:09 -08:00
Socratis Petrides	5e243e01f8	clean up	2021-11-30 18:22:31 -08:00
Socratis Petrides	56b83ea538	UW-DPG with AMR: fixing Block Prolongation/Restriction Operators	2021-11-30 16:41:21 -08:00
Socratis Petrides	cd7371896a	removing no longer needed test	2021-11-29 17:30:01 -08:00
Socratis Petrides	3fdd3e2450	style	2021-11-29 17:28:42 -08:00
Socratis Petrides	303ccefd83	completing the descructor	2021-11-29 17:28:17 -08:00
Socratis Petrides	48bcc332f7	fix piola map bug in NormalTraceIntegrator	2021-11-29 17:19:05 -08:00
Socratis Petrides	0c23d23874	new diffusion uw-dpg with adjoint graph norm	2021-11-29 17:17:54 -08:00
Socratis Petrides	e76cef225d	UW-DPG for diffusion works on quad meshes	2021-11-28 15:03:34 -08:00
Socratis Petrides	6f49201e62	Ultraweak-DPG for poisson prototype	2021-11-27 14:23:24 -08:00
Socratis Petrides	1cb75f7730	adding diffusion UW-DPG	2021-11-24 16:56:15 -08:00
Socratis Petrides	ef14d682c5	bug-fix ElementTrasformation	2021-11-24 16:55:45 -08:00
Socratis Petrides	54eb78e0df	adding block primal test	2021-11-23 17:13:22 -08:00
Socratis Petrides	cd1a3a4fc7	fixing minor bug in B^T G^-1 l	2021-11-23 17:12:02 -08:00
Socratis Petrides	ffb50c2416	example tests cleanup	2021-11-23 10:05:44 -08:00
Socratis Petrides	f568ca3c6e	bug fix on accumulating domain integrators	2021-11-23 10:03:13 -08:00
Socratis Petrides	c7fb058051	adding NormalTraceIntegrator and AssembleElementMatrix2 for DivDivIntegrator	2021-11-23 09:58:11 -08:00
Socratis Petrides	a4d87d936b	Adding NormalEquations Assembly for multiple spaces and integrators. Tested succesfully for primal DPG	2021-11-19 19:52:27 -08:00
Socratis Petrides	f26bd4bea1	generalizing NormalEquations assembly for multiple fespaces and integrators	2021-11-18 17:20:30 -08:00
Socratis Petrides	6c001c3f9a	adding example test for primal DPG	2021-11-15 17:57:09 -08:00
Socratis Petrides	82573af316	new Primal DPG reproduces ex8	2021-11-15 17:56:16 -08:00
Socratis Petrides	4bfb2d62d9	first primal dpg test runs	2021-11-15 13:41:57 -08:00
Socratis Petrides	0bf7a715db	minor cleanup	2021-11-12 20:04:01 -08:00
Socratis Petrides	ad68c8f245	Fixing doc complaints	2021-11-12 18:52:48 -08:00
Socratis Petrides	0b430fa5f6	Started NormalEquationsWeakFormulation::Assemble	2021-11-12 18:49:06 -08:00
Socratis Petrides	238d1921c7	adding TraceIntegrator and AssembleTraceFaceMatrix	2021-11-12 18:48:05 -08:00
Socratis Petrides	a25783e3b4	Simplifying P and R in BlockBilinearForm	2021-11-04 09:43:40 -07:00
Socratis Petrides	3aa8d49ca8	started implementation of trace_elem_integrators for DPG	2021-11-03 18:29:08 -07:00
Socratis Petrides	4ed6a4a014	fixing minor bug in reference BlockFOSLS case	2021-11-03 15:11:20 -07:00
Socratis Petrides	08f910343b	AMR for reference Block FOSLS	2021-11-03 11:16:59 -07:00
Socratis Petrides	9fb0340206	fixing minor bug in P/R for AMR with blockforms	2021-11-03 09:03:18 -07:00
Socratis Petrides	db40e8125e	Setting Prolongation/Restriction for BlockBilinearForm. Tested for 2D AMR	2021-11-02 18:56:07 -07:00
Socratis Petrides	3e46fce65f	simplifying BlockBilinearFrom::Assembly()	2021-11-02 17:53:50 -07:00
Socratis Petrides	f649ed6b62	investigating possible bug in high order 3D runs wrt to essential BC elimination (example 1 hexa mesh)	2021-11-01 17:34:59 -07:00
Socratis Petrides	a1a3a51d86	simplifying offset calculation wrt neg orientation	2021-11-01 17:34:05 -07:00
Socratis Petrides	6728fc7eda	make style	2021-11-01 16:44:38 -07:00
Socratis Petrides	71e9d4e1d9	poisson_fosls - reference implementation	2021-11-01 16:44:00 -07:00
Socratis Petrides	7feb560f9b	testing blockbilinearform with FOSLS poisson	2021-11-01 16:43:28 -07:00
Socratis Petrides	aa9eadfcdf	fixing orientation (sign) bug in blockbilinearform assemble	2021-11-01 16:42:59 -07:00
Socratis Petrides	82122f05cf	testing block integrator by borrowing (bi)linearIntegrators	2021-11-01 16:42:20 -07:00
Socratis Petrides	2d64fa2162	fix bug in blocklinearform assemble	2021-11-01 16:41:21 -07:00
Socratis Petrides	8041734755	adding DenseMatrix::SetSubMatrix functions	2021-11-01 16:40:24 -07:00
Socratis Petrides	523891f05f	adding BlockLinearForm (assemble) with testblocklinearinteg	2021-10-29 18:32:31 -07:00
Socratis Petrides	b3fc46b6f2	Merge branch 'master' into dpg-dev	2021-10-29 16:43:38 -07:00
Socratis Petrides	9271494723	Merge branch 'master' into dpg-dev	2021-10-27 08:45:53 -07:00
Socratis Petrides	f2dbcecb92	debugging BlockBilinearForm::Assemble()	2021-10-25 16:47:32 -07:00
Socratis Petrides	f2af9748c4	fixing offset computation in BlockBilinearForm::Assemble()	2021-10-22 19:39:51 -07:00
Socratis Petrides	9813dd7722	fix doxygen complaint	2021-10-22 19:28:52 -07:00
Socratis Petrides	3806e68ed1	adding new classes (block(bi)linearForms(integ) in support for DPG methods	2021-10-22 17:58:52 -07:00