testing dpg residual with cholesky factors

fix repo check
fix repo-check
2022-03-22 14:13:56 -07:00 · 2022-03-22 12:08:56 -07:00 · 2022-03-22 12:05:22 -07:00 · 2022-03-22 10:24:38 -07:00 · 2022-03-22 10:19:35 -07:00 · 2022-03-03 13:06:29 -08:00
270 changed files with 11498 additions and 18938 deletions
@@ -1,13 +1,10 @@
 version: '{build}'

 # https://www.appveyor.com/docs/build-environment/#build-worker-images
-image: Visual Studio 2019
+image: Visual Studio 2017

 install:

-# Start from outside clone directory
- cd ..
-
 # Install MS-MPI
 - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
 - MSMpiSetup.exe -unattend
@@ -18,11 +15,6 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

-# Set MSMPI environment variables needed for CMake detection
- set MSMPI_LIB32=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86
- set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
- set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
-
 # Install METIS, use a mirror because the original source server is not always
 # up. Original url:
 #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
@@ -33,24 +25,20 @@ install:
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
 - cmake --build build
- set METIS_PATH=%cd%
 - cd ..

 # Install hypre
 - ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
- cmake -H. -Bbuild
+- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
 - cmake --build build
 - cmake --build build --target install
 - cd ../..

-# Return to clone directory
- cd %APPVEYOR_BUILD_FOLDER%
-
 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -1,80 +0,0 @@
-name: Build Deploy Container
-
-on:
-
-  # Always have a base image ready to go - this is a nightly build
-  schedule:
-    - cron: 0 3 * * *
-
-  # Allow manual trigger of a build
-  workflow_dispatch:
-
-  # On push to main we build and deploy images
-  push:
-    branches:
-      - master
-
-  # Publish packages on release
-  release:
-    types: [published]
-
-jobs:
-  build:
-    permissions:
-      packages: write
-    strategy:
-      fail-fast: false
-      matrix:
-
-        # Dockerfiles to build, a matrix supports future expanded builds
-        container: [["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu-base"]]
-
-    runs-on: ubuntu-latest
-    name: Build
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Make Space For Build
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-
-       # It's easier to reference named variables than indexes of the matrix
-      - name: Set Environment
-        env:
-          dockerfile: ${{ matrix.container[0] }}
-          uri: ${{ matrix.container[1] }}
-        run: |
-          echo "dockerfile=$dockerfile" >> $GITHUB_ENV
-          echo "uri=$uri" >> $GITHUB_ENV
-
-      - name: Pull previous layers for cache
-        run: docker pull ${uri}:latest || echo "No container to pull"
-
-      - name: Build Container
-        run: |
-           container=$uri:latest
-           docker build -f ${dockerfile} -t ${container} .
-           echo "container=$container" >> $GITHUB_ENV
-
-      - name: GHCR Login
-        if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Deploy
-        if: (github.event_name != 'pull_request')
-        run: |
-            docker push ${container}
-
-      - name: Tag and Push Release
-        if: (github.event_name == 'release')
-        run: |
-            tag=${GITHUB_REF#refs/tags/}
-            echo "Tagging and releasing ${uri}:${tag}"
-            docker tag ${uri}:latest ${uri}:${tag}
-            docker push ${uri}:${tag}
@@ -27,7 +27,6 @@ on:
      - master
      - next
  pull_request:
-  workflow_dispatch:

 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -47,18 +46,11 @@ jobs:
  builds-and-tests:
    strategy:
      matrix:
-        os: [ubuntu-20.04, macos-10.15, windows-2022]
+        os: [ubuntu-18.04, macos-10.15]
        target: [dbg, opt]
        mpi: [seq, par]
-        build-system: [make, cmake]
+        build-system: [make]
        hypre-target: [int32]
-        exclude:
-          - os: ubuntu-20.04
-            build-system: cmake
-          - os: macos-10.15
-            build-system: cmake
-          - os: windows-2022
-            build-system: make
        # 'include' allows us to:
        # - Add a variable to all jobs without creating a new matrix dimension.
        #   Codecov is defined that way.
@@ -72,15 +64,13 @@ jobs:
            codecov: NO
          - target: opt
            codecov: YES
-          - os: windows-2022
-            codecov: NO
-          - os: ubuntu-20.04
+          - os: ubuntu-18.04
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
-          - os: ubuntu-20.04
+          - os: ubuntu-18.04
            target: opt
            codecov: NO
            mpi: par
@@ -112,13 +102,13 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
      run: |
        sudo apt-get install lcov

@@ -139,10 +129,6 @@ jobs:
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install lcov

-    - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
-      uses: mpi4py/setup-mpi@v1.0.3
-
    # Get Hypre through cache, or build it.
    # Install will only run on cache miss.
    - name: cache hypre
@@ -151,67 +137,36 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
-      uses: mfem/github-actions/build-hypre@v2.2
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.0
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
        target: ${{ matrix.hypre-target }}
-        build-system: make
-
-    - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
-      uses: mfem/github-actions/build-hypre@v2.2
-      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: ${{ matrix.hypre-target }}
-        build-system: cmake

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
+      if: matrix.mpi == 'par'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0

    - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.0
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

-    - name: cache vcpkg (Windows)
-      id: vcpkg-cache
-      uses: actions/cache@v3
-      with:
-        path: vcpkg_cache
-        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
-
-    - name: prepare binary cache location
-      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
-      run: |
-        mkdir -p vcpkg_cache
-
-    - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
-      env:
-        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-      run: |
-        vcpkg install metis --triplet=x64-windows-static
-
    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v2.2
-      env:
-        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+      uses: mfem/github-actions/build-mfem@v2.1
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -221,8 +176,6 @@ jobs:
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
        mfem-dir: ${{ env.MFEM_TOP_DIR }}
-        config-options: ${{ env.MFEM_EXTRA_CONFIG }}
-        library-only: ${{ matrix.target == 'dbg' }}

    # Run checks (and only checks) on debug targets
    - name: checks
@@ -240,27 +193,10 @@ jobs:
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

-    - name: cmake checks
-      if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
+    - name: cmake unit tests
+      if: matrix.build-system == 'cmake'
      run: |
-        CTEST_CONFIG="Debug"
-        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
-      shell: bash
-
-    - name: cmake unit tests (Ubuntu 20.04)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
-      run: |
-        CTEST_CONFIG="Release"
-        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
-      shell: bash
-
-    - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
-      run: |
-        CTEST_CONFIG="Release"
-        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
-      shell: bash
+        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure

    # Code coverage (process and upload reports)
    - name: codecov
@@ -20,7 +20,6 @@ on:
      - master
      - next
  pull_request:
-  workflow_dispatch:

 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -54,11 +53,11 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.2
+      uses: mfem/github-actions/build-hypre@v2.0
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
@@ -69,18 +68,18 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.2
+      uses: mfem/github-actions/build-metis@v2.0
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.2
+      uses: mfem/github-actions/build-mfem@v2.0
      with:
        os: ${{ runner.os }}
        target: opt
@@ -16,28 +16,16 @@ permissions:

 on:
  push:
-  pull_request:
-  workflow_dispatch:
-
-# This workflow is run on pushes to any branch in the MFEM repo (with or without
-# PRs), as well as on updates to PRs from forks. In particular, we do not
-# duplicate work by running on both pushes and updates to local PRs. We do that
-# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
-# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
-# github.repository").

 jobs:
  file-headers-check:
    runs-on: ubuntu-18.04
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+
    steps:
    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.9.0
      with:
        access_token: ${{ github.token }}
-
    - name: checkout mfem
      uses: actions/checkout@v2

@@ -61,10 +49,7 @@ jobs:
      continue-on-error: true

    - name: wrap-up
-      if: |
-        steps.copyright.outcome != 'success' ||
-        steps.license.outcome != 'success' ||
-        steps.release.outcome != 'success'
+      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
      run: |
        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
          echo "copyright check failed, unroll log for details"
@@ -79,9 +64,7 @@ jobs:

  code-style:
    runs-on: ubuntu-18.04
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -96,9 +79,7 @@ jobs:

  documentation:
    runs-on: ubuntu-18.04
-    if: |
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -113,12 +94,9 @@ jobs:
        ./runtest documentation

  branch-history:
-    if: |
-      github.ref != 'refs/heads/next' &&
-      github.ref != 'refs/heads/master' &&
-      (github.event_name == 'push' ||
-       github.event.pull_request.head.repo.full_name != github.repository)
+    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
    runs-on: ubuntu-18.04
+
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -240,7 +240,6 @@ miniapps/navier/navier_kovasznay_vs
 miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
-miniapps/navier/navier_turbchan
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output

@@ -261,14 +260,12 @@ miniapps/performance/sol.*

 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
-miniapps/shifted/ParaViewLSF
 miniapps/shifted/extrapolate
 miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
 miniapps/shifted/ParaViewDiffusion
-miniapps/shifted/lsf_integral

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -45,5 +45,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -23,7 +23,7 @@ allocate_resource:
  stage: allocate_resource
  script:
    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h
  needs: [setup]

@@ -23,7 +23,7 @@ allocate_resource:
  stage: allocate_resource
  script:
    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=45 --no-shell --job-name=${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h

 # GitLab jobs for the Quartz machine at LLNL
@@ -8,42 +8,6 @@
                               https://mfem.org


-Version 4.4.1 (development)
-===========================
- Added example for body-fitted volumetric and shape integration using the 
-  Algoim library.
-
- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
-  spatial Gaussian white noise.
-
- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
-  See fem/estimators.hpp.
-
- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
-  which provides parallel non-conforming, non-matching, variational, volumetric
-  mesh information transfer. With ParMortarAssember, fields can be exchanged
-  between arbitrarily distributed and unrelated finite element meshes in a
-  variationally consistent way.
-
- Added full assembly and device support for several LinearForm integrators:
-  * DomainLF: (f, v)
-  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
-  * DomainLFGrad: (f, grad(v))
-  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
-
- Add a new example code, Example 33/33p, to demonstrate the solution of
-  spectral fractional PDEs with MFEM.
-
- Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
-
- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration.
-
- Added Windows 2022 CI testing with GitHub actions.
-
- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
-
 Version 4.4, released on March 21, 2022
 =======================================

@@ -164,9 +128,6 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
  formatting. See the "make style" target.

- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
-
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -51,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.4.1)
+set(${PROJECT_NAME}_VERSION 4.4.0)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -268,7 +268,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
  if(APPLE)
    # On macOS, the compiler needs additional help to find the <omp.h> header.
    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
+    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
  endif(APPLE)
 endif()

@@ -379,21 +379,6 @@ if (MFEM_USE_PUMI)
  endif()
 endif()

-# Moonolith
-if(MFEM_USE_MOONOLITH)
-  find_package(ParMoonolith REQUIRED)
-  if(ParMoonolith_FOUND)
-    get_target_property(
-      MOONOLITH_INCLUDE_DIRS ParMoonolith::par_moonolith
-      INTERFACE_INCLUDE_DIRECTORIES)
-    set(MOONOLITH_FOUND TRUE)
-    set(MOONOLITH_LIBRARIES ParMoonolith::par_moonolith)
-    message(
-      STATUS
-      "MOONOLITH_LIBRARIES=${MOONOLITH_LIBRARIES}, MOONOLITH_INCLUDE_DIRS=${MOONOLITH_INCLUDE_DIRS}")
-  endif()
-endif()
-
 # HiOp optimizer
 if (MFEM_USE_HIOP)
  find_package(HIOP REQUIRED)
@@ -431,11 +416,6 @@ if (MFEM_USE_CALIPER)
  find_package(Caliper REQUIRED)
 endif()

-# Algoim
-if (MFEM_USE_ALGOIM)
-  find_package(Algoim REQUIRED)
-endif()
-
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
@@ -479,7 +459,7 @@ set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM)
+    MPI_CXX HIP HIPSPARSE)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -494,6 +474,7 @@ endforeach(TPL)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
+include_directories(${TPL_INCLUDE_DIRS})

 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
@@ -515,11 +496,6 @@ message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
 set(SOURCES "")
 set(HEADERS "")
 set(MFEM_SOURCE_DIRS general linalg mesh fem)
-
-if(MFEM_USE_MOONOLITH)
-  set(MFEM_SOURCE_DIRS ${MFEM_SOURCE_DIRS} fem/moonolith)
-endif()
-
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
  add_subdirectory(${DIR})
 endforeach()
@@ -550,15 +526,6 @@ target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
  target_link_libraries(mfem PRIVATE ws2_32)
 endif()
-if (MSVC)
-  target_compile_options(mfem PUBLIC "/wd4819")
-endif()
-message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
-target_include_directories(mfem
-  PUBLIC
-    ${TPL_INCLUDE_DIRS}
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")

@@ -687,10 +654,6 @@ set(INSTALL_LIB_DIR lib
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
  CACHE PATH "Relative path for installing cmake config files.")

-target_include_directories(mfem
-  PUBLIC
-    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
-
 # The 'install' target will not depend on 'all'.
 # set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)

@@ -724,7 +687,7 @@ endif()
 if (MFEM_USE_CEED)
  install(DIRECTORY ${MFEM_SOURCE_DIRS}
    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/integrators/*/*.h")
+    FILES_MATCHING PATTERN "fem/ceed/*.h")
 endif()

 # Install ${HEADERS}
@@ -755,8 +718,14 @@ export(TARGETS ${PROJECT_NAME}
 # TODO: How do we register the install-tree? Replacing the build-tree?
 export(PACKAGE ${PROJECT_NAME})

+# Extract the include directories required to use MFEM
+get_target_property(MFEM_TPL_INCLUDE_DIRS mfem INCLUDE_DIRECTORIES)
+if (NOT MFEM_TPL_INCLUDE_DIRS)
+  set(MFEM_TPL_INCLUDE_DIRS "")
+endif()
+
 # This is the build-tree version
-set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${PROJECT_BINARY_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}/MFEMConfig.cmake
@@ -764,7 +733,7 @@ configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR)

 # This is the version that will be installed
-set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${MFEM_TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${INSTALL_LIB_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/MFEMConfig.cmake
@@ -119,7 +119,6 @@ The MFEM source code has the following structure:
  │   ├── ceed
  │   ├── fe
  │   ├── qinterp
-  │   ├── moonolith
  │   └── tmop
  ├── general
  ├── linalg
@@ -471,14 +471,6 @@ MFEM_USE_CODIPACK = YES/NO
  Enable automatic differentiation using the CoDiPack library.
  www.scicomp.uni-kl.de/codi/

-MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical 
-  methods and C++ algorithms for working with implicitly-defined geometry and 
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
-  provides interface to Algoim v1. Thus, to check out the specific state use: 
-  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
-  https://algoim.github.io
-
 MFEM_USE_ADFORWARD = YES/NO
  Enable forward mode for AD packages. This option is valid
  only if the AD package supports two modes (backward/forward).
@@ -528,16 +520,6 @@ MFEM_USE_MKL_CPARDISO = YES/NO
   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
   OpenMP capabilities are disabled at link time.

-MFEM_USE_MOONOLITH = YES/NO
-   Enables the ParMoonolith interface for parallel non-conforming, non-matching,
-   variational, volumetric mesh information transfer. It requires the variable
-   MOONOLITH_DIR=<path to installation> to be defined in the environment in
-   order to be used with the Makefile. Makefile users are also required to
-   install moonolith using the command `make install_all`, see
-   https://bitbucket.org/zulianp/par_moonolith for details.
-   Although Moonolith is an MPI-based library, both serial (MFEM_USE_MPI=NO) and
-   parallel (MFEM_USE_MPI=YES) versions of MFEM are supported.
-
 MFEM_USE_CALIPER = YES/NO
   Enables the interface to Caliper. Caliper is a library to integrate
   performance profiling capabilities into applications. To use Caliper,
@@ -746,22 +728,6 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.7.

- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
-  headers so it just needs to be downloaded at the same level as MFEM. Download
-  the specific version we use as:
-  "git clone https://github.com/algoim/algoim.git;
-   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
-  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
-  "ln -s blitz-1.0.2 blitz".
-  Build Blitz using CMake as:
-  "cmake . -DCMAKE_INSTALL_PREFIX=.; make lib; make install"
-  URL: https://github.com/blitzpp/blitz/archive/refs/tags/1.0.2.tar.gz
-  Options: BLITZ_OPT, BLITZ_LIB
-  Versions: BLITZ = 1.0.2
-
-
-
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
@@ -793,11 +759,6 @@ The specific libraries and their options are:
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
  Versions: RAJA >= 0.14.0.

- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
-  URL: https://bitbucket.org/zulianp/par_moonolith
-  Options: MOONOLITH_DIR
-  Versions: MOONOLITH >= 1.1.0.
-
 - Caliper (optional), used when MFEM_USE_CALIPER = YES.
  URL: https://github.com/LLNL/Caliper
  Options: CALIPER_DIR
@@ -971,7 +932,6 @@ MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
-MFEM_USE_MOONOLITH
 MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
@@ -1030,7 +990,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
- - MOONOLITH
 - CALIPER
 - FMS
 - BENCHMARK
@@ -7,24 +7,21 @@

                               https://mfem.org

-[MFEM](https://mfem.org) is a modular parallel C++ library for finite element
-methods. Its goal is to enable high-performance scalable finite element
-discretization research and application development on a wide variety of
-platforms, ranging from laptops to supercomputers.
+MFEM is a modular parallel C++ library for finite element methods. Its goal is
+to enable high-performance scalable finite element discretization research and
+application development on a wide variety of platforms, ranging from laptops to
+supercomputers.

 We welcome contributions and feedback from the community. Please see the file
-[CONTRIBUTING.md](CONTRIBUTING.md) for additional details about our development
-process.
+CONTRIBUTING.md for additional details about our development process.

-* For building instructions, see the file [INSTALL](INSTALL), or type "make help".
+* For building instructions, see the file INSTALL, or type "make help".

-* Copyright and licensing information can be found in files [LICENSE](LICENSE) and [NOTICE](NOTICE).
+* Copyright and licensing information can be found in files LICENSE and NOTICE.

 * The best starting point for new users interested in MFEM's features is to
  review the examples and miniapps at https://mfem.org/examples.

-* Instructions for learning with Docker are in [config/docker](config/docker).
-
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support
@@ -61,16 +58,12 @@ solvers from the hypre library. Comprehensive support for other external
 packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
 additional linear and nonlinear solvers, preconditioners, time integrators, etc.

-For examples of using MFEM, see the [examples/](examples) and [miniapps/](miniapps)
-directories, as well as the OpenGL visualization tool GLVis which is available
-at https://glvis.org.
-
-## License
+For examples of using MFEM, see the examples/ and miniapps/ directories, as well
+as the OpenGL visualization tool GLVis which is available at https://glvis.org.

 MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See [LICENSE](LICENSE) and [NOTICE](NOTICE) for
-details.
+must be made under this license. See LICENSE and NOTICE for details.

-SPDX-License-Identifier: BSD-3-Clause <br>
-LLNL Release Number: LLNL-CODE-806117 <br>
+SPDX-License-Identifier: BSD-3-Clause
+LLNL Release Number: LLNL-CODE-806117
 DOI: 10.11578/dc.20171025.1248
@@ -54,11 +54,9 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
-set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
 set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
 set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
-set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)

@@ -128,9 +128,6 @@
 // Enable MFEM functionality based on the PUMI library
 #cmakedefine MFEM_USE_PUMI

-// Enable MFEM functionality based on the Moonolith library
-#cmakedefine MFEM_USE_MOONOLITH
-
 // Enable MFEM functionality based on the HiOp library
 #cmakedefine MFEM_USE_HIOP

@@ -160,9 +157,6 @@
 // Enable MFEM functionality based on the Caliper library
 #cmakedefine MFEM_USE_CALIPER

-// Enable MFEM functionality based on the Algoim library
-#cmakedefine MFEM_USE_ALGOIM
-
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - ALGOIM_FOUND
-#   - ALGOIM_LIBRARIES
-#   - ALGOIM_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Algoim ALGOIM ALGOIM_DIR
-       	"include" "algoim_quad.hpp"
-       	"" ""
-	"Paths to headers required by Algoim."
-	"Libraries required by Algoim.")
@@ -1,22 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - BLITZ_FOUND
-#   - BLITZ_LIBRARIES
-#   - BLITZ_INCLUDE_DIRS
-
-include(MfemCmakeUtilities)
-mfem_find_package(Blitz BLITZ BLITZ_DIR
-       	"include" "blitz/blitz.h"
-       	"lib" "blitz"
-  	"Paths to headers required by Blitz."
-       	"Libraries required by Blitz.")
@@ -15,5 +15,5 @@
 #   - GSLIB_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs
+mfem_find_package(gslib GSLIB GSLIB_DIR "include" gslib.h "lib" gs
  "Paths to headers required by GSLIB." "Libraries required by GSLIB.")
@@ -893,8 +893,7 @@ function(mfem_export_mk_files)
      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM)
+      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -978,11 +977,9 @@ function(mfem_export_mk_files)
      string(REGEX REPLACE "^SCOREC::" "" libname ${pumilib})
      string(FIND "${pumilib}" ".a" staticlib)
      string(FIND "${pumilib}" ".so" sharedlib)
-      string(FIND "${pumilib}" ".dylib" dynamiclib)
      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFAULT_PATH)
      if (NOT "${sharedlib}" MATCHES "-1" OR
-          NOT "${staticlib}" MATCHES "-1" OR
-          NOT "${dynamiclib}" MATCHES "-1"  )
+          NOT "${staticlib}" MATCHES "-1"   )
        set(MFEM_EXT_LIBS "${pumilib} ${MFEM_EXT_LIBS}")
      elseif (NOT "${lib}" MATCHES "lib-NOTFOUND")
        set(MFEM_EXT_LIBS "${lib} ${MFEM_EXT_LIBS}")
@@ -997,7 +994,7 @@ function(mfem_export_mk_files)
  foreach(lib ${TPL_LIBRARIES})
    get_filename_component(suffix ${lib} EXT)
    # handle interfaces (e.g., SCOREC::apf)
-    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
+    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
    elseif (TARGET "${lib}")
      mfem_get_target_options(${lib} CompileOpts LinkOpts)
      # Removing duplicates may lead to issues:
@@ -138,9 +138,6 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI

-// Enable Moonolith-based general interpolation between finite element spaces.
-// #define MFEM_USE_MOONOLITH
-
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP

@@ -167,9 +164,6 @@
 // Enable functionality based on the Caliper library.
 // #define MFEM_USE_CALIPER

-// Enable functionality based on the Algoim library.
-// #define MFEM_USE_ALGOIM
-
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE

@@ -58,7 +58,6 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
-MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
 MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
@@ -58,7 +58,6 @@ option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
-option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
 option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
@@ -237,11 +236,6 @@ set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
-set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
-set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
-set(ALGOIM_REQUIRED_PACKAGES "BLITZ" CACHE STRING
-    "Packages that ALGOIM depends on.")
-
 set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
    "Path to Google Benchmark")

@@ -153,12 +153,10 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_CALIPER       = NO
-MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
-MFEM_USE_MOONOLITH     = NO
 MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
@@ -389,11 +387,6 @@ ifeq ($(SLEPC_FOUND),YES)
      $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
 endif

-ifeq ($(MFEM_USE_MOONOLITH),YES)
-  include $(MOONOLITH_DIR)/config/moonolith-config.makefile
-  MOONOLITH_LIB=$(MOONOLITH_LIBRARIES)
-endif
-
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr
@@ -470,16 +463,6 @@ CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
 CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper

-# BLITZ library configuration
-BLITZ_DIR = @MFEM_DIR@/../blitz
-BLITZ_OPT = -I$(BLITZ_DIR)/include
-BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
-
-# ALGOIM library configuration
-ALGOIM_DIR = @MFEM_DIR@/../algoim
-ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT)
-ALGOIM_LIB = $(BLITZ_LIB)
-
 # BENCHMARK library configuration
 BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
 BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
@@ -1,30 +0,0 @@
-FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
-
-# docker build -t ghcr.io/mfem/mfem-ubuntu-base .
-
-RUN apt-get update && \
-    apt-get install -y unzip gfortran && \
-    spack compiler find && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev
-
-# /code is the working directory for code
-WORKDIR /code
-COPY . /code
-
-# This is for a spack environment/view to install from there
-WORKDIR /opt/mfem-env
-RUN . /opt/spack/share/spack/setup-env.sh && \
-    spack env create -d . && \
-    echo "  concretization: together" >> spack.yaml && \
-    spack env activate . && \
-    spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps && \
-    spack install
-
-# ensure mfem always on various paths
-RUN cd /opt/mfem-env && \
-    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
-
-# The user will see the view on shell into the container
-WORKDIR /opt/mfem-env/.spack-env/view/
-ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
@@ -1,130 +0,0 @@
-# mfem Docker
-
-We provide a [Dockerfile](Dockerfile) to build an ubuntu base image. You can use
-this image for a demo of using mfem! 🎉️
-
-Updated containers are built and deployed on merges to the main branch and releases.
-If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
-
-### Usage
-
-Here is how to build the container. Note that we build so it belongs to the same
-namespace as the repository here. "ghcr.io" means "GitHub Container Registry" and
-is the [GitHub packages](https://github.com/features/packages) registry that supports
- Docker images and other OCI artifacts. From the root of the repository:
-
-```bash
-$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base .
-```
-
-or this directory:
-
-```bash
-$ docker build -f Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base ../../
-```
-
-### Shell
-
-To shell into a container (here is an example with ubuntu):
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
-```
-
-Off the bat, you can see mfem libraries are in your path so you can jump into development:
-
-```bash
-env | grep mfem
-```
-```bash
-PKG_CONFIG_PATH=/opt/mfem-env/.spack-env/view/lib/pkgconfig:/opt/mfem-env/.spack-env/view/share/pkgconfig:/opt/mfem-env/.spack-env/view/lib64/pkgconfig
-PWD=/opt/mfem-env
-MANPATH=/opt/mfem-env/.spack-env/view/share/man:/opt/mfem-env/.spack-env/view/man:
-CMAKE_PREFIX_PATH=/opt/mfem-env/.spack-env/view
-SPACK_ENV=/opt/mfem-env
-ACLOCAL_PATH=/opt/mfem-env/.spack-env/view/share/aclocal
-LD_LIBRARY_PATH=/opt/mfem-env/.spack-env/view/lib:/opt/mfem-env/.spack-env/view/lib64
-PATH=/opt/mfem-env/.spack-env/view/bin:/opt/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-```
-
-#### Examples and MiniApps
-
-If you want to develop a tool that _uses_ mfem, you can find the built libraries in:
-
-```
-$ ls /opt/mfem-env/.spack-env/view/
-bin  etc  include  lib  libexec  sbin  share  var
-```
-
-And yes, this is the working directory when you shell into the container!
-You can find the examples here:
-
-
-```bash
-cd share/mfem/examples
-```
-```bash
-$ ./ex0
-Options used:
-   --mesh ../data/star.mesh
-   --order 1
-Number of unknowns: 101
-   Iteration :   0  (B r, r) = 0.184259
-   Iteration :   1  (B r, r) = 0.102754
-   Iteration :   2  (B r, r) = 0.00558141
-   Iteration :   3  (B r, r) = 1.5247e-05
-   Iteration :   4  (B r, r) = 1.13807e-07
-   Iteration :   5  (B r, r) = 6.27231e-09
-   Iteration :   6  (B r, r) = 3.76268e-11
-   Iteration :   7  (B r, r) = 6.07423e-13
-   Iteration :   8  (B r, r) = 4.10615e-15
-Average reduction factor = 0.140201
-```
-
-Try running a few, and look at the associated .cpp file for the source code!
-You can also explore the "mini apps," also in share/mfem, but under miniapps.
-
-```bash
-# This is run from the examples directory
-$ cd ../miniapps
-```
-```bash
-$ ls
-CMakeLists.txt  common            meshing  nurbs        shifted  toys
-adjoint         electromagnetics  mtop     parelag      solvers
-autodiff        gslib             navier   performance  tools
-```
-
-And an example in "toys"
-
-```bash
-cd toys
-```
-```bash
-$ ./automata -no-vis
-Options used:
-   --num-steps 16
-   --rule 90
-   --no-visualization
-
-Rule:
- 111 110 101 100 011 010 001 000
-  0   1   0   1   1   0   1   0
-
-Applying rule...done.
-```
-
-Have fun!
-
-
-#### Your own App
-If you want to develop with your own code base
-(and mfem as is in the container) you can bind to somewhere else in the container (e.g., src)
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
-```
-
-In the above, we can pretend your project is in the present working directory (PWD) and we are
-binding to source. You can then use the mfem in the container for development, and if you
-want to distribute your library or app in a container, you can use the mfem container as the base.
@@ -1,86 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-3
-
-elements
-8
-1 5  0   1   10  9   3   4   13  12 
-1 5  1   2   11  10  4   5   14  13 
-1 5  3   4   13  12  6   7   16  15
-1 5  4   5   14  13  7   8   17  16
-1 5  9   10  19  18  12  13  22  21
-1 5  10  11  20  19  13  14  23  22
-1 5  12  13  22  21  15  16  25  24
-1 5  13  14  23  22  16  17  26  25
-
-#
-
-boundary
-24
-1 3 1  0  9  10 
-1 3 2  1  10 11 
-1 3 10 9  18 19 
-1 3 11 10 19 20 
-2 3 0  3  12 9
-2 3 9  12 21 18
-2 3 3  6  15 12
-2 3 12 15 24 21
-3 3 0  1  4  3
-3 3 1  2  5  4
-3 3 3  4  7  6
-3 3 4  5  8  7 
-4 3 6  7  16 15
-4 3 7  8  17 16
-4 3 15 16 25 24
-4 3 16 17 26 25 
-5 3 18 21 22 19 
-5 3 19 22 23 20 
-5 3 21 24 25 22 
-5 3 22 25 26 23 
-6 3 2  11 14 5
-6 3 11 20 23 14
-6 3 5  14 17 8 
-6 3 14 23 26 17
-
-vertices
-27
-3
-0.0 0.0 0.0
-0.5 0.0 0.0
-1.0 0.0 0.0
-0.0 0.0 0.5
-0.5 0.0 0.5
-1.0 0.0 0.5
-0.0 0.0 1.0
-0.5 0.0 1.0
-1.0 0.0 1.0
-0.0 0.5 0.0
-0.5 0.5 0.0
-1.0 0.5 0.0
-0.0 0.5 0.5
-0.5 0.5 0.5
-1.0 0.5 0.5
-0.0 0.5 1.0
-0.5 0.5 1.0
-1.0 0.5 1.0
-0.0 1.0 0.0
-0.5 1.0 0.0
-1.0 1.0 0.0
-0.0 1.0 0.5
-0.5 1.0 0.5
-1.0 1.0 0.5
-0.0 1.0 1.0
-0.5 1.0 1.0
-1.0 1.0 1.0
@@ -1,84 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-#
-
-dimension
-3
-
-elements
-8
-1 5 0  1  4  3  9  10 13 12 
-1 5 1  2  5  4  10 11 14 13
-1 5 9  10 13 12 18 19 22 21
-1 5 10 11 14 13 19 20 23 22
-1 5 3  4  7  6  12 13 16 15
-1 5 4  5  8  7  13 14 17 16
-1 5 12 13 16 15 21 22 25 24
-1 5 13 14 17 16 22 23 26 25
-
-boundary
-24
-1 3 0  1  10 9 
-1 3 1  2  11 10 
-1 3 9  10 19 18 
-1 3 10 11 20 19 
-3 3 0  3  4  1 
-3 3 1  4  5  2 
-3 3 3  6  7  4 
-3 3 4  7  8  5 
-3 3 18 19 22 21 
-3 3 19 20 23 22 
-3 3 21 22 25 24 
-3 3 22 23 26 25 
-3 3 2  5  14 11 
-3 3 11 14 23 20 
-3 3 5  8  17 14 
-3 3 14 17 26 23 
-3 3 0  9  12 3 
-3 3 9  18 21 12 
-3 3 3  12 15 6 
-3 3 12 21 24 15 
-2 3 6  15 16 7 
-2 3 7  16 17 8 
-2 3 15 24 25 16 
-2 3 16 25 26 17 
-
-vertices
-27
-3
-0.0 0.0 0.0
-0.5 0.0 0.0
-1.0 0.0 0.0
-0.0 0.0 0.5
-0.5 0.0 0.5
-1.0 0.0 0.5
-0.0 0.0 1.0
-0.5 0.0 1.0
-1.0 0.0 1.0
-0.0 0.5 0.0
-0.5 0.5 0.0
-1.0 0.5 0.0
-0.0 0.5 0.5
-0.5 0.5 0.5
-1.0 0.5 0.5
-0.0 0.5 1.0
-0.5 0.5 1.0
-1.0 0.5 1.0
-0.0 1.0 0.0
-0.5 1.0 0.0
-1.0 1.0 0.0
-0.0 1.0 0.5
-0.5 1.0 0.5
-1.0 1.0 0.5
-0.0 1.0 1.0
-0.5 1.0 1.0
-1.0 1.0 1.0
@@ -1,56 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geomety Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-# PYRAMID     = 7
-
-dimension
-3
-
-elements
-4
-1 5 0 1 4 3 6 7 10 9
-1 6 4 1 5 10 7 11
-1 7 11 7 1 5 8
-1 4 2 5 1 8
-
-boundary
-14
-1 3 0 3 4 1
-1 3 6 7 10 9
-1 3 0 6 9 3
-1 3 0 1 7 6
-1 3 3 9 10 4
-1 2 1 5 2
-1 2 1 4 5
-1 2 1 8 7
-1 2 1 2 8
-1 2 2 5 8
-1 2 5 11 8
-1 2 7 8 11
-1 2 7 11 10
-1 3 4 10 11 5
-
-vertices
-12
-3
-0 0 0
-1 0 0
-2 0 0
-0 1 0
-1 1 0
-2 1 0
-0 0 1
-1 0 1
-2 0 1
-0 1 1
-1 1 1
-2 1 1
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.4.1
+PROJECT_NUMBER         = v4.4.0

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -765,14 +765,11 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/linalg \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/moonolith \
                         @MFEM_SOURCE_DIR@/fem/fe \
-                         @MFEM_SOURCE_DIR@/fem/lor \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/amgx \
                         @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/moonolith \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
@@ -103,8 +103,6 @@ namespace mfem {
 * - <a class="el" href="ex31_8cpp_source.html">Example 31</a>: Nedelec H(curl) FEM for the definite anisotropic Maxwell problem
 * - <a class="el" href="ex31p_8cpp_source.html">Example 31p</a>: parallel Nedelec H(curl) FEM for the definite anisotropic Maxwell problem
 * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
- * - <a class="el" href="ex33_8cpp_source.html">Example 33</a>: nodal H1 FEM for the fractional Laplacian problem
- * - <a class="el" href="ex33p_8cpp_source.html">Example 33p</a>: parallel nodal H1 FEM for the fractional Laplacian problem
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -39,7 +39,6 @@ list(APPEND ALL_EXE_SRCS
  ex29.cpp
  ex30.cpp
  ex31.cpp
-  ex33.cpp
  )

 if (MFEM_USE_MPI)
@@ -76,7 +75,6 @@ if (MFEM_USE_MPI)
    ex30p.cpp
    ex31p.cpp
    ex32p.cpp
-    ex33p.cpp
    )
 endif()

@@ -201,7 +199,3 @@ endif()
 if (MFEM_USE_SUPERLU)
  add_subdirectory(superlu)
 endif()
-
-if(MFEM_USE_MOONOLITH)
-  add_subdirectory(moonolith)
-endif()
@@ -0,0 +1,294 @@
+//                 MFEM FOSLS acoustics Example
+//
+// Compile with: make fosls
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// FOSLS:
+//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
+
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// -------------------------------------------------------------------
+// |   |            p           |            u             |   RHS    | 
+// -------------------------------------------------------------------
+// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
+// |   |                        |                          |          |
+// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
+
+// where (q,v) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int sr = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&sr, "-sr", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   for (int i = 0; i < sr; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
+   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = H1fes;
+   fespaces[1] = RTfes;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
+   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+   ConstantCoefficient omeg2(omega*omega);
+
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+
+   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
+   SumIntegrator * integ00 = new SumIntegrator();
+   integ00->AddIntegrator(new DiffusionIntegrator(one));
+   integ00->AddIntegrator(new MassIntegrator(omeg2));
+   blfi(0,0) = integ00;
+
+   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
+   SumIntegrator * integ01 = new SumIntegrator();
+#ifdef DEFINITE
+   // -ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
+#else   
+   // ω (∇⋅u,q)
+   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
+#endif
+   // - ω (u, ∇ q)
+   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
+   blfi(0,1) = integ01;
+
+   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
+   SumIntegrator * integ10 = new SumIntegrator();
+#ifdef DEFINITE
+   // - ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
+#else
+   // ω (p,∇⋅v)
+   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
+#endif
+   // - ω (∇ p,v)
+   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
+   blfi(1,0) = integ10;
+
+   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
+   SumIntegrator * integ11 = new SumIntegrator();
+   integ11->AddIntegrator(new DivDivIntegrator(one));
+   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
+   blfi(1,1) = integ11;
+
+
+   BlockLinearForm b(fespaces);
+   Array<LinearFormIntegrator * > lfi(2);
+   // ± ω (f,q)
+   FunctionCoefficient f_rhs(rhs_func);
+#ifdef DEFINITE  
+   ProductCoefficient w_f(omeg,f_rhs);
+#else   
+   ProductCoefficient w_f(negomeg,f_rhs);
+#endif   
+   // lfi[0] = new DomainLFIntegrator(w_f);
+   lfi[0] = new DomainLFIntegrator(w_f);
+
+
+   //  -(f,∇⋅v)
+   ProductCoefficient neg_f(negone,f_rhs);
+   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
+   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(H1fes);   
+
+   p_gf.MakeRef(H1fes,x,0);
+   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+   p_gf.ProjectCoefficient(p_ex);
+   pex_gf.ProjectCoefficient(p_ex);
+
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+   u_gf = 0.;
+
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-10);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   p_gf.MakeRef(H1fes,x,0);
+   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
+
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << p_gf <<
+                "window_title 'Numerical p' "
+                << flush;
+      // socketstream sols_sock(vishost, visport);
+      // sols_sock.precision(8);
+      // sols_sock << "solution\n" << mesh << u_gf <<
+               //  "window_title 'Numerical sigma' "
+               //  << flush;
+
+      socketstream solex_sock(vishost, visport);
+      solex_sock.precision(8);
+      solex_sock << "solution\n" << mesh << pex_gf <<
+                    "window_title 'Exact p' "
+                    << flush;          
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
@@ -10,19 +10,17 @@
 # CONTRIBUTING.md for details.

 # Use the MFEM build directory
-MFEM_DIR ?= ../..
-MFEM_BUILD_DIR ?= ../..
-SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/moonolith/,)
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
 CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-# Use the MFEM install directory
-# MFEM_INSTALL_DIR = ../../mfem
-# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

-SEQ_EXAMPLES = ex1
-PAR_EXAMPLES = ex1p ex2p
+SEQ_EXAMPLES = fosls uw_dpg  strong_dpg
+PAR_EXAMPLES = uw_dpgp
+
 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -31,7 +29,7 @@ endif

 .SUFFIXES:
 .SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build
+.PHONY: all clean clean-build clean-exec

 # Remove built-in rule
 %: %.cpp
@@ -40,39 +38,22 @@ endif
 %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
 	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)

-all: $(EXAMPLES)
-
-ifeq ($(MFEM_USE_MOONOLITH),NO)
-$(EXAMPLES):
-	$(error MFEM is not configured with MOONOLITH)
-endif
+all: $(EXAMPLES) 

 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)

 # Testing: Parallel vs. serial runs
-RUN_MPI_NP = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
-RUN_MPI = $(RUN_MPI_NP) $(MFEM_MPI_NP)
-SERIAL_NAME := Serial MOONOLITH example
-PARALLEL_NAME := Parallel MOONOLITH example
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
 %-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
 %-test-seq: %
-	@$(call mfem-test,$<,, $(SERIAL_NAME))
+	@$(call mfem-test,$<,, Serial example)

-# Testing: Example-specific execution options:
-ex1-test-par: ex1
-	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
-
-# Testing: "test" target and mfem-test* variables are defined in config/test.mk
-
-# Generate an error message if the MFEM library is not built and exit
-$(MFEM_LIB_FILE):
-	$(error The MFEM library is not built)
-
-clean: clean-build
+clean: clean-build clean-exec 

 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints

+clean-exec:
@@ -0,0 +1,271 @@
+//                 MFEM DPG_strong acoustics Example
+//
+// Compile with: make strong_dpg
+//
+// Definite/Indefinite Helmholtz
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// Strong DPG formulation
+// (p,u) ∈ H^1(Ω) × H(div,Ω)
+// 
+//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
+//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
+//                    p = p_0, in ∂Ω
+// 
+// ------------------------------------
+// |   |      p    |     u    |  RHS  | 
+// ------------------------------------
+// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
+// |   |           |          |       |
+// | v | (∇ p, v)  | -ω (u,v) |       |
+
+// where (q,v) ∈ L^2  × (L^2)^dim
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");                  
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   for (int i = 0; i < ref; i++ )
+   {
+      mesh.UniformRefinement();
+   }
+
+   // Define spaces
+   // H1 space for p
+   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
+   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
+
+   // H(div) for u
+   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
+   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->SetTestFECollVdim(1,dim);
+   
+   a->StoreMatrices(true);
+
+//  ± ω (p, q)
+#ifdef DEFINITE
+//  ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
+#else
+// -ω (p, q)
+   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
+#endif
+
+// -(∇⋅u, q) 
+   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
+
+// -ω (u,v)
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
+
+// (∇ p, v)
+   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
+
+// (v,δv)
+   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
+    
+// (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+   Array<int> ess_tdof_list;
+   Array<int> ess_bdr;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   FunctionCoefficient p_ex(p_exact);
+   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
+   VectorFunctionCoefficient u_ex(dim,u_exact);
+   FunctionCoefficient divu_ex(divu_exact);
+   GridFunction p_gf, u_gf;   
+   GridFunction pex_gf(p_fes);   
+
+   Array<int> offsets(3);
+   offsets[0] = 0;
+   offsets[1] = p_fes->GetVSize();
+   offsets[2] = u_fes->GetVSize();
+   offsets.PartialSum();
+   BlockVector x(offsets);
+   x = 0.0;
+
+   p_gf.MakeRef(p_fes,x.GetBlock(0));
+   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
+
+   u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+   a->Assemble();
+
+   OperatorPtr Ah;
+   Vector X,B;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = Ah.As<BlockMatrix>();
+
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream p_out;
+      socketstream u_out;
+      p_out.open(vishost, visport);
+      u_out.open(vishost, visport);
+      p_out.precision(8);
+      p_out << "solution\n" << mesh << p_gf <<
+               "window_title 'Numerical p' "
+               << flush;
+
+      u_out.precision(8);
+      u_out << "solution\n" << mesh << u_gf <<
+            "window_title 'Numerical flux' "
+            << flush;
+   }
+
+   return 0;
+}
+
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
@@ -0,0 +1,546 @@
+//                   MFEM Ultraweak DPG acoustics example
+//
+// Compile with: make uw_dpg
+//
+// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
+
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+double divu_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+enum prob_type
+{
+   plane_wave,
+   gaussian_beam  
+};
+
+prob_type prob;
+
+int main(int argc, char *argv[])
+{
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+   int iprob = 0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");    
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: plane wave, 1: Gaussian beam");                                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");                               
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 0; }
+   prob = (prob_type)iprob;
+
+
+   omega = 2.0 * M_PI * rnum;
+
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+// (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+// - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+// test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   GridFunction hatp_gf;
+
+
+   socketstream p_out;
+   // socketstream u_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+      // u_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-8);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+      GridFunction pex_gf(p_fes);
+      GridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+
+      // Error
+      int dofs = X.Size();
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = uex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = u_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+      if (visualization)
+      {
+         p_out.precision(8);
+         p_out << "solution\n" << mesh << p_gf <<
+                  "window_title 'Numerical presure' "
+                  << flush;
+
+         // u_out.precision(8);
+         // u_out << "solution\n" << mesh << u_gf <<
+         //       "window_title 'Numerical velocity' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return p;
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   double p, d2p;
+   acoustics_solution(x,p,u,d2p);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   double p, d2p;
+   Vector dp;
+   acoustics_solution(x,p,dp,d2p);
+   return d2p/omega;
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}
+
+void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
+{
+   dp.SetSize(X.Size());
+   switch (prob)
+   {
+   case plane_wave:
+   {
+      p = sin(omega*X.Sum());
+      dp = omega * cos(omega * X.Sum());
+      d2p = -dim * omega * omega * sin(omega*X.Sum());
+   }
+      break;
+   default:
+   {
+      double rk = omega;
+      double alpha = 45 * M_PI/180.;
+      double sina = sin(alpha); 
+      double cosa = cos(alpha);
+      // shift the origin
+      double xprim=X(0) + 0.1; 
+      double yprim=X(1) + 0.1;
+
+      double  x = xprim*sina - yprim*cosa;
+      double  y = xprim*cosa + yprim*sina;
+      double  dxdxprim = sina, dxdyprim = -cosa;
+      double  dydxprim = cosa, dydyprim =  sina;
+      //wavelength
+      double rl = 2.*M_PI/rk;
+
+      // beam waist radius
+      double w0 = 0.05;
+
+      // function w
+      double fact = rl/M_PI/(w0*w0);
+      double aux = 1. + (fact*y)*(fact*y);
+
+      double w = w0*sqrt(aux);
+      double dwdy = w0*fact*fact*y/sqrt(aux);
+      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
+
+      double phi0 = atan(fact*y);
+      double dphi0dy = cos(phi0)*cos(phi0)*fact;
+      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
+
+      double r = y + 1./y/(fact*fact);
+      double drdy = 1. - 1./(y*y)/(fact*fact);
+      double d2rdydy = 2./(y*y*y)/(fact*fact);
+
+      // pressure
+      complex<double> zi = complex<double>(0., 1.);
+      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
+
+      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
+      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
+      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
+      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
+      complex<double> zd2edydx = zd2edxdy;
+      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
+                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
+
+      double pf = pow(2.0/M_PI/(w*w),0.25);
+      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
+      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
+                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
+
+
+      complex<double> zp = pf*exp(ze);
+      complex<double> zdpdx = zp*zdedx;
+      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
+      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
+      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
+      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
+      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
+
+      p = zp.real();
+      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
+      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
+
+      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
+            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
+   }
+   break;
+   }
+
+}
@@ -0,0 +1,507 @@
+//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
+//
+// Compile with: make uw_dpgp
+//
+//     - Δ p ± ω^2 p = f̃ ,   in Ω
+//                 p = p_0, on ∂Ω
+//
+// First Order System
+
+//   ∇ p - ω u = 0, in Ω
+// - ∇⋅u ± ω p = f, in Ω
+//           p = p_0, in ∂Ω
+// where f:=f̃/ω 
+// 
+// UW-DPG:
+// 
+// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
+// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
+// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
+//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
+//                                  p̂ = p_0     on ∂Ω 
+
+// Note: 
+// p̂ := p on Γ_h (skeleton)
+// û := -u on Γ_h  
+
+// -------------------------------------------------------------
+// |   |     p     |     u     |    p̂      |    û    |  RHS    |
+// -------------------------------------------------------------
+// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
+// |   |           |           |           |         |         |
+// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
+
+// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// #define DEFINITE
+
+double p_exact(const Vector &x);
+void u_exact(const Vector &x, Vector & u);
+double rhs_func(const Vector &x);
+void gradp_exact(const Vector &x, Vector &gradu);
+double divu_exact(const Vector &x);
+double d2_exact(const Vector &x);
+double hatp_exact(const Vector & X);
+void hatu_exact(const Vector & X, Vector & hatu);
+
+int dim;
+double omega;
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   bool visualization = true;
+   double rnum=1.0;
+   int ref = 1;
+   double theta = 0.0;
+   bool adjoint_graph_norm = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree)");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
+                  "Number of wavelengths");      
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                    
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");                                
+   args.AddOption(&ref, "-ref", "--serial_ref",
+                  "Number of serial refinements.");  
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   omega = 2.0 * M_PI * rnum;
+
+   Mesh mesh(mesh_file, 1, 1);
+   dim = mesh.Dimension();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for p
+   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
+
+   // Vector L2 space for u 
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
+
+   // H^1/2 space for p̂  
+   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
+
+   // H^-1/2 space for û  
+   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
+
+
+   Array<ParFiniteElementSpace * > trial_fes; 
+   trial_fes.Append(p_fes);
+   trial_fes.Append(u_fes);
+   trial_fes.Append(hatp_fes);
+   trial_fes.Append(hatu_fes);
+
+   Array<FiniteElementCollection * > test_fec; 
+   test_fec.Append(q_fec);
+   test_fec.Append(v_fec);
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient omeg(omega);
+   ConstantCoefficient omeg2(omega*omega);
+   ConstantCoefficient negomeg(-omega);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+
+   // Integrators
+
+   // ± ω (p,q)
+#ifdef DEFINITE
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
+#else
+   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
+#endif   
+
+// (u , ∇ q)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+// -(p, ∇⋅v)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
+
+// - ω (u,v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
+
+// < p̂, v⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+// < û,q >
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   // test integrators 
+
+   //space-induced norm for H(div) × H1
+   // (∇q,∇δq)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+   // (q,δq)
+   a->AddTestIntegrator(new MassIntegrator(one),0,0);
+   // (∇⋅v,∇⋅δv)
+   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+
+   // additional integrators for the adjoint graph norm
+   if (adjoint_graph_norm)
+   {   
+      // -ω (∇q,δv)
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
+      // -ω (v,δq)
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
+      // ω^2 (v,δv)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
+
+#ifdef DEFINITE
+      // - ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
+      // - ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
+#else
+      // ω (∇⋅v,δq)   
+      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
+      // ω (q,∇⋅v)   
+      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
+#endif
+      // ω^2 (q,δq)
+      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
+   }   
+
+   // RHS
+   FunctionCoefficient f_rhs(rhs_func);
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
+
+
+   FunctionCoefficient hatpex(hatp_exact);
+   FunctionCoefficient pex(p_exact);
+   VectorFunctionCoefficient uex(dim,u_exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatp_gf;
+
+
+
+
+   socketstream p_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      p_out.open(vishost, visport);
+   }
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << " Refinement |" 
+               << "    Dofs    |" 
+               << "  L2 Error  |" 
+               << " Relative % |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" << endl;
+      mfem::out << " --------------------"      
+               <<  "-------------------"    
+               <<  "-------------------"    
+               <<  "-------------------" << endl;   
+   }
+
+
+   for (int i = 0; i<ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = p_fes->GetVSize();
+      offsets[2] = u_fes->GetVSize();
+      offsets[3] = hatp_fes->GetVSize();
+      offsets[4] = hatu_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
+      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+
+      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+      HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(2,2));
+      amg0->SetPrintLevel(0);
+      amg1->SetPrintLevel(0);
+      amg2->SetPrintLevel(0);
+      amg0->SetRelaxType(16);
+      amg1->SetRelaxType(16);
+      amg2->SetRelaxType(16);
+
+      M->SetDiagonalBlock(0,amg0);
+      M->SetDiagonalBlock(1,amg1);
+      M->SetDiagonalBlock(2,amg2);
+      // for (int i = 0; i < 3; i++)
+      // {
+      //    MUMPSSolver * mumps = new MUMPSSolver;
+      //    mumps->SetOperator(A->GetBlock(i,i));
+      //    M->SetDiagonalBlock(i,mumps);
+      // }
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
+      }
+      M->SetDiagonalBlock(3,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-10);
+      cg.SetMaxIter(20000);
+      cg.SetPrintLevel(-1);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+
+      ParGridFunction p_gf;
+      p_gf.MakeRef(p_fes,x.GetBlock(0));
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(1));
+
+
+      ParGridFunction pex_gf(p_fes);
+      ParGridFunction uex_gf(u_fes);
+      pex_gf.ProjectCoefficient(pex);
+      uex_gf.ProjectCoefficient(uex);
+
+      int dofs = p_fes->GlobalTrueVSize()
+               + u_fes->GlobalTrueVSize()
+               + hatp_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize();
+
+      double p_err = p_gf.ComputeL2Error(pex);
+      double p_norm = pex_gf.ComputeL2Error(zero);
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(vzero);
+
+      double L2Error = sqrt(p_err*p_err + u_err*u_err);
+      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = globalresidual;
+      dof0 = dofs;
+
+         std::ios oldState(nullptr);
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                   << std::setw(10) <<  dof0 << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  err0 << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_err << " | " 
+                   << std::setprecision(3) 
+                   << std::setw(10) << std::scientific <<  res0 << " | " 
+                   << std::setprecision(2) 
+                   << std::setw(6) << std::fixed << rate_res << " | " 
+                   << std::setprecision(5) 
+                   << std::scientific 
+                   << std::endl;
+      }   
+
+      if (visualization)
+      {
+         p_out << "parallel " << num_procs << " " << myid << "\n";
+         p_out.precision(8);
+         p_out << "solution\n" << pmesh << p_gf <<
+                  "window_title 'Numerical pressure' "
+                  << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+   }
+
+   delete a;
+   delete q_fec;
+   delete v_fec;
+   delete hatp_fes;
+   delete hatp_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete u_fec;
+   delete p_fec;
+   delete u_fes;
+   delete p_fes;
+
+   return 0;
+}
+
+double rhs_func(const Vector &x)
+{
+   double p = p_exact(x);
+   double divu = divu_exact(x);
+   // f = - ∇⋅u ± ω p, 
+#ifdef DEFINITE   
+   return -divu + omega * p;
+#else
+   return -divu - omega * p;
+#endif   
+}
+
+double p_exact(const Vector &x)
+{
+   return sin(omega*x.Sum());
+}
+
+void gradp_exact(const Vector &x, Vector &grad)
+{
+   grad.SetSize(x.Size());
+   grad = omega * cos(omega * x.Sum());
+}
+
+void u_exact(const Vector &x, Vector & u)
+{
+   gradp_exact(x,u);
+   u *= 1./omega;
+}
+
+double divu_exact(const Vector &x)
+{
+   return d2_exact(x)/omega;
+}
+
+double d2_exact(const Vector &x)
+{
+   return -dim * omega * omega * sin(omega*x.Sum());
+}
+
+double hatp_exact(const Vector & X)
+{
+   return p_exact(X);
+}
+
+void hatu_exact(const Vector & X, Vector & hatu)
+{
+   u_exact(X,hatu);
+   hatu *= -1.;
+}
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/convection-diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = uw_dpg
+PAR_EXAMPLES = uw_dpgp
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
@@ -0,0 +1,649 @@
+//                     MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");        
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                             
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: polynomial, 1: EJ ,2: General");           
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");       
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");                                              
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatf_fes = new FiniteElementSpace(&mesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   ConstantCoefficient zero(0.0);
+   Vector vec0(dim); vec0 = 0.;
+   VectorConstantCoefficient vzero(vec0);
+
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   FiniteElementSpace *coeff_fes = new FiniteElementSpace(&mesh,coeff_fec);
+   GridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         // (∇v,∇δv)
+         mfem::out << "\n Test norm: Standard" << endl;
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         mfem::out << "\n Test norm: Robust" << endl;
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+   GridFunction hatu_gf;
+   GridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   mfem::out << " Refinement |" 
+             << "    Dofs    |" 
+             << "  L2 Error  |" 
+             << " Relative % |" 
+             << "  Rate  |" 
+             << "  Residual  |" 
+             << "  Rate  |" << endl;
+   mfem::out << " --------------------"      
+             <<  "-------------------"    
+             <<  "-------------------"    
+             <<  "-------------------" << endl;   
+
+
+   for (int i = 0; i<=ref; i++)
+   {
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(mesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int i = 0; i < n; i++)
+      {
+         ess_tdof_list[i] = ess_tdof_list_uhat[i] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int i = 0; i < m; i++)
+      {
+         ess_tdof_list[i+n] = ess_tdof_list_fhat[i] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }   
+
+      GridFunction uex_gf(u_fes);
+      uex_gf.ProjectCoefficient(uex);
+
+      GridFunction sigmaex_gf(sigma_fes);
+      sigmaex_gf.ProjectCoefficient(sigmaex);
+
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = X.Size();
+      double u_err = u_gf.ComputeL2Error(uex);
+      double u_norm = uex_gf.ComputeL2Error(zero);
+      // mfem::out << "u_err = " << u_err << endl;
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double sigma_norm = sigmaex_gf.ComputeL2Error(vzero);
+      // mfem::out << "sigma_err = " << sigma_err << endl;
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+      double L2norm = sqrt(u_norm * u_norm + sigma_norm * sigma_norm);
+
+      double rel_error = L2Error/L2norm;
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = residual;
+      dof0 = dofs;
+      mfem::out << std::right << std::setw(11) << i << " | " 
+                << std::setw(10) <<  dof0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  err0 << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_err << " | " 
+                << std::setprecision(3) 
+                << std::setw(10) << std::scientific <<  res0 << " | " 
+                << std::setprecision(2) 
+                << std::setw(6) << std::fixed << rate_res << " | " 
+                << std::resetiosflags(std::ios::showbase)
+                << std::endl;
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "solution\n" << mesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+         u_out.precision(8);
+         u_out << "solution\n" << mesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      mesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> dofs;
+         for (int i = 0; i < mesh.GetNE(); i++)
+         {
+            double volume = mesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            // double c2 = 1.;
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+      }
+
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}
@@ -0,0 +1,698 @@
+//                                MFEM Ultraweak DPG example
+//
+// Compile with: make  uw_dpgp
+//
+// sample runs 
+// mpirun -np 6 ./uw_dpgp  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
+//     - εΔu + ∇⋅(βu) = f,   in Ω
+//                  u = u_0, on ∂Ω
+
+// First Order System
+
+//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
+//        1/ε σ - ∇u  = 0,   in Ω
+//                  u = u_0, on ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, f̂ ∈ H^-1/2  
+// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
+//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
+//                                        û = u_0  on ∂Ω 
+
+// Note: 
+// f̂ := βu - σ
+// û := -u
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
+// -------------------------------------------------------------
+// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
+// |   |           |           |           |          |         |
+// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
+
+// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   polynomial,
+   EJ,   
+   general  
+};
+
+enum test_norm_type
+{
+   standard,   
+   adjoint_graph,
+   robust  
+};
+
+prob_type prob;
+test_norm_type test_norm;
+Vector beta;
+double epsilon;
+// Function returns the solution u, and gradient du and the Laplacian d2u
+void solution(const Vector & x, double & u, Vector & du, double & d2u);
+double exact_u(const Vector & X);
+void exact_sigma(const Vector & X, Vector & sigma);
+double exact_hatu(const Vector & X);
+void exact_hatf(const Vector & X, Vector & hatf);
+double f_exact(const Vector & X);
+
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool visualization = true;
+   int iprob = 0;
+   int itest_norm = 0;
+   double theta = 0.7;
+   bool static_cond = false;
+   epsilon = 1e0;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&epsilon, "-eps", "--epsilon",
+                  "Epsilon coefficient");               
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");     
+   args.AddOption(&theta, "-theta", "--theta",
+                  "Theta parameter for AMR");                           
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");                         
+   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
+                  " 0: Standard, 1: Adjoint Graph, 2: Robust");     
+   args.AddOption(&beta, "-beta", "--beta",
+                  "Vector Coefficient beta");          
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 2) { iprob = 2; }
+   prob = (prob_type)iprob;
+
+   test_norm = (test_norm_type)itest_norm;
+
+   if (prob == prob_type::EJ)
+   {
+      mesh_file = "../../../data/inline-quad.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   if (beta.Size() == 0)
+   {
+      beta.SetSize(dim);
+      beta[0] = 1.;
+      beta[1] = 0.;
+   }
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatf_fes = new ParFiniteElementSpace(&pmesh,hatf_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+   ConstantCoefficient eps(epsilon);
+   ConstantCoefficient eps1(1./epsilon);
+   ConstantCoefficient negeps1(-1./epsilon);
+   ConstantCoefficient eps2(1/(epsilon*epsilon));
+
+   ConstantCoefficient negeps(-epsilon);
+   VectorConstantCoefficient betacoeff(beta);
+   Vector negbeta = beta;
+   negbeta.Neg();
+
+   DenseMatrix bbt(beta.Size());
+   MultVVt(beta, bbt); 
+   MatrixConstantCoefficient bbtcoeff(bbt);
+
+
+   VectorConstantCoefficient negbetacoeff(negbeta);
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatf_fes);
+   test_fec.Append(v_fec);
+   test_fec.Append(tau_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //-(βu , ∇v)
+   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
+   
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
+
+   // (u ,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
+
+   // 1/ε (σ,τ)
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
+
+   // <f̂ ,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,0);
+
+
+   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
+   ParFiniteElementSpace *coeff_fes = new ParFiniteElementSpace(&pmesh,coeff_fec);
+   ParGridFunction c1_gf, c2_gf;
+   GridFunctionCoefficient c1_coeff(&c1_gf);
+   GridFunctionCoefficient c2_coeff(&c2_gf);
+
+   switch (test_norm)
+   {
+      case standard:
+      {   
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Standard" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+      }
+      break;
+      case adjoint_graph:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Adjoint Graph" << endl;
+         }
+         // (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(one),0,0);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+         // (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
+         // 1/ε^2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
+         // 1/ε (∇v, δτ)
+         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
+         // - (β ⋅ ∇v,∇⋅δτ)  
+         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
+         // 1/ε (τ,∇δv) 
+         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
+         // -(β ∇⋅τ ,∇⋅δv)  
+         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
+      }
+      break;
+      default:
+      {
+         if (myid == 0)
+         {
+            mfem::out << "\n Test norm: Robust" << endl;
+         }
+         c1_gf.SetSpace(coeff_fes);
+         c2_gf.SetSpace(coeff_fes);
+         Array<int> dofs;
+         for (int i =0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,dofs);
+            c1_gf.SetSubVector(dofs,c1);
+            c2_gf.SetSubVector(dofs,c2);
+         }
+         // c1 (v,δv)
+         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
+         // ε (∇v,∇δv)
+         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
+         // (β⋅∇v, β⋅∇δv)   
+         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
+         // c2 (τ,δτ)
+         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
+         // (∇⋅τ,∇⋅δτ)
+         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
+      }
+      break;
+   }
+
+
+   FunctionCoefficient f(f_exact);
+   // if (prob != prob_type::EJ)
+   // {
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
+   // }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   VectorFunctionCoefficient hatfex(dim,exact_hatf);
+   Array<int> elements_to_refine;
+   FunctionCoefficient uex(exact_u);
+   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
+
+   ParGridFunction hatu_gf;
+   ParGridFunction hatf_gf;
+
+   // socketstream uex_out;
+   socketstream u_out;
+   // socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // uex_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+   }
+
+   double res0 = 0.;
+   double err0 = 0.;
+   int dof0;
+   if (myid == 0)
+   {
+      mfem::out << " Refinement |" 
+               << "    Dofs    |" 
+               << "  L2 Error  |" 
+               << "  Rate  |" 
+               << "  Residual  |" 
+               << "  Rate  |" << endl;
+      mfem::out << " --------------------"      
+               <<  "-------------------"    
+               <<  "-------------------"    
+               <<  "-------------------" << endl;   
+   }
+
+
+   for (int i = 0; i<=ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list_uhat;
+      Array<int> ess_tdof_list_fhat;
+      Array<int> ess_bdr_uhat;
+      Array<int> ess_bdr_fhat;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr_uhat.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr_fhat.SetSize(pmesh.bdr_attributes.Max());
+         // ess_bdr_uhat = 1;
+         // ess_bdr_fhat = 0;
+         ess_bdr_uhat = 0;
+         ess_bdr_fhat = 1;
+         ess_bdr_uhat[1] = 1;
+         ess_bdr_fhat[1] = 0;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
+         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
+      }
+
+      // shift the ess_tdofs
+      int n = ess_tdof_list_uhat.Size();
+      int m = ess_tdof_list_fhat.Size();
+      Array<int> ess_tdof_list(n+m);
+      for (int i = 0; i < n; i++)
+      {
+         ess_tdof_list[i] = ess_tdof_list_uhat[i] 
+                          + u_fes->GetTrueVSize() 
+                          + sigma_fes->GetTrueVSize();
+      }
+      for (int i = 0; i < m; i++)
+      {
+         ess_tdof_list[i+n] = ess_tdof_list_fhat[i] 
+                            + u_fes->GetTrueVSize() 
+                            + sigma_fes->GetTrueVSize()
+                            + hatu_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatf_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
+
+      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
+      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip = 2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-6);
+      cg.SetMaxIter(200000);
+      cg.SetPrintLevel(0);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      double maxresidual = residuals.Max(); 
+
+      double gresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&gresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      gresidual = sqrt(gresidual);
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      int dofs = u_fes->GlobalTrueVSize()
+               + sigma_fes->GlobalTrueVSize()
+               + hatu_fes->GlobalTrueVSize()
+               + hatf_fes->GlobalTrueVSize();
+
+      double u_err = u_gf.ComputeL2Error(uex);
+      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
+      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
+
+      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
+      double rate_res = (i) ? dim*log(res0/gresidual)/log((double)dof0/dofs) : 0.0;
+
+      err0 = L2Error;
+      res0 = gresidual;
+      dof0 = dofs;
+
+      if (myid == 0)
+      {
+         mfem::out << std::right << std::setw(11) << i << " | " 
+                  << std::setw(10) <<  dof0 << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  err0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_err << " | " 
+                  << std::setprecision(3) 
+                  << std::setw(10) << std::scientific <<  res0 << " | " 
+                  << std::setprecision(2) 
+                  << std::setw(6) << std::fixed << rate_res << " | " 
+                  << std::resetiosflags(std::ios::showbase)
+                  << std::endl;
+      }
+
+
+      if (visualization)
+      {
+         // uex_out.precision(8);
+         // uex_out << "parallel " << num_procs << " " << myid << "\n";
+         // uex_out << "solution\n" << pmesh << uex_gf <<
+         //          "window_title 'Exact u' "
+         //          << flush;
+
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << pmesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+      }
+
+      if (i == ref)
+         break;
+
+      pmesh.GeneralRefinement(elements_to_refine,1,1);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+
+      if (test_norm == test_norm_type::robust)
+      {
+         coeff_fes->Update();
+         c1_gf.Update();
+         c2_gf.Update();
+         Array<int> edofs;
+         for (int i = 0; i < pmesh.GetNE(); i++)
+         {
+            double volume = pmesh.GetElementVolume(i);
+            double c1 = min(epsilon/volume, 1.);
+            double c2 = min(1./epsilon, 1./volume);
+            coeff_fes->GetElementDofs(i,edofs);
+            c1_gf.SetSubVector(edofs,c1);
+            c2_gf.SetSubVector(edofs,c2);
+         }
+      }
+   }
+
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatf_fes;
+   delete hatf_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case polynomial:
+      {
+         int n=2;
+         int m=2;
+         u = pow(x,n)*pow(y,m);
+         du[0] = n * pow(x,n-1) * pow(y,m);
+         du[1] = m * pow(x,n) * pow(y,m-1);
+         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
+             + m * (m-1) * pow(x,n) * pow(y,m-2);
+      }
+      break;
+      case EJ:
+      {
+         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
+         double r1 = (1. + alpha) / (2.*epsilon);
+         double r2 = (1. - alpha) / (2.*epsilon);
+         double denom = exp(-r2) - exp(-r1);
+
+
+         double g1 = exp(r2*(x-1.));
+         double g1_x = r2*g1;
+         double g1_xx = r2*g1_x;
+         double g2 = exp(r1*(x-1.));
+         double g2_x = r1*g2;
+         double g2_xx = r1*g2_x;
+         double g = g1-g2;
+         double g_x = g1_x - g2_x;
+         double g_xx = g1_xx - g2_xx;
+
+
+         u = g * cos(M_PI * y)/denom;
+         double u_x = g_x * cos(M_PI * y)/denom;
+         double u_xx = g_xx * cos(M_PI * y)/denom;
+         double u_y = -M_PI * g * sin(M_PI*y)/denom;
+         double u_yy = -M_PI * M_PI * u;
+         du[0] = u_x;
+         du[1] = u_y;
+         d2u = u_xx + u_yy;
+
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ε ∇ u
+   sigma = du;
+   sigma *= epsilon;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return -exact_u(X);
+}
+
+void exact_hatf(const Vector & X, Vector & hatf)
+{
+   Vector sigma;
+   exact_sigma(X,sigma);
+   double u = exact_u(X);
+   hatf.SetSize(X.Size());
+   for (int i = 0; i<hatf.Size(); i++)
+   {
+      hatf[i] = beta[i] * u - sigma[i];
+   }
+}
+
+double f_exact(const Vector & X)
+{
+   // f = - εΔu + ∇⋅(βu)
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+
+   double s = 0;
+   for (int i = 0; i<du.Size(); i++)
+   {
+      s += beta[i] * du[i];
+   }
+   return -epsilon * d2u + s;
+}
@@ -0,0 +1,203 @@
+//                 MFEM Fosls 1
+//
+// Compile with: make blkfosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 5. Define a finite element space on the mesh. Here we use continuous
+   //    Lagrange finite elements of the specified order. If order < 1, we
+   //    instead use an isoparametric/isogeometric space.
+   FiniteElementCollection *fec0 = new H1_FECollection(order, dim);
+   FiniteElementCollection *fec1 = new RT_FECollection(order-1, dim);
+   FiniteElementSpace fespace0(&mesh, fec0);
+   FiniteElementSpace fespace1(&mesh, fec1);
+
+   Array<FiniteElementSpace *> fespaces(2);
+   fespaces[0] = &fespace0;
+   fespaces[1] = &fespace1;
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   BlockBilinearForm a(fespaces);
+   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
+
+   cout << "H1 fespace = " << fespace0.GetVSize() << endl;
+   cout << "RT fespace = " << fespace1.GetVSize() << endl;
+
+   FiniteElementCollection *fec2 = new RT_Trace_FECollection(order-1, dim);
+   FiniteElementSpace RT_trace_fes(&mesh, fec2);
+   cout << "RT trace = " << RT_trace_fes.GetVSize() << endl;
+
+   // for (int i = 0; i<mesh.GetNE(); i++)
+   // {
+   //    // const FiniteElement * fe = fespace1.GetFE(i);
+   //    // fespace1.GetTraceElement()
+   //    Array<int> faces, ori;
+   //    mesh.GetElementEdges(i, faces, ori);
+   //    for (int f = 0; f<faces.Size(); f++)
+   //    {
+   //       const FiniteElement * fe_trace = RT_trace_fes.GetFaceElement(faces[f]);
+   //       cout << fe_trace->GetDof() << endl;
+   //       Array<int> face_dofs;
+   //       RT_trace_fes.GetFaceDofs(faces[f],face_dofs);
+   //       cout << "face dofs = " << endl;
+   //       face_dofs.Print();
+   //    }
+
+   //    // cout << fe->GetGeomType() << endl;
+   //    Array<int> vdofs;
+   //    RT_trace_fes.GetElementVDofs(i, vdofs);
+   //    cout << "trace dofs = " << endl;
+   //    vdofs.Print();
+   //    fespace1.GetElementVDofs(i, vdofs);
+   //    cout << "elem dofs = " << endl;
+   //    vdofs.Print();
+   //    cin.get();
+   // }
+
+
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   Array2D<BilinearFormIntegrator * > blfi(2,2);
+   blfi(0,0) = new DiffusionIntegrator(one);
+   blfi(0,1) = new MixedVectorWeakDivergenceIntegrator(one);
+   blfi(1,0) = new MixedVectorGradientIntegrator(negone);
+
+   BilinearFormIntegrator * divdiv = new DivDivIntegrator(one);
+   BilinearFormIntegrator * mass = new VectorFEMassIntegrator(one);
+   SumIntegrator * suminteg = new SumIntegrator();
+   suminteg->AddIntegrator(divdiv);
+   suminteg->AddIntegrator(mass);
+   blfi(1,1) = suminteg;
+
+   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
+   integ->SetIntegrators(blfi);
+   a.AddDomainIntegrator(integ);
+   a.Assemble();
+
+
+   BlockLinearForm b(fespaces);
+
+   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
+   Array<LinearFormIntegrator * > lfi(2);
+   lfi[0] = nullptr;
+   lfi[1] = new VectorFEDomainLFDivIntegrator(negone);
+   lininteg->SetIntegrators(lfi);
+   b.AddDomainIntegrator(lininteg);
+   b.Assemble();
+
+
+   // need to implement blkgridfunction later but for now Vector would do
+   int size = 0;
+   for (int i = 0; i<fespaces.Size(); i++)
+   {
+      size += fespaces[i]->GetVSize();
+   }
+
+   Vector x(size);
+   x = 0.0;
+
+   OperatorPtr A;
+   Vector X,B;
+   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
+
+   GSSmoother M((SparseMatrix&)(*A));
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(200);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   a.RecoverFEMSolution(X,b,x);
+
+   GridFunction u_gf, sigma_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(fespaces[0],&data[0]);
+   sigma_gf.MakeRef(fespaces[1],&data[fespaces[0]->GetVSize()]);
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   delete fec0;
+
+   return 0;
+}
@@ -0,0 +1,223 @@
+//                   MFEM Fosls example
+//
+// Compile with: make fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// FOSLS:
+//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
+
+
+// -------------------------------------------------
+// |   |    u    |         σ           |    RHS    |
+// -------------------------------------------------
+// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
+// |   |         |                     |           |
+// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
+
+// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   // 3. Read the mesh from the given mesh file. We can handle triangular,
+   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
+   //    the same code.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   FiniteElementCollection *H1fec = new H1_FECollection(order,dim);
+   FiniteElementSpace *H1fes = new FiniteElementSpace(&mesh, H1fec);
+
+   FiniteElementCollection *RTfec = new RT_FECollection(order-1,dim);
+   FiniteElementSpace *RTfes = new FiniteElementSpace(&mesh, RTfec);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Linear forms
+   LinearForm b_0(H1fes);
+   // (f,∇⋅τ )
+   LinearForm b_1(RTfes);
+   b_1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(negone));
+
+   // Bilinear forms
+   // (∇u,∇v)
+   BilinearForm a_00(H1fes);
+   a_00.AddDomainIntegrator(new DiffusionIntegrator(one));
+
+   //   -(σ,∇v)
+   MixedBilinearForm a_01(RTfes, H1fes);
+   a_01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(
+                               one)); // (-1 is included)
+
+   //    // -(∇u,τ)
+   // MixedBilinearForm()
+   MixedBilinearForm a_10(H1fes, RTfes);
+   a_10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negone));
+
+   // (∇⋅σ, ∇⋅τ) + (σ,τ)
+
+   BilinearForm a_11(RTfes);
+   a_11.AddDomainIntegrator(new DivDivIntegrator(one));
+   a_11.AddDomainIntegrator(new VectorFEMassIntegrator(one));
+
+
+   Array<int> ess_bdr;
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Array<int> block_Toffsets(3);
+   block_Toffsets[0] = 0;
+   block_Toffsets[1] = H1fes->GetTrueVSize();
+   block_Toffsets[2] = RTfes->GetTrueVSize();
+   block_Toffsets.PartialSum();
+
+   Vector rhs_H1(H1fes->GetVSize()); rhs_H1 = 0.;
+   Vector rhs_RT(RTfes->GetVSize()); rhs_RT = 0.;
+
+   Vector x_H1(H1fes->GetVSize()); x_H1 = 0.;
+   Vector x_RT(RTfes->GetVSize()); x_RT = 0.;
+
+
+   Vector RHS_H1(H1fes->GetTrueVSize()); RHS_H1 = 0.0;
+   Vector RHS_RT(RTfes->GetTrueVSize()); RHS_RT = 0.0;
+
+   Vector X_H1(H1fes->GetTrueVSize()); X_H1 = 0.0;
+   Vector X_RT(RTfes->GetTrueVSize()); X_RT = 0.0;
+
+
+   b_0.Update(H1fes,rhs_H1,0);
+   b_0.Assemble();
+
+   b_1.Update(RTfes,rhs_RT,0);
+   b_1.Assemble();
+
+
+   // Assembly and BC
+   a_00.Assemble();
+   SparseMatrix A_00;
+   a_00.FormLinearSystem(ess_tdof_list,x_H1,rhs_H1,
+                         A_00,X_H1,RHS_H1);
+
+   a_01.Assemble();
+   SparseMatrix A_01;
+   Array<int> empty;
+   a_01.FormRectangularSystemMatrix(empty, ess_tdof_list,A_01);
+
+
+   a_10.Assemble();
+   SparseMatrix A_10;
+
+   a_10.FormRectangularLinearSystem(ess_tdof_list,empty,x_H1,rhs_RT,
+                                    A_10,X_H1,RHS_RT);
+
+   a_11.Assemble();
+   SparseMatrix A_11;
+   a_11.FormSystemMatrix(empty,A_11);
+
+
+   BlockMatrix BlockA(block_Toffsets);
+   BlockA.SetBlock(0,0,&A_00);
+   BlockA.SetBlock(0,1,&A_01);
+   BlockA.SetBlock(1,0,&A_10);
+   BlockA.SetBlock(1,1,&A_11);
+
+
+   BlockVector RHS(block_Toffsets);
+   RHS.GetBlock(0) = RHS_H1;
+   RHS.GetBlock(1) = RHS_RT;
+
+   BlockVector X(block_Toffsets);
+   X.GetBlock(0) = X_H1;
+   X.GetBlock(1) = X_RT;
+
+
+   SparseMatrix * A = BlockA.CreateMonolithic();
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(1);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(RHS, X);
+
+   GridFunction u_gf(H1fes), sigma_gf(RTfes);
+   u_gf = 0.;
+   sigma_gf = 0.;
+
+   const SparseMatrix * P = H1fes->GetConformingProlongation();
+   if (P)
+   {
+      a_00.RecoverFEMSolution(X.GetBlock(0),rhs_H1,u_gf);
+      a_11.RecoverFEMSolution(X.GetBlock(1),rhs_RT,sigma_gf);
+   }
+   else
+   {
+      u_gf.MakeRef(X.GetBlock(0),0);
+      sigma_gf.MakeRef(X.GetBlock(1),0);
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream solu_sock(vishost, visport);
+      solu_sock.precision(8);
+      solu_sock << "solution\n" << mesh << u_gf <<
+                "window_title 'Numerical u' "
+                << flush;
+      socketstream sols_sock(vishost, visport);
+      sols_sock.precision(8);
+      sols_sock << "solution\n" << mesh << sigma_gf <<
+                "window_title 'Numerical sigma' "
+                << flush;
+   }
+
+   return 0;
+}
@@ -0,0 +1,61 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/diffusion,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = blkfosls fosls primal_dpg \
+	 				uw_dpg 
+PAR_EXAMPLES = uw_dpgp 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+	rm -rf ParaView
+
+clean-exec:
@@ -0,0 +1,179 @@
+//                     MFEM primal_dpg example
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");   
+   args.ParseCheck();
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   // 3. Define a finite element space on the mesh. Here we use H1 continuous
+   //    high-order Lagrange finite elements of the given order.
+   H1_FECollection fec(order, mesh.Dimension());
+   FiniteElementSpace H1fes(&mesh, &fec);
+
+   RT_Trace_FECollection trace_fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTtrace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   H1_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+
+   trial_fes.Append(&H1fes);
+   trial_fes.Append(&RTtrace_fes);
+   test_fecs.Append(&test_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+
+
+   ConstantCoefficient one(1.0);
+   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
+   a->AddTrialIntegrator(new TraceIntegrator,1,0);
+
+   BilinearFormIntegrator * diffusion = new DiffusionIntegrator(one);
+   BilinearFormIntegrator * mass = new MassIntegrator(one);
+   a->AddTestIntegrator(diffusion,0,0);
+   a->AddTestIntegrator(mass,0,0);
+
+   a->AddDomainLFIntegrator(new DomainLFIntegrator(one),0);
+   if (static_cond) { a->EnableStaticCondensation(); }
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+   OperatorPtr Ah;
+
+   int size = H1fes.GetVSize() + RTtrace_fes.GetVSize();
+
+   Vector x(size);
+   x = 0.0;
+
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-6);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+
+   a->RecoverFEMSolution(X,x);
+
+   GridFunction u_gf;
+   double *data = x.GetData();
+   u_gf.MakeRef(&H1fes,data);
+
+   GridFunction s_gf;
+   s_gf.MakeRef(&RTtrace_fes,&data[H1fes.GetVSize()]);
+
+
+
+   RT_FECollection RTfec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &RTfec);
+
+   GridFunction sigma_gf(&RTfes);
+   sigma_gf = 0.0;
+   for (int i = 0; i<mesh.GetNE(); i++)
+   {
+      Array<int> strace_dofs;
+      Array<int> trace_dofs;
+      Vector dofs;
+      RTtrace_fes.GetElementDofs(i,trace_dofs);
+      strace_dofs.SetSize(trace_dofs.Size());
+      // shift dofs;
+      for (int j = 0; j< trace_dofs.Size(); j++)
+      {
+         int offset = trace_dofs[j] < 0 ? -H1fes.GetVSize() : H1fes.GetVSize();
+         strace_dofs[j] = offset + trace_dofs[j];
+      }
+      x.GetSubVector(strace_dofs, dofs);
+      sigma_gf.SetSubVector(trace_dofs,dofs);
+
+   }
+
+
+
+
+
+   ParaViewDataCollection paraview_dc("DPG_example", &mesh);
+   paraview_dc.SetPrefixPath("ParaView");
+   paraview_dc.SetLevelsOfDetail(order);
+   paraview_dc.SetCycle(0);
+   paraview_dc.SetDataFormat(VTKFormat::BINARY);
+   paraview_dc.SetHighOrderOutput(true);
+   paraview_dc.SetTime(0.0); // set the time
+   paraview_dc.RegisterField("field",&u_gf);
+   paraview_dc.RegisterField("flux",&sigma_gf);
+   // paraview_dc.RegisterField("flux",&s_gf);
+   paraview_dc.Save();
+
+
+
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << u_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << sigma_gf <<
+                 "window_title 'Flux sigma_n' "
+                 << flush;
+
+
+
+}
@@ -0,0 +1,403 @@
+//                       MFEM Ultraweak DPG example
+//
+// Compile with: make uw_dpg
+//
+// sample runs 
+// ./uw_dpg -m ../lshape2.mesh -o 2 -ref 20 -graph-norm -do 1 -prob 0
+
+//     - Δ u = f,   in Ω
+//         u = u_0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) - (σ , τ) + < û, τ⋅n> = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) + < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//                                û = 0       on ∂Ω 
+
+// Note: 
+// û := u
+// σ̂ := -σ
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           |  <σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u);
+
+
+double exact_u(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return u;
+}
+
+void exact_sigma(const Vector & X, Vector & sigma)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   // σ = ∇ u
+   sigma = du;
+}
+
+double exact_hatu(const Vector & X)
+{
+   return exact_u(X);
+}
+
+void exact_hatsigma(const Vector & X, Vector & hatsigma)
+{
+   exact_sigma(X,hatsigma);
+   hatsigma *= -1.;
+}
+
+double f_exact(const Vector & X)
+{
+   double u, d2u;
+   Vector du;
+   solution(X,u,du,d2u);
+   return -d2u;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");               
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space"); 
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                  
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   FiniteElementSpace *hatsigma_fes = new FiniteElementSpace(&mesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<FiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negone)),1,0);
+
+   // (σ,∇ v)
+   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   //  <σ̂,v> 
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+
+   // RHS
+   FunctionCoefficient f(f_exact);
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),1);
+   }
+
+   FunctionCoefficient hatuex(exact_hatu);
+   Array<int> elements_to_refine;
+   GridFunction hatu_gf;
+
+
+   socketstream u_out;
+   // socketstream sigma_out;
+   socketstream mesh_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      // sigma_out.open(vishost, visport);
+      mesh_out.open(vishost, visport);
+   }
+
+
+   for (int iref = 0; iref<ref; iref++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (mesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+   // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr);
+
+      OperatorPtr Ah;
+      Vector X,B;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockMatrix * A = Ah.As<BlockMatrix>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      for (int i=0; i<A->NumRowBlocks(); i++)
+      {
+         M->SetDiagonalBlock(i,new GSSmoother(A->GetBlock(i,i)));
+      }
+
+      CGSolver cg;
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      cout << "Residual = " << residual << endl;
+
+      elements_to_refine.SetSize(0);
+      double max_resid = residuals.Max();
+      double theta = 0.7;
+      for (int iel = 0; iel<mesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * max_resid)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+      
+      GridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      GridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out.precision(8);
+         string keys = (iref == 0) ? "keys em\n" : "keys";
+         u_out << "solution\n" << mesh << u_gf 
+               << "window_title 'Numerical u' "
+                  << flush;
+
+         // sigma_out.precision(8);
+         // sigma_out << "solution\n" << mesh << sigma_gf <<
+         //       "window_title 'Numerical flux' "
+         //       << flush;
+
+         mesh_out.precision(8);
+         mesh_out << "mesh\n" << mesh 
+                  << keys
+                  << "window_title 'Mesh' "
+                  << flush;
+
+      }
+
+      mesh.GeneralRefinement(elements_to_refine);
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
+
+
+void solution(const Vector & X, double & u, Vector & du, double & d2u)
+{
+   double x = X[0];
+   double y = X[1];
+   double z = 0.;
+   if (X.Size() == 3) z = X[2];
+   du.SetSize(X.Size());
+   du = 0.;
+   d2u = 0.;
+
+   switch(prob)
+   {
+      case lshape:
+      {
+         double r = sqrt(x*x + y*y);
+         double alpha = 2./3.;
+         double theta = atan2(y,x);
+         if (theta < 0) theta += 2*M_PI;
+         u = pow(r,alpha) * sin(alpha * theta);
+      }
+      break;
+      default:
+      {
+         double alpha = M_PI * (x + y + z);
+         u = sin(alpha);
+         du.SetSize(X.Size());
+         for (int i = 0; i<du.Size(); i++)
+         {
+            du[i] = M_PI * cos(alpha);
+         }
+         d2u = - M_PI*M_PI * u * du.Size();
+      }
+      break;
+   }
+}
@@ -0,0 +1,404 @@
+//                 MFEM UW DPG parallel example
+//
+// Compile with: make poisson_fosls
+//
+//     - Δ u = f, in Ω
+//         u = 0, on ∂Ω
+
+// First Order System
+
+//   ∇ u - σ = 0, in Ω
+// - ∇⋅σ     = f, in Ω
+//        u  = 0, in ∂Ω
+
+// UW-DPG:
+// 
+// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
+// û ∈ H^1/2, σ̂ ∈ H^-1/2  
+// -(u , ∇⋅τ) + < û, τ⋅n> - (σ , τ) = 0,      ∀ τ ∈ H(div,Ω)      
+//  (σ , ∇ v) - < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
+//            û = 0        on ∂Ω 
+
+// -------------------------------------------------------------
+// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
+// -------------------------------------------------------------
+// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
+// |   |           |           |           |         |         |
+// | v |           |  (σ,∇ v)  |           | -<σ̂,v>  |  (f,v)  |  
+
+// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+enum prob_type
+{
+   lshape,   
+   general  
+};
+
+prob_type prob;
+
+double exact(const Vector & X)
+{
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   return pow(r,alpha) * sin(alpha * theta);
+}
+
+void gradexact(const Vector & X, Vector & grad)
+{
+   grad.SetSize(2);
+   double x = X[0];
+   double y = X[1];
+
+   double r = sqrt(x*x + y*y);
+   double alpha = 2./3.;
+   double theta = atan2(y,x);
+   if (theta < 0) theta += 2*M_PI;
+
+   double r_x = x/r;
+   double r_y = y/r;
+   double theta_x = - y / (r*r);
+   double theta_y = x / (r*r);
+   double beta = alpha * pow(r,alpha - 1.);
+   grad[0] = beta*(r_x * sin(alpha*theta) + r * theta_x * cos(alpha*theta));
+   grad[1] = beta*(r_y * sin(alpha*theta) + r * theta_y * cos(alpha*theta));
+}
+
+
+int main(int argc, char *argv[])
+{
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../../../data/inline-quad.mesh";
+   int order = 1;
+   int delta_order = 1;
+   int ref = 1;
+   bool adjoint_graph_norm = false;
+   bool visualization = true;
+   int iprob = 0;
+   bool static_cond = false;
+   double theta = 0.7;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree).");
+   args.AddOption(&delta_order, "-do", "--delta_order",
+                  "Order enrichment for DPG test space.");
+   args.AddOption(&ref, "-ref", "--num_refinements",
+                  "Number of uniform refinements");    
+   args.AddOption(&theta, "-theta", "--theta_factor",
+                  "Refinement factor");                              
+   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
+                  "-no-graph-norm", "--no-adjoint-graph-norm",
+                  "Enable or disable Adjoint Graph Norm on the test space");   
+   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                  " 0: lshape, 1: General");       
+   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                  "--no-static-condensation", "Enable static condensation.");                                            
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   if (iprob > 1) { iprob = 1; }
+   prob = (prob_type)iprob;
+
+   if (prob == prob_type::lshape)
+   {
+      mesh_file = "../lshape2.mesh";
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   mesh.UniformRefinement();
+
+   mesh.EnsureNCMesh();
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // Define spaces
+   // L2 space for u
+   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
+
+   // Vector L2 space for σ 
+   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
+   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
+
+   // H^1/2 space for û 
+   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
+   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
+
+   // H^-1/2 space for σ̂ 
+   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
+   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,hatsigma_fec);
+
+   // testspace fe collections
+   int test_order = order+delta_order;
+   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
+   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
+
+   // Coefficients
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient negone(-1.0);
+
+   // Normal equation weak formulation
+   Array<ParFiniteElementSpace * > trial_fes; 
+   Array<FiniteElementCollection * > test_fec; 
+
+   trial_fes.Append(u_fes);
+   trial_fes.Append(sigma_fes);
+   trial_fes.Append(hatu_fes);
+   trial_fes.Append(hatsigma_fes);
+
+   test_fec.Append(tau_fec);
+   test_fec.Append(v_fec);
+
+   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
+   a->StoreMatrices(true);
+
+   //  -(u,∇⋅τ)
+   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
+
+   // -(σ,τ) 
+   TransposeIntegrator * mass = new TransposeIntegrator(new VectorFEMassIntegrator(negone));
+   a->AddTrialIntegrator(mass,1,0);
+
+   // (σ,∇ v)
+   TransposeIntegrator * grad = new TransposeIntegrator(new GradientIntegrator(one));
+   a->AddTrialIntegrator(grad,1,1);
+
+   //  <û,τ⋅n>
+   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
+
+   // -<σ̂,v> (sign is included in σ̂)
+   a->AddTrialIntegrator(new TraceIntegrator,3,1);
+
+   // test integrators (space-induced norm for H(div) × H1)
+   // (∇⋅τ,∇⋅δτ)
+   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
+   // (τ,δτ)
+   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   // (∇v,∇δv)
+   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
+   // (v,δv)
+   a->AddTestIntegrator(new MassIntegrator(one),1,1);
+
+   // additional terms for adjoint graph norm
+   if (adjoint_graph_norm)
+   {
+      // -(∇v,δτ) 
+      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
+      // -(τ,∇δv) 
+      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
+      // (τ,δτ)
+      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
+   }
+   // RHS
+   if (prob == prob_type::general)
+   {
+      a->AddDomainLFIntegrator(new DomainLFIntegrator(one),1);
+   }
+
+   FunctionCoefficient uex(exact);
+   Array<int> elements_to_refine;
+   ParGridFunction hatu_gf;
+
+
+   socketstream u_out;
+   socketstream sigma_out;
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      u_out.open(vishost, visport);
+      sigma_out.open(vishost, visport);
+   }
+
+   for (int i = 0; i<ref; i++)
+   {
+      if (static_cond) { a->EnableStaticCondensation(); }
+      a->Assemble();
+
+      Array<int> ess_tdof_list;
+      Array<int> ess_bdr;
+      if (pmesh.bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
+         ess_bdr = 1;
+         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+      }
+
+      // shift the ess_tdofs
+      for (int i = 0; i < ess_tdof_list.Size(); i++)
+      {
+         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
+      }
+
+      Array<int> offsets(5);
+      offsets[0] = 0;
+      offsets[1] = u_fes->GetVSize();
+      offsets[2] = sigma_fes->GetVSize();
+      offsets[3] = hatu_fes->GetVSize();
+      offsets[4] = hatsigma_fes->GetVSize();
+      offsets.PartialSum();
+      BlockVector x(offsets);
+      x = 0.0;
+      if (prob == prob_type::lshape)
+      {
+         hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
+         hatu_gf.ProjectBdrCoefficient(uex,ess_bdr);
+      }
+
+      Vector X,B;
+      OperatorPtr Ah;
+      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+      BlockOperator * A = Ah.As<BlockOperator>();
+
+      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+      M->owns_blocks = 1;
+      int skip = 0;
+      if (!static_cond)
+      {
+         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
+         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
+         amg0->SetPrintLevel(0);
+         amg1->SetPrintLevel(0);
+         M->SetDiagonalBlock(0,amg0);
+         M->SetDiagonalBlock(1,amg1);
+         skip=2;
+      }
+      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
+      amg2->SetPrintLevel(0);
+      M->SetDiagonalBlock(skip,amg2);
+      HypreSolver * prec;
+      if (dim == 2)
+      {
+         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      else
+      {
+         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
+      }
+      M->SetDiagonalBlock(skip+1,prec);
+
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(*M);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete M;
+
+      a->RecoverFEMSolution(X,x);
+
+      Vector & residuals = a->ComputeResidual(x);
+
+      double residual = residuals.Norml2();
+      
+      double maxresidual = residuals.Max(); 
+      double globalresidual = residual * residual; 
+
+      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
+      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+
+      globalresidual = sqrt(globalresidual);
+
+      if (myid == 0)
+      {
+         cout << "Global Residual = " << globalresidual << endl;
+      }
+
+      elements_to_refine.SetSize(0);
+      for (int iel = 0; iel<pmesh.GetNE(); iel++)
+      {
+         if (residuals[iel] > theta * maxresidual)
+         {
+            elements_to_refine.Append(iel);
+         }
+      }
+
+      ParGridFunction u_gf;
+      u_gf.MakeRef(u_fes,x.GetBlock(0));
+
+      ParGridFunction sigma_gf;
+      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
+
+      if (visualization)
+      {
+         u_out << "parallel " << num_procs << " " << myid << "\n";
+         u_out.precision(8);
+         u_out << "solution\n" << pmesh << u_gf <<
+                  "window_title 'Numerical u' "
+                  << flush;
+
+         sigma_out << "parallel " << num_procs << " " << myid << "\n";
+         sigma_out.precision(8);
+         sigma_out << "solution\n" << pmesh << sigma_gf <<
+                  "window_title 'Numerical flux' "
+                  << flush;
+      }
+
+
+      if (i == ref-1)
+      {
+         break;
+      }
+
+      pmesh.GeneralRefinement(elements_to_refine);
+
+      for (int i =0; i<trial_fes.Size(); i++)
+      {
+         trial_fes[i]->Update(false);
+      }
+      a->Update();
+   }
+
+   delete a;
+   delete tau_fec;
+   delete v_fec;
+   delete hatsigma_fes;
+   delete hatsigma_fec;
+   delete hatu_fes;
+   delete hatu_fec;
+   delete sigma_fec;
+   delete sigma_fes;
+   delete u_fec;
+   delete u_fes;
+
+   return 0;
+}
@@ -0,0 +1,59 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Use the MFEM build directory
+MFEM_DIR ?= ../../..
+MFEM_BUILD_DIR ?= ../../..
+SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/grad-div,)
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+SEQ_EXAMPLES = primal_dpg 
+PAR_EXAMPLES = 
+
+ifeq ($(MFEM_USE_MPI),NO)
+   EXAMPLES = $(SEQ_EXAMPLES)
+else
+   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
+endif
+
+.SUFFIXES:
+.SUFFIXES: .o .cpp .mk
+.PHONY: all clean clean-build clean-exec
+
+# Remove built-in rule
+%: %.cpp
+
+# Replace the default implicit rule for *.cpp files
+%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
+
+all: $(EXAMPLES) 
+
+MFEM_TESTS = EXAMPLES
+include $(MFEM_TEST_MK)
+
+# Testing: Parallel vs. serial runs
+RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+%-test-par: %
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+%-test-seq: %
+	@$(call mfem-test,$<,, Serial example)
+
+clean: clean-build clean-exec 
+
+clean-build:
+	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+	rm -rf *.dSYM *.TVD.*breakpoints
+
+clean-exec:
@@ -0,0 +1,176 @@
+//                                MFEM primal dpg example for grad-dic problem
+//
+// Compile with: make primal_dpg
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+// Exact solution, F, and r.h.s., f. See below for implementation.
+void F_exact(const Vector &, Vector &);
+void f_exact(const Vector &, Vector &);
+double freq = 1.0, kappa;
+
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command line options
+   const char *mesh_file = "../../../data/star.mesh";
+   int order = 1;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
+   args.ParseCheck();
+
+   kappa = freq * M_PI;
+
+
+   // 2. Read the mesh from the given mesh file, and refine once uniformly.
+   Mesh mesh(mesh_file);
+   // mesh.UniformRefinement();
+
+   RT_FECollection fec(order-1, mesh.Dimension());
+   FiniteElementSpace RTfes(&mesh, &fec);
+
+   H1_Trace_FECollection trace_fec(order, mesh.Dimension());
+   FiniteElementSpace H1trace_fes(&mesh, &trace_fec);
+
+   int dim = mesh.Dimension();
+   int test_order = order;
+   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
+   {
+      test_order++;
+   }
+
+   test_order++;
+
+   RT_FECollection test_fec(test_order,mesh.Dimension());
+
+   Array<FiniteElementSpace *> trial_fes; 
+   Array<FiniteElementCollection * > test_fecs; 
+   
+   trial_fes.Append(&RTfes);
+   trial_fes.Append(&H1trace_fes);
+   test_fecs.Append(&test_fec);
+
+
+   GridFunction rt_gf(&RTfes);
+   VectorFunctionCoefficient F(dim, F_exact);
+   rt_gf.ProjectCoefficient(F);
+
+   Vector x(RTfes.GetVSize()+H1trace_fes.GetVSize());
+   x = 0.;
+   x.SetVector(rt_gf,0);
+
+
+   ConstantCoefficient alpha(1.0);
+   ConstantCoefficient beta(1.0);
+   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
+   a->AddTrialIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTrialIntegrator(new VectorFEMassIntegrator(beta),0,0);
+   a->AddTrialIntegrator(new NormalTraceIntegrator,1,0);
+   a->AddTestIntegrator(new DivDivIntegrator(alpha),0,0);
+   a->AddTestIntegrator(new VectorFEMassIntegrator(beta),0,0);
+
+
+   VectorFunctionCoefficient f(dim, f_exact);
+   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
+   a->Assemble();
+
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      RTfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   Vector X,B;
+
+   OperatorPtr Ah;
+   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
+
+   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
+   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
+   M->owns_blocks = 1;
+   for (int i=0; i<A->NumRowBlocks(); i++)
+   {
+      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
+   }
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(*M);
+   cg.SetOperator(*A);
+   cg.Mult(B, X);
+
+   delete M;
+
+   a->RecoverFEMSolution(X,x);
+
+   // GridFunction u_gf;
+   double *data = x.GetData();
+   rt_gf.MakeRef(&RTfes,data);
+
+   GridFunction exact_gf(&RTfes);
+   exact_gf.ProjectCoefficient(F);
+
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream solu_sock(vishost, visport);
+   solu_sock.precision(8);
+   solu_sock << "solution\n" << mesh << rt_gf <<
+             "window_title 'Numerical u' "
+             << flush;
+
+   socketstream soltrace_sock(vishost, visport);
+   soltrace_sock.precision(8);
+   soltrace_sock << "solution\n" << mesh << exact_gf <<
+                 "window_title 'Exact' "
+                 << flush;
+
+}
+
+
+// The exact solution (for non-surface meshes)
+void F_exact(const Vector &p, Vector &F)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
+
+   F(0) = cos(kappa*x)*sin(kappa*y);
+   F(1) = cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      F(2) = 0.0;
+   }
+}
+
+// The right hand side
+void f_exact(const Vector &p, Vector &f)
+{
+   int dim = p.Size();
+
+   double x = p(0);
+   double y = p(1);
+   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
+
+   double temp = 1 + 2*kappa*kappa;
+
+   f(0) = temp*cos(kappa*x)*sin(kappa*y);
+   f(1) = temp*cos(kappa*y)*sin(kappa*x);
+   if (dim == 3)
+   {
+      f(2) = 0;
+   }
+}
@@ -0,0 +1,51 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+2
+
+elements
+3
+1 3 0 1 4 3
+1 3 3 4 7 6
+1 3 1 2 5 4
+
+boundary
+8
+1 1 0 1
+1 1 1 2
+1 1 2 5
+2 1 5 4
+2 1 4 7
+1 1 7 6
+1 1 6 3
+1 1 3 0
+
+vertices
+8
+
+nodes
+FiniteElementSpace
+FiniteElementCollection: H1_2D_P1
+VDim: 2
+Ordering: 1
+
+-1 1
+-1 -0
+-1 -1
+0 1
+0 -0
+0 -1
+1 1
+1 -0
@@ -305,8 +305,6 @@ int main(int argc, char *argv[])
         vis_w.precision(8);
         visualize(vis_w, mesh, &x, &w, "Elastic energy density", true);
      }
-      cout << "GLVis visualization paused."
-           << " Press space (in the GLVis window) to resume it.\n";
   }

   double ee0 = oper.ElasticEnergy(x.GetTrueVector());
@@ -351,11 +351,6 @@ int main(int argc, char *argv[])
         vis_w.precision(8);
         visualize(vis_w, pmesh, &x_gf, &w_gf, "Elastic energy density", true);
      }
-      if (myid == 0)
-      {
-         cout << "GLVis visualization paused."
-              << " Press space (in the GLVis window) to resume it.\n";
-      }
   }

   double ee0 = oper.ElasticEnergy(x_gf);
@@ -1,4 +1,4 @@
-//                       MFEM Example 17 - Parallel Version
+//                    MFEM Example 17 - Parallel Version
 //
 // Compile with: make ex17p
 //
@@ -43,7 +43,7 @@
 #include <sstream>
 #include <iostream>

-// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
+// Classes FE_Evolution, RiemannSolver, DomainIntegrator and FaceIntegrator
 // shared between the serial and parallel version of the example.
 #include "ex18.hpp"

@@ -189,7 +189,7 @@ int main(int argc, char *argv[])
   // 7. Set up the nonlinear form corresponding to the DG discretization of the
   //    flux divergence, and assemble the corresponding mass matrix.
   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
+   Aflux.AddDomainIntegrator(new DomainIntegrator(dim));
   Aflux.Assemble();

   NonlinearForm A(&vfes);
@@ -56,6 +56,27 @@ public:
               const Vector &nor, Vector &flux);
 };

+
+// Constant (in time) mixed bilinear form multiplying the flux grid function.
+// The form is (vec(v), grad(w)) where the trial space = vector L2 space (mesh
+// dim) and test space = scalar L2 space.
+class DomainIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector shape;
+   DenseMatrix flux;
+   DenseMatrix dshapedr;
+   DenseMatrix dshapedx;
+
+public:
+   DomainIntegrator(const int dim);
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Tr,
+                                       DenseMatrix &elmat);
+};
+
 // Interior face term: <F.n(u),[w]>
 class FaceIntegrator : public NonlinearFormIntegrator
 {
@@ -297,6 +318,60 @@ double RiemannSolver::Eval(const Vector &state1, const Vector &state2,
   return maxE;
 }

+// Implementation of class DomainIntegrator
+DomainIntegrator::DomainIntegrator(const int dim) : flux(num_equation, dim) { }
+
+void DomainIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                              const FiniteElement &test_fe,
+                                              ElementTransformation &Tr,
+                                              DenseMatrix &elmat)
+{
+   // Assemble the form (vec(v), grad(w))
+
+   // Trial space = vector L2 space (mesh dim)
+   // Test space  = scalar L2 space
+
+   const int dof_trial = trial_fe.GetDof();
+   const int dof_test = test_fe.GetDof();
+   const int dim = trial_fe.GetDim();
+
+   shape.SetSize(dof_trial);
+   dshapedr.SetSize(dof_test, dim);
+   dshapedx.SetSize(dof_test, dim);
+
+   elmat.SetSize(dof_test, dof_trial * dim);
+   elmat = 0.0;
+
+   const int maxorder = max(trial_fe.GetOrder(), test_fe.GetOrder());
+   const int intorder = 2 * maxorder;
+   const IntegrationRule *ir = &IntRules.Get(trial_fe.GetGeomType(), intorder);
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      // Calculate the shape functions
+      trial_fe.CalcShape(ip, shape);
+      shape *= ip.weight;
+
+      // Compute the physical gradients of the test functions
+      Tr.SetIntPoint(&ip);
+      test_fe.CalcDShape(ip, dshapedr);
+      Mult(dshapedr, Tr.AdjugateJacobian(), dshapedx);
+
+      for (int d = 0; d < dim; d++)
+      {
+         for (int j = 0; j < dof_test; j++)
+         {
+            for (int k = 0; k < dof_trial; k++)
+            {
+               elmat(j, k + d * dof_trial) += shape(k) * dshapedx(j, d);
+            }
+         }
+      }
+   }
+}
+
 // Implementation of class FaceIntegrator
 FaceIntegrator::FaceIntegrator(RiemannSolver &rsolver_, const int dim) :
   rsolver(rsolver_),
@@ -1,4 +1,4 @@
-//                       MFEM Example 18 - Parallel Version
+//                         MFEM Example 18 - Parallel Version
 //
 // Compile with: make ex18
 //
@@ -43,7 +43,7 @@
 #include <sstream>
 #include <iostream>

-// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
+// Classes FE_Evolution, RiemannSolver, DomainIntegrator and FaceIntegrator
 // shared between the serial and parallel version of the example.
 #include "ex18.hpp"

@@ -219,7 +219,7 @@ int main(int argc, char *argv[])
   // 9. Set up the nonlinear form corresponding to the DG discretization of the
   //    flux divergence, and assemble the corresponding mass matrix.
   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
+   Aflux.AddDomainIntegrator(new DomainIntegrator(dim));
   Aflux.Assemble();

   ParNonlinearForm A(&vfes);
@@ -1,4 +1,4 @@
-//                       MFEM Example 21 - Parallel Version
+//                                MFEM Example 21
 //
 // Compile with: make ex21p
 //
@@ -1,4 +1,4 @@
-//                                MFEM Example 22
+//                               MFEM Example 22
 //
 // Compile with: make ex22
 //
@@ -1,4 +1,4 @@
-//                                MFEM Example 24
+//                               MFEM Example 24
 //
 // Compile with: make ex24
 //
@@ -1,4 +1,4 @@
-//                                MFEM Example 27
+//                       MFEM Example 27 - Serial Version
 //
 // Compile with: make ex27
 //
@@ -1,4 +1,4 @@
-//                       MFEM Example 29 - Parallel Version
+//                      MFEM Example 29 - Parallel Version
 //
 // Compile with: make ex29p
 //
@@ -1,4 +1,4 @@
-//                                MFEM Example 30
+//                            MFEM Example 30
 //
 // Compile with: make ex30
 //
@@ -1,4 +1,4 @@
-//                       MFEM Example 30 - Parallel Version
+//                        MFEM Example 30 - Parallel Version
 //
 // Compile with: make ex30p
 //
@@ -45,10 +45,9 @@ int dim;
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
-   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();

   // 2. Parse command-line options.
   const char *mesh_file = "../data/inline-quad.mesh";
@@ -120,7 +119,7 @@ int main(int argc, char *argv[])
   }
   ParFiniteElementSpace fespace(&pmesh, fec);
   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (Mpi::Root()) { cout << "Number of H(Curl) unknowns: " << size << endl; }
+   if (mpi.Root()) { cout << "Number of H(Curl) unknowns: " << size << endl; }

   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
@@ -182,7 +181,7 @@ int main(int argc, char *argv[])
   // 12. Solve the system AX=B using PCG with the AMS preconditioner from hypre
   if (use_ams)
   {
-      if (Mpi::Root())
+      if (mpi.Root())
      {
         cout << "Size of linear system: "
              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
@@ -200,7 +199,7 @@ int main(int argc, char *argv[])
   else
 #ifdef MFEM_USE_SUPERLU
   {
-      if (Mpi::Root())
+      if (mpi.Root())
      {
         cout << "Size of linear system: "
              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
@@ -213,7 +212,7 @@ int main(int argc, char *argv[])
   }
 #else
   {
-      if (Mpi::Root()) { cout << "No solvers available." << endl; }
+      if (mpi.Root()) { cout << "No solvers available." << endl; }
      return 1;
   }
 #endif
@@ -225,7 +224,7 @@ int main(int argc, char *argv[])
   // 14. Compute and print the H(Curl) norm of the error.
   {
      double error = sol.ComputeHCurlError(&E, &CurlE);
-      if (Mpi::Root())
+      if (mpi.Root())
      {
         cout << "\n|| E_h - E ||_{H(Curl)} = " << error << '\n' << endl;
      }
@@ -41,10 +41,9 @@ double GetScalarMax(const ParGridFunction &x);
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
-   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
+   MPI_Session mpi;
+   int num_procs = mpi.WorldSize();
+   int myid = mpi.WorldRank();

   // 2. Parse command-line options.
   const char *mesh_file = "../data/inline-quad.mesh";
@@ -123,7 +122,7 @@ int main(int argc, char *argv[])
   ParFiniteElementSpace fespace_rt(&pmesh, fec_rt);
   HYPRE_Int size_nd = fespace_nd.GlobalTrueVSize();
   HYPRE_Int size_rt = fespace_rt.GlobalTrueVSize();
-   if (Mpi::Root())
+   if (mpi.Root())
   {
      cout << "Number of H(Curl) unknowns: " << size_nd << endl;
      cout << "Number of H(Div) unknowns: " << size_rt << endl;
@@ -165,7 +164,7 @@ int main(int argc, char *argv[])
         // closed surface.
         a.AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
         shift = 1.0;
-         if (Mpi::Root())
+         if (mpi.Root())
         {
            cout << "Computing eigenvalues shifted by " << shift << endl;
         }
@@ -288,7 +287,7 @@ int main(int argc, char *argv[])

         for (int i=0; i<nev; i++)
         {
-            if (Mpi::Root())
+            if (mpi.Root())
            {
               cout << "Eigenmode " << i+1 << '/' << nev
                    << ", Lambda = " << eigenvalues[i] - shift << endl;
@@ -409,7 +408,7 @@ int main(int argc, char *argv[])
               MPI_Barrier(MPI_COMM_WORLD);
            }
            char c;
-            if (Mpi::Root())
+            if (mpi.Root())
            {
               cout << "press (q)uit or (c)ontinue --> " << flush;
               cin >> c;
@@ -462,7 +461,7 @@ int main(int argc, char *argv[])

         for (int i=0; i<nev; i++)
         {
-            if (Mpi::Root())
+            if (mpi.Root())
            {
               cout << "Eigenmode " << i+1 << '/' << nev
                    << ", Lambda = " << eigenvalues[i] - shift << endl;
@@ -566,7 +565,7 @@ int main(int argc, char *argv[])
               MPI_Barrier(MPI_COMM_WORLD);
            }
            char c;
-            if (Mpi::Root())
+            if (mpi.Root())
            {
               cout << "press (q)uit or (c)ontinue --> " << flush;
               cin >> c;
@@ -592,7 +591,7 @@ int main(int argc, char *argv[])

         for (int i=0; i<nev; i++)
         {
-            if (Mpi::Root())
+            if (mpi.Root())
            {
               cout << "Eigenmode " << i+1 << '/' << nev
                    << ", Lambda = " << eigenvalues[i] - shift << endl;
@@ -621,7 +620,7 @@ int main(int argc, char *argv[])
            MPI_Barrier(MPI_COMM_WORLD);

            char c;
-            if (Mpi::Root())
+            if (mpi.Root())
            {
               cout << "press (q)uit or (c)ontinue --> " << flush;
               cin >> c;
@@ -1,192 +0,0 @@
-//                                MFEM Example 33
-//
-// Compile with: make ex33
-//
-// Sample runs:  ex33 -m ../data/square-disc.mesh -alpha 0.33 -o 2
-//               ex33 -m ../data/star.mesh -alpha 0.99 -o 3
-//               ex33 -m ../data/inline-quad.mesh -alpha 0.5 -o 3
-//               ex33 -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
-//               ex33 -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
-//
-// Description:
-//
-//  In this example we solve the following fractional PDE with MFEM:
-//
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
-//
-//  To solve this FPDE, we rely on a rational approximation [2] of the normal
-//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
-//  boundary conditions). Namely, we first approximate the operator
-//
-//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
-//
-//  where I is the L2-identity operator and the coefficients c_i and d_i
-//  are generated offline to a prescribed accuracy in a pre-processing step.
-//  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve N+1
-//  independent integer-order PDEs,
-//
-//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
-//
-//  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
-//
-//    u ≈ Σ_{i=0}^N u_i.
-//
-// References:
-//
-// [1] Nakatsukasa, Y., Sète, O., & Trefethen, L. N. (2018). The AAA algorithm
-//     for rational approximation. SIAM Journal on Scientific Computing, 40(3),
-//     A1494-A1522.
-//
-// [2] Harizanov, S., Lazarov, R., Margenov, S., Marinov, P., & Pasciak, J.
-//     (2020). Analysis of numerical methods for spectral fractional elliptic
-//     equations based on the best uniform rational approximation. Journal of
-//     Computational Physics, 408, 109285.
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#include "ex33.hpp"
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int num_refs = 3;
-   bool visualization = true;
-   double alpha = 0.5;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&num_refs, "-r", "--refs",
-                  "Number of uniform refinements");
-   args.AddOption(&alpha, "-alpha", "--alpha",
-                  "Fractional exponent");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Array<double> coeffs, poles;
-
-   // 2. Compute the coefficients that define the integer-order PDEs.
-   ComputePartialFractionApproximation(alpha,coeffs,poles);
-
-   // 3. Read the mesh from the given mesh file.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 4. Refine the mesh to increase the resolution.
-   for (int i = 0; i < num_refs; i++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   // 5. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new H1_FECollection(order, dim);
-   FiniteElementSpace fespace(&mesh, fec);
-   cout << "Number of finite element unknowns: "
-        << fespace.GetTrueVSize() << endl;
-
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 7. Define diffusion coefficient, load, and solution GridFunction.
-   ConstantCoefficient f(1.0);
-   ConstantCoefficient one(1.0);
-   GridFunction u(&fespace);
-   u = 0.;
-
-   // 8. Prepare for visualization.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream xout, uout;
-   ostringstream oss_x, oss_u;
-   if (visualization)
-   {
-      xout.open(vishost, visport);
-      xout.precision(8);
-      uout.open(vishost, visport);
-      uout.precision(8);
-   }
-
-   for (int i = 0; i < coeffs.Size(); i++)
-   {
-      // 9. Set up the linear form b(.) for integer-order PDE solve.
-      LinearForm b(&fespace);
-      ProductCoefficient cf(coeffs[i], f);
-      b.AddDomainIntegrator(new DomainLFIntegrator(cf));
-      b.Assemble();
-
-      // 10. Define GridFunction for integer-order PDE solve.
-      GridFunction x(&fespace);
-      x = 0.0;
-
-      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
-      BilinearForm a(&fespace);
-      a.AddDomainIntegrator(new DiffusionIntegrator(one));
-      ConstantCoefficient c2(-poles[i]);
-      a.AddDomainIntegrator(new MassIntegrator(c2));
-      a.Assemble();
-
-      // 12. Assemble the bilinear form and the corresponding linear system.
-      OperatorPtr A;
-      Vector B, X;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-      // 13. Solve the linear system A X = B.
-      GSSmoother M((SparseMatrix&)(*A));
-
-      mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
-                << " u = " << coeffs[i] << " f " << endl;
-      PCG(*A, M, B, X, 3, 200, 1e-12, 0.0);
-
-      // 14. Recover the solution as a finite element grid function.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 15. Accumulate integer-order PDE solutions.
-      u+=x;
-
-      // 16. Send the solutions by socket to a GLVis server.
-      if (visualization)
-      {
-         oss_x.str(""); oss_x.clear();
-         oss_x << "Solution of PDE -Δ u + " << -poles[i]
-               << " u = " << coeffs[i] << " f";
-         xout << "solution\n" << mesh << x
-              << "window_title '" << oss_x.str() << "'" << flush;
-
-         oss_u.str(""); oss_u.clear();
-         oss_u << "Solution of fractional PDE -Δ^" << alpha
-               << " u = f";
-         uout << "solution\n" << mesh << u
-              << "window_title '" << oss_u.str() << "'" << flush;
-      }
-   }
-
-   // 17. Free the used memory.
-   delete fec;
-   return 0;
-}
@@ -1,390 +0,0 @@
-//                  MFEM Example 33 - Serial/Parallel Shared Code
-//                      (Implementation of the AAA algorithm)
-//
-//  Here, we implement the triple-A algorithm [1] for the rational approximation
-//  of complex-valued functions,
-//
-//          p(z)/q(z) ≈ f(z).
-//
-//  In this file, we always assume f(z) = z^{-α}. The triple-A algorithm
-//  provides a robust, accurate approximation in rational barycentric form.
-//  This representation must be transformed into a partial fraction
-//  representation in order to be used to solve a spectral FPDE.
-//
-//  More specifically, we first expand the numerator in terms of the zeros of
-//  the rational approximation,
-//
-//          p(z) ∝ Π_i (z - z_i),
-//
-//  and expand the denominator in terms of the poles of the rational
-//  approximation,
-//
-//          q(z) ∝ Π_i (z - p_i).
-//
-//  We then use these zeros and poles to derive the partial fraction expansion
-//
-//          f(z) ≈ p(z)/q(z) = Σ_i c_i / (z - p_i).
-//
-//  [1] Nakatsukasa, Y., Sète, O., & Trefethen, L. N. (2018). The AAA algorithm
-//      for rational approximation. SIAM Journal on Scientific Computing, 40(3),
-//      A1494-A1522.
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-/** RationalApproximation_AAA: compute the rational approximation (RA) of data
-    @a val [in] at the set of points @a pt [in].
-
-    @param[in]  val        Vector of data values
-    @param[in]  pt         Vector of sample points
-    @param[in]  tol        Relative tolerance
-    @param[in]  max_order  Maximum number of terms (order) of the RA
-    @param[out] z          Support points of the RA in rational barycentric form
-    @param[out] f          Data values at support points @a z
-    @param[out] w          Weights of the RA in rational barycentric form
-
-    See pg. A1501 of Nakatsukasa et al. [1]. */
-void RationalApproximation_AAA(const Vector &val, const Vector &pt,
-                               Array<double> &z, Array<double> &f, Vector &w,
-                               double tol, int max_order)
-{
-
-   // number of sample points
-   int size = val.Size();
-   MFEM_VERIFY(pt.Size() == size, "size mismatch");
-
-   // Initializations
-   Array<int> J(size);
-   for (int i = 0; i < size; i++) { J[i] = i; }
-   z.SetSize(0);
-   f.SetSize(0);
-
-   DenseMatrix C, Ctemp, A, Am;
-   // auxiliary arrays and vectors
-   Vector f_vec;
-   Array<double> c_i;
-
-   // mean of the value vector
-   Vector R(val.Size());
-   double mean_val = val.Sum()/size;
-
-   for (int i = 0; i<R.Size(); i++) { R(i) = mean_val; }
-
-   for (int k = 0; k < max_order; k++)
-   {
-      // select next support point
-      int idx = 0;
-      double tmp_max = 0;
-      for (int j = 0; j < size; j++)
-      {
-         double tmp = abs(val(j)-R(j));
-         if (tmp > tmp_max)
-         {
-            tmp_max = tmp;
-            idx = j;
-         }
-      }
-
-      // Append support points and data values
-      z.Append(pt(idx));
-      f.Append(val(idx));
-
-      // Update index vector
-      J.DeleteFirst(idx);
-
-      // next column in Cauchy matrix
-      Array<double> C_tmp(size);
-      for (int j = 0; j < size; j++)
-      {
-         C_tmp[j] = 1.0/(pt(j)-pt(idx));
-      }
-      c_i.Append(C_tmp);
-      int h_C = C_tmp.Size();
-      int w_C = k+1;
-      C.UseExternalData(c_i.GetData(),h_C,w_C);
-
-      Ctemp = C;
-
-      f_vec.SetDataAndSize(f.GetData(),f.Size());
-      Ctemp.InvLeftScaling(val);
-      Ctemp.RightScaling(f_vec);
-
-      A.SetSize(C.Height(), C.Width());
-      Add(C,Ctemp,-1.0,A);
-      A.LeftScaling(val);
-
-      int h_Am = J.Size();
-      int w_Am = A.Width();
-      Am.SetSize(h_Am,w_Am);
-      for (int i = 0; i<h_Am; i++)
-      {
-         int ii = J[i];
-         for (int j = 0; j<w_Am; j++)
-         {
-            Am(i,j) = A(ii,j);
-         }
-      }
-
-#ifdef MFEM_USE_LAPACK
-      DenseMatrixSVD svd(Am,false,true);
-      svd.Eval(Am);
-      DenseMatrix &v = svd.RightSingularvectors();
-      v.GetRow(k,w);
-#else
-      mfem_error("Compiled without LAPACK");
-#endif
-
-      // N = C*(w.*f); D = C*w; % numerator and denominator
-      Vector aux(w);
-      aux *= f_vec;
-      Vector N(C.Height()); // Numerator
-      C.Mult(aux,N);
-      Vector D(C.Height()); // Denominator
-      C.Mult(w,D);
-
-      R = val;
-      for (int i = 0; i<J.Size(); i++)
-      {
-         int ii = J[i];
-         R(ii) = N(ii)/D(ii);
-      }
-
-      Vector verr(val);
-      verr-=R;
-
-      if (verr.Normlinf() <= tol*val.Normlinf()) { break; }
-   }
-}
-
-/** ComputePolesAndZeros: compute the @a poles [out] and @a zeros [out] of the
-    rational function f(z) = C p(z)/q(z) from its ration barycentric form.
-
-    @param[in]  z      Support points in rational barycentric form
-    @param[in]  f      Data values at support points @a z
-    @param[in]  w      Weights in rational barycentric form
-    @param[out] poles  Array of poles (roots of p(z))
-    @param[out] zeros  Array of zeros (roots of q(z))
-    @param[out] scale  Scaling constant in f(z) = C p(z)/q(z)
-
-    See pg. A1501 of Nakatsukasa et al. [1]. */
-void ComputePolesAndZeros(const Vector &z, const Vector &f, const Vector &w,
-                          Array<double> & poles, Array<double> & zeros, double &scale)
-{
-   // Initialization
-   poles.SetSize(0);
-   zeros.SetSize(0);
-
-   // Compute the poles
-   int m = w.Size();
-   DenseMatrix B(m+1); B = 0.;
-   DenseMatrix E(m+1); E = 0.;
-   for (int i = 1; i<=m; i++)
-   {
-      B(i,i) = 1.;
-      E(0,i) = w(i-1);
-      E(i,0) = 1.;
-      E(i,i) = z(i-1);
-   }
-
-#ifdef MFEM_USE_LAPACK
-   DenseMatrixGeneralizedEigensystem eig1(E,B);
-   eig1.Eval();
-   Vector & evalues = eig1.EigenvaluesRealPart();
-   for (int i = 0; i<evalues.Size(); i++)
-   {
-      if (IsFinite(evalues(i)))
-      {
-         poles.Append(evalues(i));
-      }
-   }
-#else
-   mfem_error("Compiled without LAPACK");
-#endif
-   // compute the zeros
-   B = 0.;
-   E = 0.;
-   for (int i = 1; i<=m; i++)
-   {
-      B(i,i) = 1.;
-      E(0,i) = w(i-1) * f(i-1);
-      E(i,0) = 1.;
-      E(i,i) = z(i-1);
-   }
-
-#ifdef MFEM_USE_LAPACK
-   DenseMatrixGeneralizedEigensystem eig2(E,B);
-   eig2.Eval();
-   evalues = eig2.EigenvaluesRealPart();
-   for (int i = 0; i<evalues.Size(); i++)
-   {
-      if (IsFinite(evalues(i)))
-      {
-         zeros.Append(evalues(i));
-      }
-   }
-#else
-   mfem_error("Compiled without LAPACK");
-#endif
-
-   scale = w * f / w.Sum();
-}
-
-/** PartialFractionExpansion: compute the partial fraction expansion of the
-    rational function f(z) = Σ_i c_i / (z - p_i) from its @a poles [in] and
-    @a zeros [in].
-
-    @param[in]  poles   Array of poles (same as p_i above)
-    @param[in]  zeros   Array of zeros
-    @param[in]  scale   Scaling constant
-    @param[out] coeffs  Coefficients c_i */
-void PartialFractionExpansion(double scale, Array<double> & poles,
-                              Array<double> & zeros, Array<double> & coeffs)
-{
-   int psize = poles.Size();
-   int zsize = zeros.Size();
-   coeffs.SetSize(psize);
-   coeffs = scale;
-
-   for (int i=0; i<psize; i++)
-   {
-      double tmp_numer=1.0;
-      for (int j=0; j<zsize; j++)
-      {
-         tmp_numer *= poles[i]-zeros[j];
-      }
-
-      double tmp_denom=1.0;
-      for (int k=0; k<psize; k++)
-      {
-         if (k != i) { tmp_denom *= poles[i]-poles[k]; }
-      }
-      coeffs[i] *= tmp_numer / tmp_denom;
-   }
-}
-
-
-/** ComputePartialFractionApproximation: compute a rational approximation (RA)
-    in partial fraction form, e.g., f(z) ≈ Σ_i c_i / (z - p_i), from sampled
-    values of the function f(z) = z^{-a}, 0 < a < 1.
-
-    @param[in]  alpha         Exponent a in f(z) = z^-a
-    @param[in] lmax, npoints  f(z) is uniformly sampled @a npoints times in the
-                              interval [ 0, @a lmax ]
-    @param[in]  tol           Relative tolerance
-    @param[in]  max_order     Maximum number of terms (order) of the RA
-    @param[out] coeffs        Coefficients c_i
-    @param[out] poles         Poles p_i
-
-    NOTES: When MFEM is not built with LAPACK support, only @a alpha = 0.33,
-           0.5, and 0.99 are possible. In this case, if @a alpha != 0.33 and
-           @a alpha != 0.99, then @a alpha = 0.5 is used by default.
-
-   See pg. A1501 of Nakatsukasa et al. [1]. */
-void ComputePartialFractionApproximation(double & alpha,
-                                         Array<double> & coeffs, Array<double> & poles,
-                                         double lmax = 1000.,
-                                         double tol=1e-10, int npoints = 1000,
-                                         int max_order = 100)
-{
-   MFEM_VERIFY(alpha < 1., "alpha must be less than 1");
-   MFEM_VERIFY(alpha > 0., "alpha must be greater than 0");
-   MFEM_VERIFY(npoints > 2, "npoints must be greater than 2");
-   MFEM_VERIFY(lmax > 0,  "lmin must be greater than 0");
-   MFEM_VERIFY(tol > 0,  "tol must be greater than 0");
-
-   bool print_warning = true;
-#ifdef MFEM_USE_MPI
-   if ((Mpi::IsInitialized() && !Mpi::Root())) { print_warning = false; }
-#endif
-
-#ifndef MFEM_USE_LAPACK
-   if (print_warning)
-   {
-      mfem::out
-            << "\nMFEM is compiled without LAPACK.\nUsing precomputed values for PartialFractionApproximation. \n"
-            << "Only alpha = 0.33, 0.5, and 0.99 are available.\nThe default is alpha = 0.5."
-            << std::endl;
-   }
-   const double eps = std::numeric_limits<double>::epsilon();
-
-   if (abs(alpha - 0.33) < eps)
-   {
-      coeffs = Array<double> ({1.821898e+03, 9.101221e+01, 2.650611e+01,
-                               1.174937e+01, 6.140444e+00, 3.441713e+00,
-                               1.985735e+00, 1.162634e+00, 6.891560e-01,
-                               4.111574e-01, 2.298736e-01});
-      poles = Array<double> ({-4.155583e+04, -2.956285e+03, -8.331715e+02,
-                              -3.139332e+02, -1.303448e+02, -5.563385e+01,
-                              -2.356255e+01, -9.595516e+00, -3.552160e+00,
-                              -1.032136e+00, -1.241480e-01});
-   }
-   else if (abs(alpha - 0.99) < eps)
-   {
-      coeffs = Array<double>({2.919591e-02, 1.419750e-02, 1.065798e-02,
-                              9.395094e-03, 8.915329e-03, 8.822991e-03,
-                              9.058247e-03, 9.814521e-03, 1.180396e-02,
-                              1.834554e-02, 9.840482e-01});
-      poles = Array<double> ({-1.069683e+04, -1.769370e+03, -5.718374e+02,
-                              -2.242095e+02, -9.419132e+01, -4.031012e+01,
-                              -1.701525e+01, -6.810088e+00, -2.382810e+00,
-                              -5.700059e-01, -1.384324e-03});
-   }
-   else
-   {
-      if (abs(alpha - 0.5) > eps && print_warning)
-      {
-         alpha = 0.5;
-      }
-      coeffs = Array<double>({2.290262e+02, 2.641819e+01, 1.005566e+01,
-                              5.390411e+00, 3.340725e+00, 2.211205e+00,
-                              1.508883e+00, 1.049474e+00, 7.462709e-01,
-                              5.482686e-01, 4.232510e-01, 3.578967e-01});
-      poles = Array<double>({-3.168211e+04, -3.236077e+03, -9.868287e+02,
-                             -3.945597e+02, -1.738889e+02, -7.925178e+01,
-                             -3.624992e+01, -1.629196e+01, -6.982956e+00,
-                             -2.679984e+00, -7.782607e-01, -7.649166e-02});
-   }
-
-   if (print_warning)
-   {
-      mfem::out << "Using precomputed values for alpha = "
-                << alpha << "\n" << std::endl;
-   }
-
-
-   return;
-#endif
-
-   Vector x(npoints);
-   Vector val(npoints);
-   double dx = lmax / (double)(npoints-1);
-   for (int i = 0; i<npoints; i++)
-   {
-      x(i) = dx * (double)i;
-      val(i) = pow(x(i),1.-alpha);
-   }
-
-   // Apply triple-A algorithm to f(x) = x^{1-a}
-   Array<double> z, f;
-   Vector w;
-   RationalApproximation_AAA(val,x,z,f,w,tol,max_order);
-
-   Vector vecz, vecf;
-   vecz.SetDataAndSize(z.GetData(), z.Size());
-   vecf.SetDataAndSize(f.GetData(), f.Size());
-
-   // Compute poles and zeros for RA of f(x) = x^{1-a}
-   double scale;
-   Array<double> zeros;
-   ComputePolesAndZeros(vecz, vecf, w, poles, zeros, scale);
-
-   // Remove the zero at x=0, thus, delivering a RA for f(x) = x^{-a}
-   zeros.DeleteFirst(0.0);
-
-   // Compute partial fraction approximation of f(x) = x^{-a}
-   PartialFractionExpansion(scale, poles, zeros, coeffs);
-}
@@ -1,290 +0,0 @@
-//                       MFEM Example 33 - Parallel Version
-//
-// Compile with: make ex33p
-//
-// Sample runs:  mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 0.33 -o 2
-//               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 0.99 -o 3
-//               mpirun -np 4 ex33p -m ../data/inline-quad.mesh -alpha 0.5 -o 3
-//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
-//               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
-//
-// Description:
-//
-//  In this example we solve the following fractional PDE with MFEM:
-//
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
-//
-//  To solve this FPDE, we rely on a rational approximation [2] of the normal
-//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
-//  boundary conditions). Namely, we first approximate the operator
-//
-//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
-//
-//  where I is the L2-identity operator and the coefficients c_i and d_i
-//  are generated offline to a prescribed accuracy in a pre-processing step.
-//  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve N+1
-//  independent integer-order PDEs,
-//
-//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
-//
-//  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
-//
-//    u ≈ Σ_{i=0}^N u_i.
-//
-// References:
-//
-// [1] Nakatsukasa, Y., Sète, O., & Trefethen, L. N. (2018). The AAA algorithm
-//     for rational approximation. SIAM Journal on Scientific Computing, 40(3),
-//     A1494-A1522.
-//
-// [2] Harizanov, S., Lazarov, R., Margenov, S., Marinov, P., & Pasciak, J.
-//     (2020). Analysis of numerical methods for spectral fractional elliptic
-//     equations based on the best uniform rational approximation. Journal of
-//     Computational Physics, 408, 109285.
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-#include "ex33.hpp"
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 0. Initialize MPI.
-   Mpi::Init(argc, argv);
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/star.mesh";
-   int order = 1;
-   int num_refs = 3;
-   bool visualization = true;
-   bool visualize_x = false;
-   double alpha = 0.5;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&num_refs, "-r", "--refs",
-                  "Number of uniform refinements");
-   args.AddOption(&alpha, "-alpha", "--alpha",
-                  "Fractional exponent");
-   args.AddOption(&visualize_x, "-vis_x", "--visualize_x", "-no-vis_x",
-                  "--no-visualization_x",
-                  "Enable or disable GLVis visualization of each integer-order PDE solution.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization of the fractional PDE solution.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   if (Mpi::Root())
-   {
-      args.PrintOptions(cout);
-   }
-
-   Array<double> coeffs, poles;
-
-   // 2. Compute the coefficients that define the integer-order PDEs.
-   ComputePartialFractionApproximation(alpha,coeffs,poles);
-
-   int num_par_solves;
-   int max_par_solves = max(1,num_procs/2);
-   for (num_par_solves=max_par_solves; num_par_solves>0; num_par_solves--)
-   {
-      if (num_procs%num_par_solves==0 && num_par_solves<coeffs.Size())
-      {
-         break;
-      }
-   }
-   if (num_par_solves == 1) {num_par_solves = num_procs;}
-
-   int solver_ranks = num_procs/num_par_solves;
-
-   // 3. Split the MPI communicator:
-   //    row_comm is used for parallel partition of the mesh
-   //    col_comm is used for independent integer-order solves
-   int row_color = myid / solver_ranks; // Determine color based on row
-   int col_color = myid % solver_ranks; // Determine color based on col
-
-   MPI_Comm row_comm, col_comm;
-   MPI_Comm_split(MPI_COMM_WORLD, row_color, myid, &row_comm);
-   MPI_Comm_split(MPI_COMM_WORLD, col_color, myid, &col_comm);
-
-   int row_rank, row_size, col_rank, col_size;
-   MPI_Comm_rank(row_comm, &row_rank);
-   MPI_Comm_size(row_comm, &row_size);
-   MPI_Comm_rank(col_comm, &col_rank);
-   MPI_Comm_size(col_comm, &col_size);
-
-   if (Mpi::Root())
-   {
-      mfem::out << "\nTotal number of MPI ranks = " << num_procs << endl;
-      mfem::out << "Number of independent parallel solves = " << col_size << endl;
-      mfem::out << "Number of MPI ranks within each solve = " << row_size
-                <<"\n" << endl;
-   }
-
-   // 4. Read the mesh from the given mesh file.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Refine the mesh to increase the resolution.
-   for (int i = 0; i < num_refs; i++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   ParMesh pmesh(row_comm, mesh);
-   mesh.Clear();
-
-   // 6. Define a finite element space on the mesh.
-   H1_FECollection fec(order, dim);
-   ParFiniteElementSpace fespace(&pmesh, &fec);
-   if (Mpi::Root())
-   {
-      cout << "Number of finite element unknowns: "
-           << fespace.GetTrueVSize() << endl;
-   }
-
-   // 7. Determine the list of true (i.e. conforming) essential boundary dofs.
-   Array<int> ess_tdof_list;
-   if (pmesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   // 8. Define diffusion coefficient, load, and solution GridFunction.
-   ConstantCoefficient f(1.0);
-   ConstantCoefficient one(1.0);
-   ParGridFunction u(&fespace);
-   ParGridFunction x(&fespace);
-   u = 0.0;
-
-   // 9. Set up the linear form b(.) for integer-order PDE solves.
-   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(f));
-   b.Assemble();
-
-   int my_coeff_size = max(coeffs.Size()/col_size,1);
-   int ibeg = col_rank*my_coeff_size;
-   if (ibeg + 2*my_coeff_size > coeffs.Size())
-   {
-      my_coeff_size = coeffs.Size()-col_rank*my_coeff_size;
-   }
-   else if (ibeg > coeffs.Size() - 1)
-   {
-      my_coeff_size = 0;
-   }
-
-   int iend = ibeg+my_coeff_size;
-
-
-   for (int i = ibeg; i < iend; i++)
-   {
-      // 10. Reset GridFunction for integer-order PDE solve.
-      x = 0.0;
-
-      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
-      ParBilinearForm a(&fespace);
-      a.AddDomainIntegrator(new DiffusionIntegrator(one));
-      ConstantCoefficient d_i(-poles[i]);
-      a.AddDomainIntegrator(new MassIntegrator(d_i));
-      a.Assemble();
-
-      // 12. Assemble the bilinear form and the corresponding linear system.
-      OperatorPtr A;
-      Vector B, X;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-      // 13. Solve the linear system A X = B.
-      HypreBoomerAMG * prec = new HypreBoomerAMG;
-      prec->SetPrintLevel(-1);
-
-      int print_level = (col_rank==0) ? 3 : 0;
-      if (Mpi::Root())
-      {
-         mfem::out << "\nMPI rank " <<  myid
-                   << ": Solving PDE -Δ u + " << -poles[i]
-                   << " u = " << coeffs[i] << " f " << endl;
-      }
-      CGSolver cg(row_comm);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(print_level);
-      cg.SetPreconditioner(*prec);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
-
-      // 14. Recover the solution as a finite element grid function.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 15. Accumulate integer-order PDE solutions.
-      x *= coeffs[i];
-      u += x;
-
-      // 16. Send integer-order PDE solutions to a GLVis server.
-      if (visualize_x)
-      {
-         if (col_rank > 0 && i < iend-1)
-         {
-            MPI_Status status;
-            MPI_Recv(nullptr,0,MPI_INT, col_rank-1,0,col_comm,&status);
-         }
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream xout(vishost, visport);
-         xout.precision(8);
-         ostringstream oss;
-         oss << "Solution of PDE -Δ u + " << -poles[i]
-             << " u = " << coeffs[i] << " f" ;
-         xout << "parallel " << row_size << " " << row_rank << "\n";
-         xout << "solution\n" << pmesh << x
-              << "window_title '" << oss.str() << "'" << flush;
-         if (col_rank < col_size-1)
-         {
-            MPI_Send(nullptr,0,MPI_INT,col_rank+1,0,col_comm);
-         }
-      }
-   }
-
-   // 17. Accumulate for the fractional PDE solution
-   MPI_Allreduce(MPI_IN_PLACE, u.GetData(), u.Size(),
-                 MPI_DOUBLE, MPI_SUM,col_comm);
-
-   // 18. Send fractional PDE solution to a GLVis server.
-   if (visualization)
-   {
-      if (col_rank == 0)
-      {
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream uout(vishost, visport);
-         uout.precision(8);
-         ostringstream oss;
-         oss << "Solution of fractional PDE -Δ^" << alpha
-             << " u = f" ;
-         uout << "parallel " << row_size << " " << row_rank << "\n";
-         uout << "solution\n" << pmesh << u
-              << "window_title '" << oss.str() << "'" << flush;
-      }
-   }
-
-   return 0;
-}
@@ -55,7 +55,6 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   int max_dofs = 50000;
-   bool LSZZ = false;
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -69,9 +68,6 @@ int main(int argc, char *argv[])
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&max_dofs, "-md", "--max-dofs",
                  "Stop after reaching this many degrees of freedom.");
-   args.AddOption(&LSZZ, "-ls", "--ls-zz", "-no-ls",
-                  "--no-ls-zz",
-                  "Switch to least-squares ZZ estimator.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -155,29 +151,15 @@ int main(int argc, char *argv[])
   //     recover a smoothed flux (gradient) that is subtracted from the element
   //     flux to get an error indicator. We need to supply the space for the
   //     smoothed flux: an (H1)^sdim (i.e., vector-valued) space is used here.
-   ErrorEstimator *estimator{nullptr};
-
-   if (LSZZ)
-   {
-      estimator = new LSZienkiewiczZhuEstimator(*integ, x);
-      if (dim == 3 && mesh.GetElementType(0) != Element::HEXAHEDRON)
-      {
-         dynamic_cast<LSZienkiewiczZhuEstimator *>
-         (estimator)->SetTichonovRegularization();
-      }
-   }
-   else
-   {
-      auto flux_fes = new FiniteElementSpace(&mesh, &fec, sdim);
-      estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
-      dynamic_cast<ZienkiewiczZhuEstimator *>(estimator)->SetAnisotropic();
-   }
+   FiniteElementSpace flux_fespace(&mesh, &fec, sdim);
+   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
+   estimator.SetAnisotropic();

   // 11. A refiner selects and refines elements based on a refinement strategy.
   //     The strategy here is to refine elements with errors larger than a
   //     fraction of the maximum element error. Other strategies are possible.
   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(*estimator);
+   ThresholdRefiner refiner(estimator);
   refiner.SetTotalErrorFraction(0.7);

   // 12. The main AMR loop. In each iteration we solve the problem on the
@@ -274,6 +256,5 @@ int main(int argc, char *argv[])
      b.Update();
   }

-   delete estimator;
   return 0;
 }
@@ -72,8 +72,8 @@ int main(int argc, char *argv[])
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
   {
-      int ref_levels =
-         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels = 1;
+      // (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
@@ -147,6 +147,8 @@ int main(int argc, char *argv[])
   F.AddDomainIntegrator(new DomainLFIntegrator(one));
   F.Assemble();

+
+
   // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
   //    the mixed bilinear form for the interfacial unknowns, Bhat,
   //    the inverse stiffness matrix on the discontinuous test space, Sinv,
@@ -187,10 +189,17 @@ int main(int argc, char *argv[])
   // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
   //    the normal equation operator, A = B^t Sinv B, and
   //    the normal equation right-hand-size, b = B^t Sinv F.
-   BlockOperator B(offsets_test, offsets);
+   // BlockOperator B(offsets_test, offsets);
+
+   BlockMatrix B(offsets_test, offsets);
   B.SetBlock(0,0,&matB0);
   B.SetBlock(0,1,&matBhat);
-   RAPOperator A(B, matSinv, B);
+   SparseMatrix * Bh = B.CreateMonolithic();
+
+   SparseMatrix * A = RAP(*Bh, matSinv, *Bh);
+
+
+   // RAPOperator A(B, matSinv, B);
   {
      Vector SinvF(s_test);
      matSinv.Mult(F,SinvF);
@@ -234,7 +243,20 @@ int main(int argc, char *argv[])
   // 10. Solve the normal equation system using the PCG iterative solver.
   //     Check the weighted norm of residual for the DPG least square problem.
   //     Wrap the primal variable in a GridFunction for visualization purposes.
-   PCG(A, P, b, x, 1, 200, 1e-12, 0.0);
+
+
+   // PCG(*A, P, b, x, 1, 200, 1e-12, 0.0);
+
+   GSSmoother M(*A);
+
+   CGSolver cg;
+   cg.SetRelTol(1e-12);
+   cg.SetMaxIter(2000);
+   cg.SetPrintLevel(3);
+   cg.SetPreconditioner(M);
+   cg.SetOperator(*A);
+   cg.Mult(b, x);
+

   {
      Vector LSres(s_test);
@@ -23,10 +23,10 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33
+  ex31
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p
+  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
  ex24p ex25p ex26p
@@ -58,9 +58,6 @@ endif
 ifeq ($(MFEM_USE_SUPERLU),YES)
   SUBDIRS += superlu
 endif
-ifeq ($(MFEM_USE_MOONOLITH),YES)
-   SUBDIRS += moonolith
-endif
 ifeq ($(MFEM_USE_CALIPER),YES)
   SUBDIRS += caliper
 endif
@@ -91,11 +88,8 @@ $(SUBDIRS_TPRINT):

 # Additional dependencies
 ex18: $(SRC)ex18.hpp
-ex33: $(SRC)ex33.hpp
-
 ifeq ($(MFEM_USE_MPI),YES)
 ex18p: $(SRC)ex18.hpp
-ex33p: $(SRC)ex33.hpp
 endif

 MFEM_TESTS = EXAMPLES
@@ -1,57 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-set(MOONOLITH_EXAMPLES_SRCS)
-list(APPEND MOONOLITH_EXAMPLES_SRCS ex1.cpp)
-
-if (MFEM_USE_MPI)
-  list(APPEND MOONOLITH_EXAMPLES_SRCS ex1p.cpp ex2p.cpp)
-endif()
-
-# Include the source directory where mfem.hpp and mfem-performance.hpp are.
-include_directories(BEFORE ${PROJECT_BINARY_DIR})
-
-# Add "test_moonolith" target, see below.
-add_custom_target(test_moonolith
-  ${CMAKE_CTEST_COMMAND} -R moonolith USES_TERMINAL)
-
-# Add one executable per cpp file, adding "moonolith_" as prefix. Sets
-# "test_moonolith" as a target that depends on the given examples.
-set(PFX moonolith_)
-add_mfem_examples(MOONOLITH_EXAMPLES_SRCS ${PFX} "" test_moonolith)
-
-# Testing.
-# The MOONOLITH tests can be run separately using the target "test_moonolith"
-# which builds the examples and runs:
-#   ctest -R moonolith
-
-# Add the tests: one test per source file.
-foreach(SRC_FILE ${MOONOLITH_EXAMPLES_SRCS})
-  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
-  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
-  set(TEST_NAME ${PFX}${TEST_NAME})
-
-  set(THIS_TEST_OPTIONS "-no-vis")
-  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
-  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
-
-  if (NOT (${TEST_NAME} MATCHES ".*p$"))
-    add_test(NAME ${TEST_NAME}_ser
-      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
-  else()
-    add_test(NAME ${TEST_NAME}_np=4
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
-      ${MPIEXEC_POSTFLAGS})
-  endif()
-endforeach()
@@ -1,18 +0,0 @@
-                    Finite Element Discretization Library
-                                   __
-                       _ __ ___   / _|  ___  _ __ ___
-                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
-                      | | | | | ||  _||  __/| | | | | |
-                      |_| |_| |_||_|   \___||_| |_| |_|
-
-                               https://mfem.org
-
-This directory contains modifications of the example codes that illustrate the
-use of MFEM features based on ParMoonolith library for variational information
-transfer between non-matching meshes.
-
-To build these examples, make sure that MFEM is configured with the option
-"MFEM_USE_MOONOLITH = YES".
-
-For the parallel version add also the options MFEM_USE_MPI = YES", and
-MFEM_USE_METIS = YES". See the top-level INSTALL file for details.
@@ -1,229 +0,0 @@
-//                         MFEM + Moonolith Example 1
-//
-// Compile with: make ex1
-//
-// Moonolith sample runs:
-//   ex1
-//   ex1 --source_refinements 1 --dest_refinements 2
-//   ex1 --source_refinements 1 --dest_refinements 2 --use_vector_fe
-//   ex1 -s ../../data/inline-hex.mesh -d ../../data/inline-tet.mesh
-//
-// Description:  This example code demonstrates the use of MFEM for transferring
-//               discrete fields from one finite element mesh to another. The
-//               meshes can be of arbitrary shape and completely unrelated with
-//               each other. This feature can be used for implementing immersed
-//               domain methods for fluid-structure interaction or general
-//               multi-physics applications.
-//
-//               This particular example is only for serial runtimes.
-
-#include "example_utils.hpp"
-#include "mfem.hpp"
-
-using namespace mfem;
-using namespace std;
-
-int main(int argc, char *argv[])
-{
-   // Init transfer library context
-   InitTransfer(argc, argv);
-
-   const char *source_mesh_file = "../../data/inline-tri.mesh";
-   const char *destination_mesh_file = "../../data/inline-quad.mesh";
-
-   int src_n_refinements = 0;
-   int dest_n_refinements = 0;
-   int source_fe_order = 1;
-   int dest_fe_order = 1;
-   bool visualization = true;
-   bool use_vector_fe = false;
-   bool verbose = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&source_mesh_file, "-s", "--source_mesh",
-                  "Mesh file to use for src.");
-   args.AddOption(&destination_mesh_file, "-d", "--destination_mesh",
-                  "Mesh file to use for dest.");
-   args.AddOption(&src_n_refinements, "-sr", "--source_refinements",
-                  "Number of src refinements");
-   args.AddOption(&dest_n_refinements, "-dr", "--dest_refinements",
-                  "Number of dest refinements");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&source_fe_order, "-so", "--source_fe_order",
-                  "Order of the src finite elements");
-   args.AddOption(&dest_fe_order, "-do", "--dest_fe_order",
-                  "Order of the dest finite elements");
-   args.AddOption(&verbose, "-verb", "--verbose", "--no-verb", "--no-verbose",
-                  "Enable/Disable verbose output");
-   args.AddOption(&use_vector_fe, "-vfe", "--use_vector_fe", "-no-vfe",
-                  "--no-vector_fe", "Use vector finite elements");
-   args.Parse();
-   check_options(args);
-
-   shared_ptr<Mesh> src_mesh, dest_mesh;
-
-   ifstream imesh;
-
-   imesh.open(destination_mesh_file);
-   if (imesh)
-   {
-      dest_mesh = make_shared<Mesh>(imesh, 1, 1);
-      imesh.close();
-   }
-   else
-   {
-      mfem::err << "WARNING: Destination mesh file not found: "
-                << destination_mesh_file << "\n"
-                << "Using default 2D quad mesh.";
-
-      dest_mesh = make_shared<Mesh>(4, 4, Element::QUADRILATERAL);
-   }
-
-   const int dim = dest_mesh->Dimension();
-
-   Vector box_min(dim), box_max(dim), range(dim);
-   dest_mesh->GetBoundingBox(box_min, box_max);
-   range = box_max;
-   range -= box_min;
-
-   imesh.open(source_mesh_file);
-
-   if (imesh)
-   {
-      src_mesh = make_shared<Mesh>(imesh, 1, 1);
-      imesh.close();
-   }
-   else
-   {
-      mfem::err << "WARNING: Source mesh file not found: " << source_mesh_file
-                << "\n"
-                << "Using default box mesh.\n";
-
-      if (dim == 2)
-      {
-         src_mesh =
-            make_shared<Mesh>(4, 4, Element::TRIANGLE, 1, range[0], range[1]);
-      }
-      else if (dim == 3)
-      {
-         src_mesh = make_shared<Mesh>(4, 4, 4, Element::TETRAHEDRON, 1, range[0],
-                                      range[1], range[2]);
-      }
-
-      for (int i = 0; i < src_mesh->GetNV(); ++i)
-      {
-         double *v = src_mesh->GetVertex(i);
-
-         for (int d = 0; d < dim; ++d)
-         {
-            v[d] += box_min[d];
-         }
-      }
-   }
-
-   for (int i = 0; i < src_n_refinements; ++i)
-   {
-      src_mesh->UniformRefinement();
-   }
-
-   for (int i = 0; i < dest_n_refinements; ++i)
-   {
-      dest_mesh->UniformRefinement();
-   }
-
-   shared_ptr<FiniteElementCollection> src_fe_coll, dest_fe_coll;
-
-   if (use_vector_fe)
-   {
-      src_fe_coll =
-         make_shared<RT_FECollection>(source_fe_order, src_mesh->Dimension());
-      dest_fe_coll =
-         make_shared<RT_FECollection>(dest_fe_order, dest_mesh->Dimension());
-   }
-   else
-   {
-      src_fe_coll =
-         make_shared<L2_FECollection>(source_fe_order, src_mesh->Dimension());
-      dest_fe_coll =
-         make_shared<L2_FECollection>(dest_fe_order, dest_mesh->Dimension());
-   }
-
-   auto src_fe =
-      make_shared<FiniteElementSpace>(src_mesh.get(), src_fe_coll.get());
-
-   auto dest_fe =
-      make_shared<FiniteElementSpace>(dest_mesh.get(), dest_fe_coll.get());
-
-   GridFunction src_fun(src_fe.get());
-   GridFunction dest_fun(dest_fe.get());
-   src_fun = 1.0;
-
-   // To be used with standard fe
-   FunctionCoefficient coeff(example_fun);
-
-   // To be used with vector fe
-   VectorFunctionCoefficient vector_coeff(dim, &vector_fun);
-
-   if (use_vector_fe)
-   {
-      src_fun.ProjectCoefficient(vector_coeff);
-      src_fun.Update();
-   }
-   else
-   {
-      src_fun.ProjectCoefficient(coeff);
-      src_fun.Update();
-   }
-
-   dest_fun = 0.0;
-   dest_fun.Update();
-
-   MortarAssembler assembler(src_fe, dest_fe);
-   assembler.SetVerbose(verbose);
-
-   if (use_vector_fe)
-   {
-      assembler.AddMortarIntegrator(make_shared<VectorL2MortarIntegrator>());
-   }
-   else
-   {
-      assembler.AddMortarIntegrator(make_shared<L2MortarIntegrator>());
-   }
-
-   if (assembler.Transfer(src_fun, dest_fun))
-   {
-      if (visualization)
-      {
-         dest_fun.Update();
-
-         double src_err = 0;
-         double dest_err = 0;
-
-         if (use_vector_fe)
-         {
-            src_err = src_fun.ComputeL2Error(vector_coeff);
-            dest_err = dest_fun.ComputeL2Error(vector_coeff);
-         }
-         else
-         {
-            src_err = src_fun.ComputeL2Error(coeff);
-            dest_err = dest_fun.ComputeL2Error(coeff);
-         }
-
-         mfem::out << "l2 error: src: " << src_err << ", dest: " << dest_err
-                   << std::endl;
-
-         plot(*src_mesh, src_fun, "source");
-         plot(*dest_mesh, dest_fun, "destination");
-      }
-   }
-   else
-   {
-      mfem::out << "No intersection -> no transfer!" << std::endl;
-   }
-
-   // Finalize transfer library context
-   return FinalizeTransfer();
-}
@@ -1,258 +0,0 @@
-//                   MFEM + Moonolith Example 1 (parallel version)
-//
-// Compile with: make ex1p
-//
-// Moonolith sample runs:
-//   mpirun -np 4 ex1p
-//   mpirun -np 4 ex1p --source_refinements 1 --dest_refinements 2
-//   mpirun -np 4 ex1p -s ../../data/inline-hex.mesh -d ../../data/inline-tet.mesh
-//
-// Description:  This example code demonstrates the use of MFEM for transferring
-//               discrete fields from one finite element mesh to another. The
-//               meshes can be of arbitrary shape and completely unrelated with
-//               each other. This feature can be used for implementing immersed
-//               domain methods for fluid-structure interaction or general
-//               multi-physics applications.
-//
-//               This particular example is for parallel runtimes. Vector FE is
-//               an experimental feature in parallel.
-
-#include "example_utils.hpp"
-#include "mfem.hpp"
-
-using namespace mfem;
-using namespace std;
-
-void destination_transform(const Vector &x, Vector &x_new)
-{
-   x_new = x;
-   // x_new *= 0.5;
-}
-
-int main(int argc, char *argv[])
-{
-   MPI_Init(&argc, &argv);
-
-   int num_procs, rank;
-
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-   // Init transfer library context, with MPI handled outside the library
-   InitTransfer(argc, argv, MPI_COMM_WORLD);
-
-   const char *source_mesh_file = "../../data/inline-tri.mesh";
-   const char *destination_mesh_file = "../../data/inline-quad.mesh";
-
-   int src_n_refinements = 0;
-   int dest_n_refinements = 0;
-   int source_fe_order = 1;
-   int dest_fe_order = 1;
-   bool visualization = true;
-   bool use_vector_fe = false;
-   bool verbose = false;
-   bool assemble_mass_and_coupling_together = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&source_mesh_file, "-s", "--source_mesh",
-                  "Mesh file to use for src.");
-   args.AddOption(&destination_mesh_file, "-d", "--destination_mesh",
-                  "Mesh file to use for dest.");
-   args.AddOption(&src_n_refinements, "-sr", "--source_refinements",
-                  "Number of src refinements");
-   args.AddOption(&dest_n_refinements, "-dr", "--dest_refinements",
-                  "Number of dest refinements");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&source_fe_order, "-so", "--source_fe_order",
-                  "Order of the src finite elements");
-   args.AddOption(&dest_fe_order, "-do", "--dest_fe_order",
-                  "Order of the dest finite elements");
-   args.AddOption(&verbose, "-verb", "--verbose", "--no-verb", "--no-verbose",
-                  "Enable/Disable verbose output");
-   args.AddOption(&use_vector_fe, "-vfe", "--use_vector_fe", "-no-vfe",
-                  "--no-vector_fe", "Use vector finite elements (Experimental)");
-   args.AddOption(&assemble_mass_and_coupling_together, "-act",
-                  "--assemble_mass_and_coupling_together", "-no-act",
-                  "--no-assemble_mass_and_coupling_together",
-                  "Assemble mass and coupling operators together (better for non-affine elements)");
-   args.Parse();
-   check_options(args);
-
-   shared_ptr<Mesh> src_mesh, dest_mesh;
-
-   ifstream imesh;
-
-   imesh.open(destination_mesh_file);
-   if (imesh)
-   {
-      dest_mesh = make_shared<Mesh>(imesh, 1, 1);
-      imesh.close();
-   }
-   else
-   {
-      if (rank == 0)
-         mfem::err << "WARNING: Destination mesh file not found: "
-                   << destination_mesh_file << "\n"
-                   << "Using default 2D quad mesh.";
-
-      dest_mesh = make_shared<Mesh>(4, 4, Element::QUADRILATERAL);
-   }
-
-   const int dim = dest_mesh->Dimension();
-
-   dest_mesh->Transform(&destination_transform);
-
-   Vector box_min(dim), box_max(dim), range(dim);
-   dest_mesh->GetBoundingBox(box_min, box_max);
-   range = box_max;
-   range -= box_min;
-
-   imesh.open(source_mesh_file);
-
-   if (imesh)
-   {
-      src_mesh = make_shared<Mesh>(imesh, 1, 1);
-      imesh.close();
-   }
-   else
-   {
-      if (rank == 0)
-         mfem::err << "WARNING: Source mesh file not found: " << source_mesh_file
-                   << "\n"
-                   << "Using default box mesh.\n";
-
-      if (dim == 2)
-      {
-         src_mesh =
-            make_shared<Mesh>(4, 4, Element::TRIANGLE, 1, range[0], range[1]);
-      }
-      else if (dim == 3)
-      {
-         src_mesh = make_shared<Mesh>(4, 4, 4, Element::TETRAHEDRON, 1, range[0],
-                                      range[1], range[2]);
-      }
-
-      for (int i = 0; i < src_mesh->GetNV(); ++i)
-      {
-         double *v = src_mesh->GetVertex(i);
-
-         for (int d = 0; d < dim; ++d)
-         {
-            v[d] += box_min[d];
-         }
-      }
-   }
-
-   for (int i = 0; i < src_n_refinements; ++i)
-   {
-      src_mesh->UniformRefinement();
-   }
-
-   for (int i = 0; i < dest_n_refinements; ++i)
-   {
-      dest_mesh->UniformRefinement();
-   }
-
-   auto p_src_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *src_mesh);
-   auto p_dest_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *dest_mesh);
-
-   shared_ptr<FiniteElementCollection> src_fe_coll, dest_fe_coll;
-
-   if (use_vector_fe)
-   {
-      src_fe_coll =
-         make_shared<RT_FECollection>(source_fe_order, src_mesh->Dimension());
-      dest_fe_coll =
-         make_shared<RT_FECollection>(dest_fe_order, dest_mesh->Dimension());
-   }
-   else
-   {
-      src_fe_coll =
-         make_shared<L2_FECollection>(source_fe_order, src_mesh->Dimension());
-      dest_fe_coll =
-         make_shared<L2_FECollection>(dest_fe_order, dest_mesh->Dimension());
-   }
-
-   auto src_fe =
-      make_shared<ParFiniteElementSpace>(p_src_mesh.get(), src_fe_coll.get());
-
-   auto dest_fe =
-      make_shared<ParFiniteElementSpace>(p_dest_mesh.get(), dest_fe_coll.get());
-
-   ParGridFunction src_fun(src_fe.get());
-
-   // To be used with standard fe
-   FunctionCoefficient coeff(example_fun);
-
-   // To be used with vector fe
-   VectorFunctionCoefficient vector_coeff(dim, &vector_fun);
-
-   if (use_vector_fe)
-   {
-      src_fun.ProjectCoefficient(vector_coeff);
-      src_fun.Update();
-   }
-   else
-   {
-      src_fun.ProjectCoefficient(coeff);
-      src_fun.Update();
-   }
-
-   ParGridFunction dest_fun(dest_fe.get());
-   dest_fun = 0.0;
-   dest_fun.Update();
-
-   ParMortarAssembler assembler(src_fe, dest_fe);
-   assembler.SetAssembleMassAndCouplingTogether(
-      assemble_mass_and_coupling_together);
-   assembler.SetVerbose(verbose);
-
-   if (use_vector_fe)
-   {
-      assembler.AddMortarIntegrator(make_shared<VectorL2MortarIntegrator>());
-   }
-   else
-   {
-      assembler.AddMortarIntegrator(make_shared<L2MortarIntegrator>());
-   }
-
-   if (assembler.Transfer(src_fun, dest_fun))
-   {
-
-      if (visualization)
-      {
-         double src_err = 0;
-         double dest_err = 0;
-
-         if (use_vector_fe)
-         {
-            src_err = src_fun.ComputeL2Error(vector_coeff);
-            dest_err = dest_fun.ComputeL2Error(vector_coeff);
-         }
-         else
-         {
-            src_err = src_fun.ComputeL2Error(coeff);
-            dest_err = dest_fun.ComputeL2Error(coeff);
-         }
-
-         if (rank == 0)
-         {
-            mfem::out << "l2 error: src: " << src_err << ", dest: " << dest_err
-                      << std::endl;
-         }
-
-         plot(*p_src_mesh, src_fun, "source");
-         plot(*p_dest_mesh, dest_fun, "destination");
-      }
-   }
-   else
-   {
-      mfem::out << "No intersection no transfer!" << std::endl;
-   }
-
-   // Finalize transfer library context
-   FinalizeTransfer();
-   return MPI_Finalize();
-}
@@ -1,202 +0,0 @@
-//                  MFEM + Moonolith Example 2 (parallel version)
-//
-// Compile with: make ex2p
-//
-// Moonolith sample runs:
-//   mpirun -np 4 ex2p
-//   mpirun -np 4 ex2p --source_refinements 1 --dest_refinements 2
-//   mpirun -np 4 ex2p -s ../../data/inline-hex.mesh -d ../../data/inline-tet.mesh
-//
-// Description:  This example code demonstrates the use of MFEM for transferring
-//               discrete fields from one finite element mesh to another. The
-//               meshes can be of arbitrary shape and completely unrelated with
-//               each other. This feature can be used for implementing immersed
-//               domain methods for fluid-structure interaction or general
-//               multi-physics applications.
-//
-//               This particular example concerns discontinuous Galerkin FEM with
-//               adaptive mesh refinement for parallel runtimes.
-
-#include "example_utils.hpp"
-#include "mfem.hpp"
-
-using namespace mfem;
-using namespace std;
-
-void destination_transform(const Vector &x, Vector &x_new)
-{
-   x_new = x;
-   // x_new *= .5;
-}
-
-int main(int argc, char *argv[])
-{
-   MPI_Init(&argc, &argv);
-
-   int num_procs, rank;
-
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-
-   // Init transfer library context, with MPI handled outside the library
-   InitTransfer(argc, argv, MPI_COMM_WORLD);
-
-   const char *source_mesh_file = "../../data/inline-tri.mesh";
-   const char *destination_mesh_file = "../../data/inline-quad.mesh";
-
-   int src_n_refinements = 0;
-   int dest_n_refinements = 0;
-
-   // Source fe order has to be greater or equal than destination order
-   int source_fe_order = 1;
-   int dest_fe_order = 0;
-   bool visualization = true;
-   bool verbose = false;
-   int max_iterations = 30000;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&source_mesh_file, "-s", "--source_mesh",
-                  "Mesh file to use for src.");
-   args.AddOption(&destination_mesh_file, "-d", "--destination_mesh",
-                  "Mesh file to use for dest.");
-   args.AddOption(&src_n_refinements, "-sr", "--source_refinements",
-                  "Number of src refinements");
-   args.AddOption(&dest_n_refinements, "-dr", "--dest_refinements",
-                  "Number of dest refinements");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&source_fe_order, "-so", "--source_fe_order",
-                  "Order of the src finite elements");
-   args.AddOption(&dest_fe_order, "-do", "--dest_fe_order",
-                  "Order of the dest finite elements");
-   args.AddOption(&verbose, "-verb", "--verbose", "--no-verb", "--no-verbose",
-                  "Enable/Disable verbose output");
-   args.AddOption(&max_iterations, "-m", "--max_iterations",
-                  "Max number of solver iterations");
-   args.Parse();
-   check_options(args);
-
-   if (source_fe_order == 0 &&  dest_fe_order != 0)
-   {
-      mfem::out <<
-                "Source fe order should not be 0 unless destination fe order is also 0!\n";
-
-      FinalizeTransfer();
-      return MPI_Finalize();
-   }
-
-   ifstream imesh(source_mesh_file);
-   shared_ptr<Mesh> src_mesh, dest_mesh;
-   if (imesh)
-   {
-      src_mesh = make_shared<Mesh>(imesh, 1, 1);
-      imesh.close();
-   }
-   else
-   {
-      if (rank == 0)
-         mfem::err << "WARNING: Source mesh file not found: " << source_mesh_file
-                   << "\n"
-                   << "Using default 2D triangle mesh.";
-      src_mesh = make_shared<Mesh>(4, 4, Element::TRIANGLE);
-   }
-
-   imesh.open(destination_mesh_file);
-   if (imesh)
-   {
-      dest_mesh = make_shared<Mesh>(imesh, 1, 1);
-      imesh.close();
-   }
-   else
-   {
-      if (rank == 0)
-         mfem::err << "WARNING: Destination mesh file not found: "
-                   << destination_mesh_file << "\n"
-                   << "Using default 2D quad mesh.";
-      dest_mesh = make_shared<Mesh>(4, 4, Element::QUADRILATERAL);
-   }
-
-   dest_mesh->Transform(&destination_transform);
-
-   for (int i = 0; i < src_n_refinements; ++i)
-   {
-      src_mesh->UniformRefinement();
-   }
-
-   for (int i = 0; i < dest_n_refinements; ++i)
-   {
-      dest_mesh->UniformRefinement();
-   }
-
-   src_mesh->EnsureNCMesh();
-   dest_mesh->EnsureNCMesh();
-
-   {
-      for (int l = 0; l < 4; l++)
-      {
-         src_mesh->RandomRefinement(0.1); // 10% probability
-      }
-   }
-
-   {
-      for (int l = 0; l < 4; l++)
-      {
-         dest_mesh->RandomRefinement(0.1); // 10% probability
-      }
-   }
-
-   auto p_src_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *src_mesh);
-   auto p_dest_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *dest_mesh);
-
-   auto src_fe_coll =
-      make_shared<DG_FECollection>(source_fe_order, p_src_mesh->Dimension());
-   auto src_fe =
-      make_shared<ParFiniteElementSpace>(p_src_mesh.get(), src_fe_coll.get());
-
-   auto dest_fe_coll =
-      make_shared<DG_FECollection>(dest_fe_order, p_dest_mesh->Dimension());
-   auto dest_fe =
-      make_shared<ParFiniteElementSpace>(p_dest_mesh.get(), dest_fe_coll.get());
-
-   ParGridFunction src_fun(src_fe.get());
-   FunctionCoefficient coeff(example_fun);
-   make_fun(*src_fe, coeff, src_fun);
-
-   ParGridFunction dest_fun(dest_fe.get());
-   dest_fun = 0.0;
-   dest_fun.Update();
-
-   ParMortarAssembler assembler(src_fe, dest_fe);
-   assembler.SetVerbose(verbose);
-   assembler.SetMaxSolverIterations(max_iterations);
-
-   assembler.AddMortarIntegrator(make_shared<L2MortarIntegrator>());
-   if (assembler.Transfer(src_fun, dest_fun))
-   {
-      if (visualization)
-      {
-
-         const double src_err = src_fun.ComputeL2Error(coeff);
-         const double dest_err = dest_fun.ComputeL2Error(coeff);
-
-         if (rank == 0)
-         {
-            mfem::out << "l2 error: src: " << src_err << ", dest: " << dest_err
-                      << std::endl;
-         }
-
-         plot(*p_src_mesh, src_fun, "source");
-         plot(*p_dest_mesh, dest_fun, "destination");
-      }
-   }
-   else
-   {
-      mfem::out << "Transfer failed! Use --verbose option for diagnostic!" <<
-                std::endl;
-   }
-
-   // Finalize transfer library context
-   FinalizeTransfer();
-   return MPI_Finalize();
-}
@@ -1,96 +0,0 @@
-#include <algorithm>
-#include <assert.h>
-#include <cstdlib>
-#include <memory>
-
-#ifdef MFEM_USE_MPI
-#include <mpi.h>
-#endif // MFEM_USE_MPI
-
-#include "mfem.hpp"
-
-inline void check_options(mfem::OptionsParser &args)
-{
-   using namespace std;
-   using namespace mfem;
-
-   int rank = 0;
-#ifdef MFEM_USE_MPI
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-#endif // MFEM_USE_MPI
-
-   if (!args.Good())
-   {
-      if (rank == 0)
-      {
-         args.PrintUsage(cout);
-      }
-
-#ifdef MFEM_USE_MPI
-      MPI_Finalize();
-      MPI_Abort(MPI_COMM_WORLD, 1);
-#else
-      abort();
-#endif // MFEM_USE_MPI
-   }
-
-   if (rank == 0)
-   {
-      args.PrintOptions(cout);
-   }
-}
-
-inline void make_fun(mfem::FiniteElementSpace &fe, mfem::Coefficient &c,
-                     mfem::GridFunction &f)
-{
-   using namespace std;
-   using namespace mfem;
-
-   f.SetSpace(&fe);
-   f.ProjectCoefficient(c);
-   f.Update();
-}
-
-inline double example_fun(const mfem::Vector &x)
-{
-   using namespace std;
-   using namespace mfem;
-
-   const int n = x.Size();
-   double ret = 0;
-   for (int k = 0; k < n; ++k)
-   {
-      ret += x(k) * x(k);
-   }
-
-   return sqrt(ret);
-}
-
-void vector_fun(const mfem::Vector &x, mfem::Vector &f)
-{
-   const double n = x.Norml2();
-   f.SetSize(x.Size());
-   f = n;
-}
-
-inline void plot(mfem::Mesh &mesh, mfem::GridFunction &x, std::string title)
-{
-   using namespace std;
-   using namespace mfem;
-
-   int num_procs = 1, rank = 0;
-
-#ifdef MFEM_USE_MPI
-   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
-#endif // MFEM_USE_MPI
-
-   char vishost[] = "localhost";
-   int visport = 19916;
-   socketstream sol_sock(vishost, visport);
-   sol_sock << "parallel " << num_procs << " " << rank << "\n";
-   sol_sock.precision(8);
-   sol_sock << "solution\n" << mesh << x
-            << "window_title '"<< title << "'\n" << flush;
-   sol_sock << flush;
-}
@@ -58,29 +58,17 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  ceed/interface/basis.cpp
-  ceed/interface/restriction.cpp
-  ceed/interface/operator.cpp
-  ceed/interface/util.cpp
-  ceed/integrators/convection/convection.cpp
-  ceed/integrators/diffusion/diffusion.cpp
-  ceed/integrators/nlconvection/nlconvection.cpp
-  ceed/integrators/mass/mass.cpp
-  ceed/solvers/algebraic.cpp
-  ceed/solvers/full-assembly.cpp
-  ceed/solvers/solvers-atpmg.cpp
+  ceed/algebraic.cpp
+  ceed/full-assembly.cpp
+  ceed/solvers-atpmg.cpp
+  ceed/convection.cpp
+  ceed/diffusion.cpp
+  ceed/nlconvection.cpp
+  ceed/mass.cpp
+  ceed/operator.cpp
+  ceed/util.cpp
  linearform.cpp
-  linearform_ext.cpp
  lininteg.cpp
-  lininteg_domain.cpp
-  lininteg_domain_grad.cpp
-  lor/lor.cpp
-  lor/lor_ads.cpp
-  lor/lor_ams.cpp
-  lor/lor_batched.cpp
-  lor/lor_h1.cpp
-  lor/lor_nd.cpp
-  lor/lor_rt.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -130,6 +118,7 @@ set(SRCS
  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
+  lor.cpp
  )

 set(HDRS
@@ -160,31 +149,19 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  ceed/interface/basis.hpp
-  ceed/interface/integrator.hpp
-  ceed/interface/interface.hpp
-  ceed/interface/operator.hpp
-  ceed/interface/restriction.hpp
-  ceed/interface/util.hpp
-  ceed/integrators/convection/convection.hpp
-  ceed/integrators/diffusion/diffusion.hpp
-  ceed/integrators/mass/mass.hpp
-  ceed/integrators/nlconvection/nlconvection.hpp
-  ceed/interface/coefficient.hpp
-  ceed/solvers/algebraic.hpp
-  ceed/solvers/full-assembly.hpp
-  ceed/solvers/solvers-atpmg.hpp
+  ceed/algebraic.hpp
+  ceed/full-assembly.hpp
+  ceed/solvers-atpmg.hpp
+  ceed/coefficient.hpp
+  ceed/convection.hpp
+  ceed/diffusion.hpp
+  ceed/integrator.hpp
+  ceed/mass.hpp
+  ceed/nlconvection.hpp
+  ceed/operator.hpp
+  ceed/util.hpp
  linearform.hpp
-  linearform_ext.hpp
  lininteg.hpp
-  lor/lor.hpp
-  lor/lor_ads.hpp
-  lor/lor_ams.hpp
-  lor/lor_batched.hpp
-  lor/lor_h1.hpp
-  lor/lor_nd.hpp
-  lor/lor_rt.hpp
-  lor/lor_util.hpp
  multigrid.hpp
  nonlinearform.hpp
  nonlinearform_ext.hpp
@@ -211,6 +188,7 @@ set(HDRS
  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
+  lor.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -1763,24 +1763,12 @@ void MixedBilinearForm::FormRectangularSystemMatrix(

   mat->Finalize();

-   if (test_P && trial_P)
+   if (test_P) // TODO: Must actually check for trial_P too
   {
      SparseMatrix *m = RAP(*test_P, *mat, *trial_P);
      delete mat;
      mat = m;
   }
-   else if (test_P)
-   {
-      SparseMatrix *m = TransposeMult(*test_P, *mat);
-      delete mat;
-      mat = m;
-   }
-   else if (trial_P)
-   {
-      SparseMatrix *m = mfem::Mult(*mat, *trial_P);
-      delete mat;
-      mat = m;
-   }

   Array<int> ess_trial_tdof_marker, ess_test_tdof_marker;
   FiniteElementSpace::ListToMarker(trial_tdof_list, trial_fes->GetTrueVSize(),
@@ -16,7 +16,7 @@
 #include "bilinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pgridfunc.hpp"
-#include "ceed/interface/util.hpp"
+#include "ceed/util.hpp"

 namespace mfem
 {
@@ -903,8 +903,7 @@ void FABilinearFormExtension::Assemble()
   }
   else // We create, compute the sparsity, and fill the sparse matrix
   {
-      mat = new SparseMatrix;
-      mat->OverrideSize(height, width);
+      mat = new SparseMatrix(height, width, 0);
      if (fes.IsDGSpace())
      {
         const L2ElementRestriction *restE =
@@ -144,6 +144,14 @@ void BilinearFormIntegrator::AssembleFaceMatrix (
               "   is not implemented for this class.");
 }

+void BilinearFormIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                     const FiniteElement &trial_face_fe,  const FiniteElement &test_fe,
+                                                     FaceElementTransformations &Trans, DenseMatrix &elmat)
+{
+   mfem_error ("BilinearFormIntegrator::AssembleTraceFaceMatrix(...)\n"
+               "   is not implemented for this class.");
+}
+
 void BilinearFormIntegrator::AssembleFaceMatrix(
   const FiniteElement &trial_face_fe, const FiniteElement &test_fe1,
   const FiniteElement &test_fe2, FaceElementTransformations &Trans,
@@ -1036,8 +1044,7 @@ void DiffusionIntegrator::AssembleElementVector(

 void DiffusionIntegrator::ComputeElementFlux
 ( const FiniteElement &el, ElementTransformation &Trans,
-  Vector &u, const FiniteElement &fluxelem, Vector &flux, bool with_coef,
-  const IntegrationRule *ir)
+  Vector &u, const FiniteElement &fluxelem, Vector &flux, bool with_coef )
 {
   int nd, spaceDim, fnd;

@@ -1058,6 +1065,8 @@ void DiffusionIntegrator::ComputeElementFlux
                  "Unexpected height for MatrixCoefficient");
   }

+   MFEM_VERIFY(!SMQ, "SymmetricMatrixCoefficient not supported here");
+
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), invdfdx(dim, spaceDim);
   DenseMatrix M(MQ ? spaceDim : 0);
@@ -1072,16 +1081,13 @@ void DiffusionIntegrator::ComputeElementFlux
   vecdxt.SetSize(spaceDim);
   pointflux.SetSize(MQ || VQ ? spaceDim : 0);

-   if (!ir)
-   {
-      ir = &fluxelem.GetNodes();
-   }
-   fnd = ir->GetNPoints();
+   const IntegrationRule &ir = fluxelem.GetNodes();
+   fnd = ir.GetNPoints();
   flux.SetSize( fnd * spaceDim );

   for (int i = 0; i < fnd; i++)
   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
+      const IntegrationPoint &ip = ir.IntPoint(i);
      el.CalcDShape(ip, dshape);
      dshape.MultTranspose(u, vec);

@@ -1148,6 +1154,8 @@ double DiffusionIntegrator::ComputeFluxEnergy
   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif

+   MFEM_VERIFY(!SMQ, "SymmetricMatrixCoefficient not supported here");
+
   shape.SetSize(nd);
   pointflux.SetSize(spaceDim);
   if (d_energy) { vec.SetSize(spaceDim); }
@@ -1672,8 +1680,7 @@ void VectorFEDivergenceIntegrator::AssembleElementMatrix2(
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
      trial_fe.CalcDivShape(ip, divshape);
-      Trans.SetIntPoint(&ip);
-      test_fe.CalcPhysShape(Trans, shape);
+      test_fe.CalcShape(ip, shape);
      double w = ip.weight;
      if (Q)
      {
@@ -2006,14 +2013,12 @@ void CurlCurlIntegrator::AssembleElementMatrix
 void CurlCurlIntegrator
 ::ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans,
                     Vector &u, const FiniteElement &fluxelem, Vector &flux,
-                     bool with_coef, const IntegrationRule *ir)
+                     bool with_coef)
 {
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix projcurl;
 #endif

-   MFEM_VERIFY(ir == NULL, "Integration rule (ir) must be NULL")
-
   fluxelem.ProjectCurl(el, Trans, projcurl);

   flux.SetSize(projcurl.Height());
@@ -2586,6 +2591,55 @@ void DivDivIntegrator::AssembleElementMatrix(
   }
 }

+void DivDivIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   double c;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector divshape(tr_nd);
+   Vector te_divshape(te_nd);
+#else
+   divshape.SetSize(tr_nd);
+   te_divshape.SetSize(te_nd);
+#endif
+   elmat.SetSize(te_nd,tr_nd);
+
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(test_fe.GetOrder(),
+                          trial_fe.GetOrder()) - 2; // <--- OK for RTk
+      ir = &IntRules.Get(test_fe.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      trial_fe.CalcDivShape(ip,divshape);
+      test_fe.CalcDivShape(ip,te_divshape);
+
+      Trans.SetIntPoint (&ip);
+      c = ip.weight / Trans.Weight();
+
+      if (Q)
+      {
+         c *= Q -> Eval (Trans, ip);
+      }
+
+      te_divshape *= c;
+      AddMultVWt(te_divshape, divshape, elmat);
+   }
+}

 void VectorDiffusionIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -2845,7 +2899,7 @@ void ElasticityIntegrator::AssembleElementMatrix(
 void ElasticityIntegrator::ComputeElementFlux(
   const mfem::FiniteElement &el, ElementTransformation &Trans,
   Vector &u, const mfem::FiniteElement &fluxelem, Vector &flux,
-   bool with_coef, const IntegrationRule *ir)
+   bool with_coef)
 {
   const int dof = el.GetDof();
   const int dim = el.GetDim();
@@ -2868,17 +2922,14 @@ void ElasticityIntegrator::ComputeElementFlux(
   DenseMatrix gh(gh_data, dim, dim);
   DenseMatrix grad(grad_data, dim, dim);

-   if (!ir)
-   {
-      ir = &fluxelem.GetNodes();
-   }
-   const int fnd = ir->GetNPoints();
+   const IntegrationRule &ir = fluxelem.GetNodes();
+   const int fnd = ir.GetNPoints();
   flux.SetSize(fnd * tdim);

   DenseMatrix loc_data_mat(u.GetData(), dof, dim);
   for (int i = 0; i < fnd; i++)
   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
+      const IntegrationPoint &ip = ir.IntPoint(i);
      el.CalcDShape(ip, dshape);
      MultAtB(loc_data_mat, dshape, gh);

@@ -3780,7 +3831,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
      for (i = 0; i < ndof1; i++)
         for (j = 0; j < face_ndof; j++)
         {
-            elmat(i, j) -= shape1_n(i) * face_shape(j);
+            elmat(i, j) += shape1_n(i) * face_shape(j);
         }
      if (ndof2)
      {
@@ -3788,12 +3839,135 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
         for (i = 0; i < ndof2; i++)
            for (j = 0; j < face_ndof; j++)
            {
-               elmat(ndof1+i, j) += shape2_n(i) * face_shape(j);
+               elmat(ndof1+i, j) -= shape2_n(i) * face_shape(j);
            }
      }
   }
 }

+void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                              const FiniteElement &trial_face_fe,
+                                              const FiniteElement &test_fe,
+                                              FaceElementTransformations & Trans,
+                                              DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof;
+   int order;
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+
+   face_shape.SetSize(face_ndof);
+   shape.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
+      {
+         order += Trans.OrderW();
+      }
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+
+      // Set the integration point in the face and the neighboring elements
+      Trans.SetAllIntPoints(&ip);
+      // Trace finite element shape function
+      trial_face_fe.CalcPhysShape(Trans,face_shape);
+
+      // Finite element shape function
+      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcPhysShape(*eltrans, shape);
+
+      face_shape *= Trans.Weight()*ip.weight;
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += scale * shape(i) * face_shape(j);
+         }
+      }
+   }
+}
+
+void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
+                                                    const FiniteElement &trial_face_fe,
+                                                    const FiniteElement &test_fe,
+                                                    FaceElementTransformations &Trans,
+                                                    DenseMatrix &elmat)
+{
+   int i, j, face_ndof, ndof, dim;
+   int order;
+
+   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE, "");
+
+   face_ndof = trial_face_fe.GetDof();
+   ndof = test_fe.GetDof();
+   dim = test_fe.GetDim();
+
+   face_shape.SetSize(face_ndof);
+   normal.SetSize(dim);
+   shape.SetSize(ndof,dim);
+   shape_n.SetSize(ndof);
+
+   elmat.SetSize(ndof, face_ndof);
+   elmat = 0.0;
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      order = test_fe.GetOrder();
+      order += trial_face_fe.GetOrder();
+      ir = &IntRules.Get(Trans.GetGeometryType(), order);
+   }
+
+   int iel = Trans.Elem1->ElementNo;
+   if (iel != elem)
+   {
+      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
+   }
+
+   double scale = 1.0;
+   if (iel != elem) { scale = -1.; }
+
+   for (int p = 0; p < ir->GetNPoints(); p++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(p);
+      Trans.SetAllIntPoints(&ip);
+      trial_face_fe.CalcPhysShape(Trans, face_shape);
+      CalcOrtho(Trans.Jacobian(),normal);
+      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
+      test_fe.CalcVShape(*etrans, shape);
+      shape.Mult(normal, shape_n);
+      face_shape *= ip.weight;
+
+      for (i = 0; i < ndof; i++)
+      {
+         for (j = 0; j < face_ndof; j++)
+         {
+            elmat(i, j) += scale * shape_n(i) * face_shape(j);
+         }
+      }
+   }
+}
+

 void NormalInterpolator::AssembleElementMatrix2(
   const FiniteElement &dom_fe, const FiniteElement &ran_fe,
@@ -15,7 +15,6 @@
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
 #include "fespace.hpp"
-#include "ceed/interface/util.hpp"

 namespace mfem
 {
@@ -151,6 +150,12 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

+   virtual void AssembleTraceFaceMatrix(int elem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+
   /** Abstract method used for assembling TraceFaceIntegrators in a
       MixedBilinearForm. */
   virtual void AssembleFaceMatrix(const FiniteElement &trial_face_fe,
@@ -214,18 +219,12 @@ public:
                             of the method may choose not to scale the "flux"
                             function by any coefficients describing the
                             integrator.
-       @param[in] ir  If passed (the default value is NULL), the implementation
-                        of the method will ignore the integration rule provided
-                        by the @a fluxelem parameter and, instead, compute the
-                        discrete flux at the points specified by the integration
-                        rule @a ir.
    */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u,
                                   const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true,
-                                   const IntegrationRule *ir = NULL) { }
+                                   Vector &flux, bool with_coef = true) { }

   /** @brief Virtual method required for Zienkiewicz-Zhu type error estimators.

@@ -1906,7 +1905,6 @@ protected:
                           const FiniteElementSpace &test_fes);

   virtual void AddMultPA(const Vector&, Vector&) const;
-   virtual void AddMultTransposePA(const Vector&, Vector&) const;

 private:
   // PA extension
@@ -1965,7 +1963,6 @@ protected:
                           const FiniteElementSpace &test_fes);

   virtual void AddMultPA(const Vector&, Vector&) const;
-   virtual void AddMultTransposePA(const Vector&, Vector&) const;

 private:
   // PA extension
@@ -2077,6 +2074,7 @@ protected:
   Coefficient *Q;
   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;
+   SymmetricMatrixCoefficient *SMQ;

 private:
   Vector vec, vecdxt, pointflux, shape;
@@ -2098,24 +2096,30 @@ public:
   /// Construct a diffusion integrator with coefficient Q = 1
   DiffusionIntegrator(const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL) { }
+        Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a scalar coefficient q
   DiffusionIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL) { }
+        Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a vector coefficient q
   DiffusionIntegrator(VectorCoefficient &q,
                       const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL) { }
+        Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a matrix coefficient q
   DiffusionIntegrator(MatrixCoefficient &q,
                       const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL) { }
+        Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a diffusion integrator with a symmetric matrix coefficient q
+   DiffusionIntegrator(SymmetricMatrixCoefficient &q,
+                       const IntegrationRule *ir = nullptr)
+      : BilinearFormIntegrator(ir),
+        Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL) { }

   /** Given a particular Finite Element computes the element stiffness matrix
       elmat. */
@@ -2137,8 +2141,7 @@ public:
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true,
-                                   const IntegrationRule *ir = NULL);
+                                   Vector &flux, bool with_coef = true);

   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
@@ -2167,8 +2170,6 @@ public:
                                         const FiniteElement &test_fe);

   bool SupportsCeed() const { return DeviceCanUseCeed(); }
-
-   Coefficient *GetCoefficient() const { return Q; }
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2228,8 +2229,6 @@ public:
                                         ElementTransformation &Trans);

   bool SupportsCeed() const { return DeviceCanUseCeed(); }
-
-   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Mass integrator (u, v) restricted to the boundary of a domain */
@@ -2398,9 +2397,8 @@ public:
    scalar function given by FiniteElement through standard transformation.
    Here, u is the trial function and p is the test function.

-    Note: if the test space does not have map type INTEGRAL, then the element
-    matrix returned by AssembleElementMatrix2 will not depend on the
-    ElementTransformation Trans. */
+    Note: the element matrix returned by AssembleElementMatrix2 does NOT depend
+    on the ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2531,6 +2529,7 @@ protected:
   Coefficient *Q;
   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;
+   SymmetricMatrixCoefficient *SMQ;

   // PA extension
   Vector pa_data;
@@ -2541,15 +2540,18 @@ protected:
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; SMQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL) { }
+      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL), SMQ(NULL) { }
   CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
                      const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL) { }
+      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL), SMQ(NULL) { }
   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq) { }
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq), SMQ(NULL) { }
+   CurlCurlIntegrator(SymmetricMatrixCoefficient &smq,
+                      const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(NULL), SMQ(&smq) { }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2560,8 +2562,7 @@ public:
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef,
-                                   const IntegrationRule *ir = NULL);
+                                   Vector &flux, bool with_coef);

   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
@@ -2571,8 +2572,6 @@ public:
   virtual void AssemblePA(const FiniteElementSpace &fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
-
-   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Integrator for (curl u, curl v) for FE spaces defined by 'dim' copies of a
@@ -2609,8 +2608,9 @@ public:
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
-   { Q = q; DQ = dq; MQ = mq; }
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq,
+             SymmetricMatrixCoefficient *smq)
+   { Q = q; DQ = dq; MQ = mq; SMQ = smq; }

 #ifndef MFEM_THREAD_SAFE
   Vector shape;
@@ -2625,6 +2625,7 @@ protected:
   Coefficient *Q;
   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;
+   SymmetricMatrixCoefficient *SMQ;

   // PA extension
   Vector pa_data;
@@ -2637,13 +2638,15 @@ protected:
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
-   VectorFEMassIntegrator(Coefficient *q_) { Init(q_, NULL, NULL); }
-   VectorFEMassIntegrator(Coefficient &q) { Init(&q, NULL, NULL); }
-   VectorFEMassIntegrator(DiagonalMatrixCoefficient *dq_) { Init(NULL, dq_, NULL); }
-   VectorFEMassIntegrator(DiagonalMatrixCoefficient &dq) { Init(NULL, &dq, NULL); }
-   VectorFEMassIntegrator(MatrixCoefficient *mq_) { Init(NULL, NULL, mq_); }
-   VectorFEMassIntegrator(MatrixCoefficient &mq) { Init(NULL, NULL, &mq); }
+   VectorFEMassIntegrator() { Init(NULL, NULL, NULL, NULL); }
+   VectorFEMassIntegrator(Coefficient *q_) { Init(q_, NULL, NULL, NULL); }
+   VectorFEMassIntegrator(Coefficient &q) { Init(&q, NULL, NULL, NULL); }
+   VectorFEMassIntegrator(DiagonalMatrixCoefficient *dq_) { Init(NULL, dq_, NULL, NULL); }
+   VectorFEMassIntegrator(DiagonalMatrixCoefficient &dq) { Init(NULL, &dq, NULL, NULL); }
+   VectorFEMassIntegrator(MatrixCoefficient *mq_) { Init(NULL, NULL, mq_, NULL); }
+   VectorFEMassIntegrator(MatrixCoefficient &mq) { Init(NULL, NULL, &mq, NULL); }
+   VectorFEMassIntegrator(SymmetricMatrixCoefficient &smq) { Init(NULL, NULL, NULL, &smq); }
+   VectorFEMassIntegrator(SymmetricMatrixCoefficient *smq) { Init(NULL, NULL, NULL, smq); }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -2658,10 +2661,7 @@ public:
   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                           const FiniteElementSpace &test_fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
-   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
-
-   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Integrator for (Q div u, p) where u=(v1,...,vn) and all vi are in the same
@@ -2725,7 +2725,7 @@ protected:

 private:
 #ifndef MFEM_THREAD_SAFE
-   Vector divshape;
+   Vector divshape, te_divshape;
 #endif

   // PA extension
@@ -2742,7 +2742,10 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
-   const Coefficient *GetCoefficient() const { return Q; }
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
 };

 /** Integrator for
@@ -2882,14 +2885,12 @@ public:
       of the stress components is: s_xx, s_yy, s_xy. In 3D, it is: s_xx, s_yy,
       s_zz, s_xy, s_xz, s_yz. In other words, @a flux is the local vector for
       a FE space with dim*(dim+1)/2 vector components, based on the finite
-       element @a fluxelem. The integration rule is taken from @a fluxelem.
-       @a ir exists to specific an alternative integration rule. */
+       element @a fluxelem. */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u,
                                   const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true,
-                                   const IntegrationRule *ir = NULL);
+                                   Vector &flux, bool with_coef = true);

   /** Compute the element energy (integral of the strain energy density)
       corresponding to the stress represented by @a flux which is a vector of
@@ -3239,6 +3240,19 @@ public:
                                   DenseMatrix &elmat);
 };

+class TraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, shape;
+public:
+   TraceIntegrator() { }
+   void AssembleTraceFaceMatrix(int elem,
+                                const FiniteElement &trial_face_fe,
+                                const FiniteElement &test_fe,
+                                FaceElementTransformations &Trans,
+                                DenseMatrix &elmat);
+};
+
 /** Integrator for the form: < v, [w.n] > over all faces (the interface) where
    the trial variable v is defined on the interface and the test variable w is
    in an H(div)-conforming space. */
@@ -3258,6 +3272,23 @@ public:
                                   DenseMatrix &elmat);
 };

+class NormalTraceIntegrator : public BilinearFormIntegrator
+{
+private:
+   Vector face_shape, normal, shape_n;
+   DenseMatrix shape;
+
+public:
+   NormalTraceIntegrator() { }
+   virtual void AssembleTraceFaceMatrix(int ielem,
+                                        const FiniteElement &trial_face_fe,
+                                        const FiniteElement &test_fe,
+                                        FaceElementTransformations &Trans,
+                                        DenseMatrix &elmat);
+};
+
+
+
 /** Abstract class to serve as a base for local interpolators to be used in the
    DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/convection/convection.hpp"
+#include "ceed/convection.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/convection/convection.hpp"
+#include "ceed/convection.hpp"
 #include "quadinterpolator.hpp"

 namespace mfem
@@ -43,10 +43,9 @@ static void PADGTraceSetup2D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);

-   MFEM_FORALL(tid, Q1D*NF,
+   MFEM_FORALL(f, NF, // can be optimized with Q1D thread for NF blocks
   {
-      const int f = tid / Q1D;
-      const int q = tid % Q1D;
+      for (int q = 0; q < Q1D; ++q)
      {
         const double r = const_r ? R(0,0) : R(q,f);
         const double v0 = const_v ? V(0,0,0) : V(0,q,f);
@@ -86,12 +85,11 @@ static void PADGTraceSetup3D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);

-   MFEM_FORALL(tid, Q1D*Q1D*NF,
+   MFEM_FORALL(f, NF, // can be optimized with Q1D*Q1D threads for NF blocks
   {
-      int f = tid / (Q1D * Q1D);
-      int q2 = (tid / Q1D) % Q1D;
-      int q1 = tid % Q1D;
+      for (int q1 = 0; q1 < Q1D; ++q1)
      {
+         for (int q2 = 0; q2 < Q1D; ++q2)
         {
            const double r = const_r ? R(0,0,0) : R(q1,q2,f);
            const double v0 = const_v ? V(0,0,0,0) : V(0,q1,q2,f);
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
+#include "ceed/diffusion.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
+#include "ceed/diffusion.hpp"

 using namespace std;

@@ -362,7 +362,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      MFEM_VERIFY(!VQ && !MQ,
+      MFEM_VERIFY(!VQ && !MQ && !SMQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
@@ -381,33 +381,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   int coeffDim = 1;
   Vector coeff;
   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
-   {
-      MFEM_VERIFY(SMQ->GetSize() == dim, "");
-      coeffDim = symmDims;
-      coeff.SetSize(symmDims * nq * ne);
-
-      DenseSymmetricMatrix sym_mat;
-      sym_mat.SetSize(dim);
-
-      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
-            int cnt = 0;
-            for (int i=0; i<dim; ++i)
-               for (int j=i; j<dim; ++j, ++cnt)
-               {
-                  C(cnt, p, e) = sym_mat(i,j);
-               }
-         }
-      }
-   }
-   else if (MQ)
+   if (MQ)
   {
      symmetric = false;
      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
@@ -416,8 +390,8 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)

      coeff.SetSize(MQfullDim * nq * ne);

-      DenseMatrix mat;
-      mat.SetSize(dim);
+      DenseMatrix GM;
+      GM.SetSize(dim);

      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
      for (int e=0; e<ne; ++e)
@@ -425,11 +399,37 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            MQ->Eval(mat, *tr, ir->IntPoint(p));
+            MQ->Eval(GM, *tr, ir->IntPoint(p));
            for (int i=0; i<dim; ++i)
               for (int j=0; j<dim; ++j)
               {
-                  C(j+(i*dim), p, e) = mat(i,j);
+                  C(j+(i*dim), p, e) = GM(i,j);
+               }
+         }
+      }
+   }
+   else if (SMQ)
+   {
+      MFEM_VERIFY(SMQ->GetSize() == dim, "");
+      coeffDim = symmDims;
+      coeff.SetSize(symmDims * nq * ne);
+
+      DenseSymmetricMatrix SM;
+      SM.SetSize(dim);
+
+      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            SMQ->Eval(SM, *tr, ir->IntPoint(p));
+            int cnt = 0;
+            for (int i=0; i<dim; ++i)
+               for (int j=i; j<dim; ++j, ++cnt)
+               {
+                  C(cnt, p, e) = SM(i,j);
               }
         }
      }
@@ -978,14 +978,12 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
-
   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+   const int MQdim = MQ ? MQfullDim : MQsymmDim;
+   const int coeffDim = (MQ || SMQ) ? MQdim : (DQ ? DQ->GetVDim() : 1);

-   symmetric = (SMQ || MQ == NULL);
+   symmetric = (MQ == NULL);

   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
@@ -994,7 +992,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   Vector coeff(coeffDim * ne * nq);
   coeff = 1.0;
   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
+   if (Q || DQ || MQ || SMQ)
   {
      Vector DM(DQ ? coeffDim : 0);
      DenseMatrix GM;
@@ -1004,24 +1002,35 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
      {
         MFEM_VERIFY(coeffDim == dimc, "");
      }
-      if (SMQ)
-      {
-         SM.SetSize(dimc);
-         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
-      }
-      else if (MQ)
+      if (MQ)
      {
         GM.SetSize(dimc);
         MFEM_VERIFY(coeffDim == MQdim, "");
         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
      }
+      if (SMQ)
+      {
+         SM.SetSize(dimc);
+         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
+      }

      for (int e=0; e<ne; ++e)
      {
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            if (SMQ)
+            if (MQ)
+            {
+               MQ->Eval(GM, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dimc; ++i)
+                  for (int j=0; j<dimc; ++j)
+                  {
+                     coeffh(j+(i*dimc), p, e) = GM(i,j);
+                  }
+
+            }
+            else if (SMQ)
            {
               SMQ->Eval(SM, *tr, ir->IntPoint(p));

@@ -1032,17 +1041,6 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
                     coeffh(cnt, p, e) = SM(i,j);
                  }

-            }
-            else if (MQ)
-            {
-               MQ->Eval(GM, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dimc; ++i)
-                  for (int j=0; j<dimc; ++j)
-                  {
-                     coeffh(j+(i*dimc), p, e) = GM(i,j);
-                  }
-
            }
            else if (DQ)
            {
@@ -3635,8 +3633,8 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      }
      else
      {
-         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(),
-                            geom->J, coeff, pa_data);
+         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(), geom->J,
+                            coeff, pa_data);
      }
   }
   else if (testType == mfem::FiniteElement::DIV &&
@@ -4681,362 +4679,6 @@ static void PAHcurlHdivApply3D(const int D1D,
   }); // end of element loop
 }

-// Apply to x corresponding to DOF's in H(div) (test), integrated against the
-// curl of H(curl) trial functions corresponding to y.
-template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
-static void PAHcurlHdivApply3DTranspose(const int D1D,
-                                        const int D1Dtest,
-                                        const int Q1D,
-                                        const int NE,
-                                        const Array<double> &bo,
-                                        const Array<double> &bc,
-                                        const Array<double> &bot,
-                                        const Array<double> &bct,
-                                        const Array<double> &gct,
-                                        const Vector &pa_data,
-                                        const Vector &x,
-                                        Vector &y)
-{
-   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
-   // Using Piola transformations (\nabla\times u) F = 1/det(dF) dF \hat{\nabla}\times\hat{u}
-   // for u in H(curl) and w = (1 / det (dF)) dF \hat{w} for w in H(div), we get
-   // (\nabla\times u) \cdot w = 1/det(dF)^2 \hat{\nabla}\times\hat{u}^T dF^T dF \hat{w}
-   // If c = 0, \hat{\nabla}\times\hat{u} reduces to [0, (u_0)_{x_2}, -(u_0)_{x_1}]
-   // If c = 1, \hat{\nabla}\times\hat{u} reduces to [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
-   // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
-
-   constexpr static int VDIM = 3;
-
-   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
-   auto Bc = Reshape(bc.Read(), Q1D, D1D);
-   auto Bot = Reshape(bot.Read(), D1Dtest-1, Q1D);
-   auto Bct = Reshape(bct.Read(), D1Dtest, Q1D);
-   auto Gct = Reshape(gct.Read(), D1D, Q1D);
-   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, 6, NE);
-   auto X = Reshape(x.Read(), 3*(D1Dtest-1)*(D1Dtest-1)*D1D, NE);
-   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
-
-   MFEM_FORALL(e, NE,
-   {
-      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
-
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               for (int c = 0; c < VDIM; ++c)
-               {
-                  mass[qz][qy][qx][c] = 0.0;
-               }
-            }
-         }
-      }
-
-      int osc = 0;
-
-      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
-      {
-         const int D1Dz = (c == 2) ? D1D : D1D - 1;
-         const int D1Dy = (c == 1) ? D1D : D1D - 1;
-         const int D1Dx = (c == 0) ? D1D : D1D - 1;
-
-         for (int dz = 0; dz < D1Dz; ++dz)
-         {
-            double massXY[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  massXY[qy][qx] = 0.0;
-               }
-            }
-
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               double massX[HDIV_MAX_Q1D];
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  massX[qx] = 0.0;
-               }
-
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     massX[qx] += t * ((c == 0) ? Bc(qx,dx) : Bo(qx,dx));
-                  }
-               }
-
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const double wy = (c == 1) ? Bc(qy,dy) : Bo(qy,dy);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const double wx = massX[qx];
-                     massXY[qy][qx] += wx * wy;
-                  }
-               }
-            }
-
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const double wz = (c == 2) ? Bc(qz,dz) : Bo(qz,dz);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
-                  }
-               }
-            }
-         }
-
-         osc += D1Dx * D1Dy * D1Dz;
-      }  // loop (c) over components
-
-      // Apply D operator.
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double O11 = op(qx,qy,qz,0,e);
-               const double O12 = op(qx,qy,qz,1,e);
-               const double O13 = op(qx,qy,qz,2,e);
-               const double O22 = op(qx,qy,qz,3,e);
-               const double O23 = op(qx,qy,qz,4,e);
-               const double O33 = op(qx,qy,qz,5,e);
-               const double massX = mass[qz][qy][qx][0];
-               const double massY = mass[qz][qy][qx][1];
-               const double massZ = mass[qz][qy][qx][2];
-               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
-               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
-               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
-            }
-         }
-      }
-
-      // x component
-      osc = 0;
-      {
-         const int D1Dz = D1D;
-         const int D1Dy = D1D;
-         const int D1Dx = D1D - 1;
-
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            double gradXY12[MAX_D1D][MAX_D1D];
-            double gradXY21[MAX_D1D][MAX_D1D];
-
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  gradXY12[dy][dx] = 0.0;
-                  gradXY21[dy][dx] = 0.0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               double massX[MAX_D1D][2];
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  for (int n = 0; n < 2; ++n)
-                  {
-                     massX[dx][n] = 0.0;
-                  }
-               }
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     const double wx = Bot(dx,qx);
-
-                     massX[dx][0] += wx * mass[qz][qy][qx][1];
-                     massX[dx][1] += wx * mass[qz][qy][qx][2];
-                  }
-               }
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  const double wy = Bct(dy,qy);
-                  const double wDy = Gct(dy,qy);
-
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     gradXY21[dy][dx] += massX[dx][0] * wy;
-                     gradXY12[dy][dx] += massX[dx][1] * wDy;
-                  }
-               }
-            }
-
-            for (int dz = 0; dz < D1Dz; ++dz)
-            {
-               const double wz = Bct(dz,qz);
-               const double wDz = Gct(dz,qz);
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     // \hat{\nabla}\times\hat{u} is [0, (u_0)_{x_2}, -(u_0)_{x_1}]
-                     // (u_0)_{x_2} * (op * curl)_1 - (u_0)_{x_1} * (op * curl)_2
-                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
-                       e) += (gradXY21[dy][dx] * wDz) - (gradXY12[dy][dx] * wz);
-                  }
-               }
-            }
-         }  // loop qz
-
-         osc += D1Dx * D1Dy * D1Dz;
-      }
-
-      // y component
-      {
-         const int D1Dz = D1D;
-         const int D1Dy = D1D - 1;
-         const int D1Dx = D1D;
-
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            double gradXY02[MAX_D1D][MAX_D1D];
-            double gradXY20[MAX_D1D][MAX_D1D];
-
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  gradXY02[dy][dx] = 0.0;
-                  gradXY20[dy][dx] = 0.0;
-               }
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               double massY[MAX_D1D][2];
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  massY[dy][0] = 0.0;
-                  massY[dy][1] = 0.0;
-               }
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int dy = 0; dy < D1Dy; ++dy)
-                  {
-                     const double wy = Bot(dy,qy);
-
-                     massY[dy][0] += wy * mass[qz][qy][qx][2];
-                     massY[dy][1] += wy * mass[qz][qy][qx][0];
-                  }
-               }
-               for (int dx = 0; dx < D1Dx; ++dx)
-               {
-                  const double wx = Bct(dx,qx);
-                  const double wDx = Gct(dx,qx);
-
-                  for (int dy = 0; dy < D1Dy; ++dy)
-                  {
-                     gradXY02[dy][dx] += massY[dy][0] * wDx;
-                     gradXY20[dy][dx] += massY[dy][1] * wx;
-                  }
-               }
-            }
-
-            for (int dz = 0; dz < D1Dz; ++dz)
-            {
-               const double wz = Bct(dz,qz);
-               const double wDz = Gct(dz,qz);
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  for (int dx = 0; dx < D1Dx; ++dx)
-                  {
-                     // \hat{\nabla}\times\hat{u} is [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
-                     // -(u_1)_{x_2} * (op * curl)_0 + (u_1)_{x_0} * (op * curl)_2
-                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
-                       e) += (-gradXY20[dy][dx] * wDz) + (gradXY02[dy][dx] * wz);
-                  }
-               }
-            }
-         }  // loop qz
-
-         osc += D1Dx * D1Dy * D1Dz;
-      }
-
-      // z component
-      {
-         const int D1Dz = D1D - 1;
-         const int D1Dy = D1D;
-         const int D1Dx = D1D;
-
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            double gradYZ01[MAX_D1D][MAX_D1D];
-            double gradYZ10[MAX_D1D][MAX_D1D];
-
-            for (int dy = 0; dy < D1Dy; ++dy)
-            {
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  gradYZ01[dz][dy] = 0.0;
-                  gradYZ10[dz][dy] = 0.0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               double massZ[MAX_D1D][2];
-               for (int dz = 0; dz < D1Dz; ++dz)
-               {
-                  for (int n = 0; n < 2; ++n)
-                  {
-                     massZ[dz][n] = 0.0;
-                  }
-               }
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  for (int dz = 0; dz < D1Dz; ++dz)
-                  {
-                     const double wz = Bot(dz,qz);
-
-                     massZ[dz][0] += wz * mass[qz][qy][qx][0];
-                     massZ[dz][1] += wz * mass[qz][qy][qx][1];
-                  }
-               }
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  const double wy = Bct(dy,qy);
-                  const double wDy = Gct(dy,qy);
-
-                  for (int dz = 0; dz < D1Dz; ++dz)
-                  {
-                     gradYZ01[dz][dy] += wy * massZ[dz][1];
-                     gradYZ10[dz][dy] += wDy * massZ[dz][0];
-                  }
-               }
-            }
-
-            for (int dx = 0; dx < D1Dx; ++dx)
-            {
-               const double wx = Bct(dx,qx);
-               const double wDx = Gct(dx,qx);
-
-               for (int dy = 0; dy < D1Dy; ++dy)
-               {
-                  for (int dz = 0; dz < D1Dz; ++dz)
-                  {
-                     // \hat{\nabla}\times\hat{u} is [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
-                     // (u_2)_{x_1} * (op * curl)_0 - (u_2)_{x_0} * (op * curl)_1
-                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
-                       e) += (gradYZ10[dz][dy] * wx) - (gradYZ01[dz][dy] * wDx);
-                  }
-               }
-            }
-         }  // loop qx
-      }
-   }); // end of element loop
-}
-
 void MixedVectorCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
   if (testType == mfem::FiniteElement::CURL &&
@@ -5081,20 +4723,6 @@ void MixedVectorCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

-void MixedVectorCurlIntegrator::AddMultTransposePA(const Vector &x,
-                                                   Vector &y) const
-{
-   if (testType == mfem::FiniteElement::DIV &&
-       trialType == mfem::FiniteElement::CURL && dim == 3)
-      PAHcurlHdivApply3DTranspose(dofs1D, dofs1Dtest, quad1D, ne, mapsO->B,
-                                  mapsC->B, mapsOtest->Bt, mapsCtest->Bt,
-                                  mapsC->Gt, pa_data, x, y);
-   else
-   {
-      MFEM_ABORT("Unsupported dimension or space!");
-   }
-}
-
 void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
                                               &trial_fes,
                                               const FiniteElementSpace &test_fes)
@@ -5133,16 +4761,8 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   testType = test_el->GetDerivType();
-   trialType = trial_el->GetDerivType();
-
-   const bool curlSpaces = (testType == mfem::FiniteElement::CURL &&
-                            trialType == mfem::FiniteElement::CURL);
-
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-
   coeffDim = DQ ? 3 : 1;
-   const int ndata = curlSpaces ? (DQ ? 9 : 1) : symmDims;
+   const int ndata = DQ ? 9 : 1;

   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

@@ -5179,6 +4799,9 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
      }
   }

+   testType = test_el->GetDerivType();
+   trialType = trial_el->GetDerivType();
+
   if (trialType == mfem::FiniteElement::CURL && dim == 3)
   {
      if (coeffDim == 1)
@@ -5187,16 +4810,10 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
      }
      else
      {
-         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(),
-                            geom->J, coeff, pa_data);
+         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(), geom->J,
+                            coeff, pa_data);
      }
   }
-   else if (trialType == mfem::FiniteElement::DIV && dim == 3 &&
-            test_el->GetOrder() == trial_el->GetOrder())
-   {
-      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
-                        pa_data);
-   }
   else
   {
      MFEM_ABORT("Unknown kernel.");
@@ -5846,29 +5463,6 @@ void MixedVectorWeakCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
         PAHcurlL2Apply3DTranspose(dofs1D, quad1D, ndata, ne, mapsO->B,
                                   mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->Gt, pa_data, x, y);
   }
-   else if (testType == mfem::FiniteElement::CURL &&
-            trialType == mfem::FiniteElement::DIV && dim == 3)
-   {
-      PAHcurlHdivApply3DTranspose(dofs1D, dofs1D, quad1D, ne, mapsO->B,
-                                  mapsC->B, mapsO->Bt, mapsC->Bt,
-                                  mapsC->Gt, pa_data, x, y);
-   }
-   else
-   {
-      MFEM_ABORT("Unsupported dimension or space!");
-   }
-}
-
-void MixedVectorWeakCurlIntegrator::AddMultTransposePA(const Vector &x,
-                                                       Vector &y) const
-{
-   if (testType == mfem::FiniteElement::CURL &&
-       trialType == mfem::FiniteElement::DIV && dim == 3)
-   {
-      PAHcurlHdivApply3D(dofs1D, dofs1D, quad1D, ne, mapsO->B,
-                         mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G,
-                         pa_data, x, y);
-   }
   else
   {
      MFEM_ABORT("Unsupported dimension or space!");
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/mass/mass.hpp"
+#include "ceed/mass.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/mass/mass.hpp"
+#include "ceed/mass.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
+#include "ceed/diffusion.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/diffusion/diffusion.hpp"
+#include "ceed/diffusion.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/mass/mass.hpp"
+#include "ceed/mass.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/integrators/mass/mass.hpp"
+#include "ceed/mass.hpp"

 using namespace std;

@@ -372,8 +372,8 @@ void PAHcurlHdivSetup2D(const int Q1D,
               const double R21 = D2*J21;
               const double R22 = D2*J22;
               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               y(i21,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2 (transpose)
-               y(i12,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1 (transpose)
+               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
+               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
            }
         }
@@ -389,7 +389,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
                            const int NE,
                            const bool scalarCoeff,
                            const bool trialHcurl,
-                            const bool transpose,
                            const Array<double> &Bo_,
                            const Array<double> &Bc_,
                            const Array<double> &Bot_,
@@ -414,13 +413,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
   auto y = Reshape(y_.ReadWrite(), 3*(D1Dtest-1)*D1Dtest*
                    (trialHcurl ? D1Dtest-1 : D1Dtest), NE);

-   const int i12 = transpose ? 3 : 1;
-   const int i13 = transpose ? 6 : 2;
-   const int i21 = transpose ? 1 : 3;
-   const int i23 = transpose ? 7 : 5;
-   const int i31 = transpose ? 2 : 6;
-   const int i32 = transpose ? 5 : 7;
-
   MFEM_FORALL(e, NE,
   {
      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
@@ -515,13 +507,13 @@ void PAHcurlHdivMassApply3D(const int D1D,
            for (int qx = 0; qx < Q1D; ++qx)
            {
               const double O11 = op(0,qx,qy,qz,e);
-               const double O12 = scalarCoeff ? 0.0 : op(i12,qx,qy,qz,e);
-               const double O13 = scalarCoeff ? 0.0 : op(i13,qx,qy,qz,e);
-               const double O21 = scalarCoeff ? 0.0 : op(i21,qx,qy,qz,e);
+               const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,qz,e);
+               const double O13 = scalarCoeff ? 0.0 : op(2,qx,qy,qz,e);
+               const double O21 = scalarCoeff ? 0.0 : op(3,qx,qy,qz,e);
               const double O22 = scalarCoeff ? O11 : op(4,qx,qy,qz,e);
-               const double O23 = scalarCoeff ? 0.0 : op(i23,qx,qy,qz,e);
-               const double O31 = scalarCoeff ? 0.0 : op(i31,qx,qy,qz,e);
-               const double O32 = scalarCoeff ? 0.0 : op(i32,qx,qy,qz,e);
+               const double O23 = scalarCoeff ? 0.0 : op(5,qx,qy,qz,e);
+               const double O31 = scalarCoeff ? 0.0 : op(6,qx,qy,qz,e);
+               const double O32 = scalarCoeff ? 0.0 : op(7,qx,qy,qz,e);
               const double O33 = scalarCoeff ? O11 : op(8,qx,qy,qz,e);
               const double massX = mass[qz][qy][qx][0];
               const double massY = mass[qz][qy][qx][1];
@@ -609,7 +601,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
                            const int NE,
                            const bool scalarCoeff,
                            const bool trialHcurl,
-                            const bool transpose,
                            const Array<double> &Bo_,
                            const Array<double> &Bc_,
                            const Array<double> &Bot_,
@@ -633,9 +624,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
   auto x = Reshape(x_.Read(), 2*(D1D-1)*D1D, NE);
   auto y = Reshape(y_.ReadWrite(), 2*(D1Dtest-1)*D1Dtest, NE);

-   const int i12 = transpose ? 2 : 1;
-   const int i21 = transpose ? 1 : 2;
-
   MFEM_FORALL(e, NE,
   {
      double mass[MAX_Q1D][MAX_Q1D][VDIM];
@@ -697,8 +685,8 @@ void PAHcurlHdivMassApply2D(const int D1D,
         for (int qx = 0; qx < Q1D; ++qx)
         {
            const double O11 = op(0,qx,qy,e);
-            const double O12 = scalarCoeff ? 0.0 : op(i12,qx,qy,e);
-            const double O21 = scalarCoeff ? 0.0 : op(i21,qx,qy,e);
+            const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,e);
+            const double O21 = scalarCoeff ? 0.0 : op(2,qx,qy,e);
            const double O22 = scalarCoeff ? O11 : op(3,qx,qy,e);
            const double massX = mass[qy][qx][0];
            const double massY = mass[qy][qx][1];
@@ -797,14 +785,12 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   trial_fetype = trial_el->GetDerivType();
   test_fetype = test_el->GetDerivType();

-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
-
   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+   const int MQdim = MQ ? MQfullDim : MQsymmDim;
+   const int coeffDim = (MQ || SMQ) ? MQdim : (DQ ? DQ->GetVDim() : 1);

-   symmetric = (SMQ || MQ == NULL);
+   symmetric = (MQ == NULL);

   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -821,7 +807,7 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   Vector coeff(coeffDim * ne * nq);
   coeff = 1.0;
   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
+   if (Q || DQ || MQ || SMQ)
   {
      Vector DM(DQ ? coeffDim : 0);
      DenseMatrix M;
@@ -831,35 +817,24 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      {
         MFEM_VERIFY(coeffDim == dim, "");
      }
-      if (SMQ)
-      {
-         MFEM_VERIFY(SMQ->GetSize() == dim, "");
-         SM.SetSize(dim);
-      }
-      else if (MQ)
+      if (MQ)
      {
         MFEM_VERIFY(coeffDim == MQdim, "");
         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
         M.SetSize(dim);
      }
-
+      if (SMQ)
+      {
+         MFEM_VERIFY(SMQ->GetSize() == dim, "");
+         SM.SetSize(dim);
+      }

      for (int e=0; e<ne; ++e)
      {
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            if (SMQ)
-            {
-               SMQ->Eval(SM, *tr, ir->IntPoint(p));
-               int cnt = 0;
-               for (int i=0; i<dim; ++i)
-                  for (int j=i; j<dim; ++j, ++cnt)
-                  {
-                     coeffh(cnt, p, e) = SM(i,j);
-                  }
-            }
-            else if (MQ)
+            if (MQ)
            {
               MQ->Eval(M, *tr, ir->IntPoint(p));

@@ -869,6 +844,16 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                     coeffh(j+(i*dim), p, e) = M(i,j);
                  }
            }
+            else if (SMQ)
+            {
+               SMQ->Eval(SM, *tr, ir->IntPoint(p));
+               int cnt = 0;
+               for (int i=0; i<dim; ++i)
+                  for (int j=i; j<dim; ++j, ++cnt)
+                  {
+                     coeffh(cnt, p, e) = SM(i,j);
+                  }
+            }
            else if (DQ)
            {
               DQ->Eval(DM, *tr, ir->IntPoint(p));
@@ -1039,16 +1024,16 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_curl && test_div)
      {
-         const bool scalarCoeff = !(DQ || MQ);
+         const bool scalarCoeff = !(DQ || MQ || SMQ);
         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                true, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
      }
      else if (trial_div && test_curl)
      {
-         const bool scalarCoeff = !(DQ || MQ);
+         const bool scalarCoeff = !(DQ || MQ || SMQ);
         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                false, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
      }
      else
@@ -1070,10 +1055,10 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if ((trial_curl && test_div) || (trial_div && test_curl))
      {
-         const bool scalarCoeff = !(DQ || MQ);
+         const bool scalarCoeff = !(DQ || MQ || SMQ);
         PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                trial_curl, false, mapsO->B, mapsC->B,
-                                mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
+                                trial_curl, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                mapsCtest->Bt, pa_data, x, y);
      }
      else
      {
@@ -1082,39 +1067,6 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

-void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,
-                                                Vector &y) const
-{
-   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
-   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
-   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
-   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
-
-   bool symmetricSpaces = true;
-
-   if (dim == 3 && ((trial_div && test_curl) || (trial_curl && test_div)))
-   {
-      const bool scalarCoeff = !(DQ || MQ);
-      PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                             trial_div, true, mapsO->B, mapsC->B, mapsOtest->Bt,
-                             mapsCtest->Bt, pa_data, x, y);
-      symmetricSpaces = false;
-   }
-   else if (dim == 2 && ((trial_curl && test_div) || (trial_div && test_curl)))
-   {
-      const bool scalarCoeff = !(DQ || MQ);
-      PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                             !trial_curl, true, mapsO->B, mapsC->B, mapsOtest->Bt,
-                             mapsCtest->Bt, pa_data, x, y);
-      symmetricSpaces = false;
-   }
-
-   if (symmetricSpaces)
-   {
-      this->AddMultPA(x, y);
-   }
-}
-
 void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
                                               &trial_fes,
                                               const FiniteElementSpace &test_fes)
@@ -0,0 +1,512 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+BlockBilinearForm::BlockBilinearForm(Array<FiniteElementSpace *> & fespaces_) :
+   Matrix(0), fespaces(fespaces_)
+{
+   height = 0;
+   nblocks = fespaces.Size();
+   dof_offsets.SetSize(nblocks+1);
+   tdof_offsets.SetSize(nblocks+1);
+   dof_offsets[0] = 0;
+   tdof_offsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      dof_offsets[i+1] = fespaces[i]->GetVSize();
+      tdof_offsets[i+1] = fespaces[i]->GetTrueVSize();
+   }
+   dof_offsets.PartialSum();
+   tdof_offsets.PartialSum();
+   height = dof_offsets[nblocks];
+   width = height;
+   mat = mat_e = NULL;
+   extern_bfs = 0;
+   element_matrices = NULL;
+   diag_policy = DIAG_KEEP;
+}
+
+
+// Allocate appropriate SparseMatrix and assign it to mat
+void BlockBilinearForm::AllocMat()
+{
+   mat = new SparseMatrix(height);
+}
+
+void BlockBilinearForm::BuildProlongation()
+{
+   P = new BlockMatrix(dof_offsets, tdof_offsets);
+   R = new BlockMatrix(tdof_offsets, dof_offsets);
+   for (int i = 0; i<nblocks; i++)
+   {
+      const SparseMatrix *P_ = fespaces[i]->GetConformingProlongation();
+      const SparseMatrix *R_ = fespaces[i]->GetRestrictionMatrix();
+      P->SetBlock(i,i,const_cast<SparseMatrix*>(P_));
+      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
+   }
+}
+
+void BlockBilinearForm::ConformingAssemble()
+{
+   Finalize(0);
+   MFEM_ASSERT(mat, "the BilinearForm is not assembled");
+
+   if (!P) { BuildProlongation(); }
+
+   SparseMatrix * Pm = P->CreateMonolithic();
+
+   SparseMatrix *Pt = Transpose(*Pm);
+
+   SparseMatrix *PtA = mfem::Mult(*Pt, *mat);
+   delete mat;
+   if (mat_e)
+   {
+      SparseMatrix *PtAe = mfem::Mult(*Pt, *mat_e);
+      delete mat_e;
+      mat_e = PtAe;
+   }
+   delete Pt;
+   mat = mfem::Mult(*PtA, *Pm);
+   delete PtA;
+   if (mat_e)
+   {
+      SparseMatrix *PtAeP = mfem::Mult(*mat_e, *Pm);
+      delete mat_e;
+      mat_e = PtAeP;
+   }
+   delete Pm;
+   height = mat->Height();
+   width = mat->Width();
+}
+
+void BlockBilinearForm::Mult(const Vector &x, Vector &y) const
+{
+   // TODO
+}
+
+
+double& BlockBilinearForm::Elem (int i, int j)
+{
+   return mat -> Elem(i,j);
+}
+
+const double& BlockBilinearForm::Elem (int i, int j) const
+{
+   return mat -> Elem(i,j);
+}
+
+MatrixInverse * BlockBilinearForm::Inverse() const
+{
+   return mat -> Inverse();
+}
+
+void BlockBilinearForm::Finalize(int skip_zeros)
+{
+   mat->Finalize(skip_zeros);
+   if (mat_e) { mat_e->Finalize(skip_zeros); }
+}
+
+/// Adds new Block Domain Integrator. Assumes ownership of @a bfi.
+void BlockBilinearForm::AddDomainIntegrator(BlockBilinearFormIntegrator *bfi)
+{
+   domain_integs.Append(bfi);
+}
+
+/// Assembles the form i.e. sums over all domain integrators.
+void BlockBilinearForm::Assemble(int skip_zeros)
+{
+   ElementTransformation *eltrans;
+   DofTransformation * doftrans_j, *doftrans_k;
+   Mesh *mesh = fespaces[0] -> GetMesh();
+   DenseMatrix elmat, *elmat_p;
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   Array<int> vdofs_j, vdofs_k;
+   Array<int> offsetvdofs_j;
+   Array<int> elementblockoffsets(nblocks+1);
+   elementblockoffsets[0] = 0;
+   Array<int> blockoffsets(nblocks+1);
+   blockoffsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      blockoffsets[i+1] = fespaces[i]->GetVSize();
+   }
+   blockoffsets.PartialSum();
+   // mfem::out << "blockoffsets = " ; blockoffsets.Print();
+
+   if (mat == NULL)
+   {
+      AllocMat();
+   }
+
+   if (domain_integs.Size())
+   {
+      // loop through elements
+      for (int i = 0; i < mesh -> GetNE(); i++)
+      {
+         if (element_matrices)
+         {
+            elmat_p = &(*element_matrices)(i);
+         }
+         else
+         {
+            elmat.SetSize(0);
+            for (int k = 0; k < domain_integs.Size(); k++)
+            {
+               for (int j = 0; j<nblocks; j++)
+               {
+                  fe[j] = fespaces[j]->GetFE(i);
+                  elementblockoffsets[j+1] = fe[j]->GetDof();
+               }
+               elementblockoffsets.PartialSum();
+               eltrans = mesh->GetElementTransformation(i);
+               domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+               if (elmat.Size() == 0)
+               {
+                  elmat = elemmat;
+               }
+               else
+               {
+                  elmat += elemmat;
+               }
+            }
+         }
+         if (elmat.Size() == 0)
+         {
+            continue;
+         }
+         else
+         {
+            elmat_p = &elmat;
+         }
+         vdofs.SetSize(0);
+         for (int j = 0; j<nblocks; j++)
+         {
+            doftrans_j = fespaces[j]->GetElementVDofs(i, vdofs_j);
+            int jbeg = elementblockoffsets[j];
+            int jend = elementblockoffsets[j+1]-1;
+            int offset_j = blockoffsets[j];
+            offsetvdofs_j.SetSize(vdofs_j.Size());
+
+            for (int l = 0; l<vdofs_j.Size(); l++)
+            {
+               offsetvdofs_j[l] = vdofs_j[l]<0 ? -offset_j + vdofs_j[l]
+                                  :  offset_j + vdofs_j[l];
+            }
+            vdofs.Append(offsetvdofs_j);
+            for (int k = 0; k<nblocks; k++)
+            {
+               doftrans_k = fespaces[k]->GetElementVDofs(i, vdofs_k);
+               if (doftrans_k || doftrans_j)
+               {
+                  int kbeg = elementblockoffsets[k];
+                  int kend = elementblockoffsets[k+1]-1;
+                  DenseMatrix A;
+                  elmat_p->GetSubMatrix(jbeg,jend,kbeg, kend, A);
+                  TransformDual(doftrans_j, doftrans_k, A);
+                  elmat_p->SetSubMatrix(jbeg,kbeg,A);
+               }
+            }
+         }
+         mat->AddSubMatrix(vdofs,vdofs,*elmat_p, skip_zeros);
+      }
+   }
+
+}
+
+
+
+void BlockBilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                         Vector &x,
+                                         Vector &b, OperatorHandle &A, Vector &X,
+                                         Vector &B, int copy_interior)
+{
+   FormSystemMatrix(ess_tdof_list, A);
+
+   if (!P)
+   {
+      EliminateVDofsInRHS(ess_tdof_list, x, b);
+      X.MakeRef(x, 0, x.Size());
+      B.MakeRef(b, 0, b.Size());
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+   else // non conforming space
+   {
+      B.SetSize(P->Width());
+      P->MultTranspose(b, B);
+      X.SetSize(R->Height());
+
+      mfem::out << "R height, width  = " << R->Height() <<" x "<< R->Width() <<
+                std::endl;
+
+      R->Mult(x, X);
+      EliminateVDofsInRHS(ess_tdof_list, X, B);
+      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
+   }
+
+}
+
+void BlockBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                         OperatorHandle &A)
+{
+   if (!mat_e)
+   {
+      const SparseMatrix *P_ = fespaces[0]->GetConformingProlongation();
+      if (P_) { ConformingAssemble(); }
+      EliminateVDofs(ess_tdof_list, diag_policy);
+      const int remove_zeros = 0;
+      Finalize(remove_zeros);
+   }
+   A.Reset(mat, false);
+}
+
+void BlockBilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
+                                           Vector &x)
+{
+   if (!P)
+   {
+      x.SyncMemory(X);
+   }
+   else
+   {
+      // Apply conforming prolongation
+      x.SetSize(P->Height());
+      P->Mult(X, x);
+   }
+}
+
+
+
+void BlockBilinearForm::ComputeElementMatrices()
+{
+   MFEM_ABORT("BlockBilinearForm::ComputeElementMatrices:not implemented yet")
+}
+
+void BlockBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
+{
+   if (element_matrices)
+   {
+      elmat.SetSize(element_matrices->SizeI(), element_matrices->SizeJ());
+      elmat = element_matrices->GetData(i);
+      return;
+   }
+
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   ElementTransformation *eltrans;
+
+   elmat.SetSize(0);
+   if (domain_integs.Size())
+   {
+      for (int j = 0; j<nblocks; j++)
+      {
+         fe[j] = fespaces[j]->GetFE(i);
+      }
+      eltrans = fespaces[0]->GetElementTransformation(i);
+      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
+      for (int k = 1; k < domain_integs.Size(); k++)
+      {
+         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+         elmat += elemmat;
+      }
+   }
+   else
+   {
+      int matsize = 0;
+      for (int j = 0; j<nblocks; j++)
+      {
+         matsize += fespaces[j]->GetFE(i)->GetDof();
+      }
+      elmat.SetSize(matsize);
+      elmat = 0.0;
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                                             const Vector &sol, Vector &rhs,
+                                             DiagonalPolicy dpolicy)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofs(ess_dofs, sol, rhs, dpolicy);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofs(conf_ess_dofs, sol, rhs, dpolicy);
+   // }
+}
+
+void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                                             DiagonalPolicy dpolicy)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofs(ess_dofs, dpolicy);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofs(conf_ess_dofs, dpolicy);
+   // }
+}
+
+void BlockBilinearForm::EliminateEssentialBCDiag (const Array<int>
+                                                  &bdr_attr_is_ess,
+                                                  double value)
+{
+   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBCDiag: not implemented yet");
+   // Array<int> ess_dofs, conf_ess_dofs;
+   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
+
+   // if (fes->GetVSize() == height)
+   // {
+   //    EliminateEssentialBCFromDofsDiag(ess_dofs, value);
+   // }
+   // else
+   // {
+   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
+   //    EliminateEssentialBCFromDofsDiag(conf_ess_dofs, value);
+   // }
+}
+
+void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
+                                       const Vector &sol, Vector &rhs,
+                                       DiagonalPolicy dpolicy)
+{
+   vdofs.HostRead();
+   for (int i = 0; i < vdofs.Size(); i++)
+   {
+      int vdof = vdofs[i];
+      if ( vdof >= 0 )
+      {
+         mat -> EliminateRowCol (vdof, sol(vdof), rhs, dpolicy);
+      }
+      else
+      {
+         mat -> EliminateRowCol (-1-vdof, sol(-1-vdof), rhs, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
+                                       DiagonalPolicy dpolicy)
+{
+   if (mat_e == NULL)
+   {
+      mat_e = new SparseMatrix(height);
+   }
+
+   // mat -> EliminateCols(vdofs, *mat_e,)
+
+   for (int i = 0; i < vdofs.Size(); i++)
+   {
+      int vdof = vdofs[i];
+      if ( vdof >= 0 )
+      {
+         mat -> EliminateRowCol (vdof, *mat_e, dpolicy);
+      }
+      else
+      {
+         mat -> EliminateRowCol (-1-vdof, *mat_e, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofs(
+   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
+   DiagonalPolicy dpolicy)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
+   MFEM_ASSERT(rhs.Size() == height, "incorrect rhs Vector size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowCol (i, sol(i), rhs, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofs (const Array<int>
+                                                      &ess_dofs,
+                                                      DiagonalPolicy dpolicy)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowCol (i, dpolicy);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateEssentialBCFromDofsDiag (
+   const Array<int> &ess_dofs,
+   double value)
+{
+   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+
+   for (int i = 0; i < ess_dofs.Size(); i++)
+   {
+      if (ess_dofs[i] < 0)
+      {
+         mat -> EliminateRowColDiag (i, value);
+      }
+   }
+}
+
+void BlockBilinearForm::EliminateVDofsInRHS(
+   const Array<int> &vdofs, const Vector &x, Vector &b)
+{
+   mat_e->AddMult(x, b, -1.);
+   mat->PartMult(vdofs, x, b);
+}
+
+
+
+BlockBilinearForm::~BlockBilinearForm()
+{
+   delete mat_e;
+   delete mat;
+   delete element_matrices;
+
+   for (int k=0; k < domain_integs.Size(); k++)
+   {
+      delete domain_integs[k];
+   }
+   for (int k=0; k < trace_integs.Size(); k++)
+   {
+      delete trace_integs[k];
+   }
+   delete P;
+   delete R;
+}
+
+
+
+} // namespace mfem
@@ -0,0 +1,288 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKBILINEARFORM
+#define MFEM_BLOCKBILINEARFORM
+
+#include "../config/config.hpp"
+#include "../linalg/linalg.hpp"
+
+namespace mfem
+{
+
+/** @brief A "square matrix" operator for the associated FE space and
+    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
+    M.  */
+class BlockBilinearForm : public Matrix
+{
+
+protected:
+
+   int nblocks;
+   Array<int> dof_offsets;
+   Array<int> tdof_offsets;
+
+   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
+   SparseMatrix *mat;
+
+   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
+        from the b.c.  Owned.
+       \f$ M + M_e = M_{original} \f$ */
+   SparseMatrix *mat_e;
+
+   /// FE spaces on which the block form lives. Not owned.
+   Array<FiniteElementSpace * > fespaces;
+
+   /** @brief Indicates the Mesh::sequence corresponding to the current state of
+      the BilinearForm. */
+   long sequence;
+
+   /** @brief Indicates the BlockBilinearFormIntegrator%s stored in #domain_integs,
+     are owned by another BlockBilinearForm. */
+   int extern_bfs;
+
+   /// Set of Domain Integrators to be applied.
+   Array<BlockBilinearFormIntegrator * > domain_integs;
+
+   /// Trace integrators.
+   Array<BlockBilinearFormIntegrator * > trace_integs;
+
+   DenseMatrix elemmat;
+   Array<int>  vdofs;
+
+   DenseTensor *element_matrices; ///< Owned.
+
+   BlockMatrix * P = nullptr; // Block Prolongation
+   BlockMatrix * R = nullptr; // Block Restriction
+
+   /** This data member allows one to specify what should be done to the
+    diagonal matrix entries and corresponding RHS values upon elimination of
+    the constrained DoFs. */
+   DiagonalPolicy diag_policy;
+
+   // Allocate appropriate SparseMatrix and assign it to mat
+   void AllocMat();
+
+   void ConformingAssemble();
+
+   void BuildProlongation();
+
+
+private:
+
+public:
+
+   /// Creates bilinear form associated with FE spaces @a *fespaces.
+   BlockBilinearForm(Array<FiniteElementSpace * > & fespaces_);
+
+   /// Get the size of the BilinearForm as a square matrix.
+   int Size() const { return height; }
+
+
+   /// Pre-allocate the internal SparseMatrix before assembly.
+   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
+
+   /// Returns a reference to: \f$ M_{ij} \f$
+   const double &operator()(int i, int j) { return (*mat)(i,j); }
+
+
+   /// Matrix vector multiplication:  \f$ y = M x \f$
+   virtual void Mult(const Vector &x, Vector &y) const;
+
+   /** @brief Matrix vector multiplication with the original uneliminated
+       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
+       \f$ y = M x + M_e x \f$ */
+   void FullMult(const Vector &x, Vector &y) const
+   { mat->Mult(x, y); mat_e->AddMult(x, y); }
+
+   virtual double &Elem(int i, int j);
+   virtual const double &Elem(int i, int j) const;
+   virtual MatrixInverse *Inverse() const;
+
+   /// Finalizes the matrix initialization.
+   virtual void Finalize(int skip_zeros = 1);
+
+   /// Returns a const reference to the sparse matrix.
+   const SparseMatrix &SpMat() const
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
+   SparseMatrix &SpMat()
+   {
+      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
+      return *mat;
+   }
+
+   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   const SparseMatrix &SpMatElim() const
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   SparseMatrix &SpMatElim()
+   {
+      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
+      return *mat_e;
+   }
+
+   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
+   void AddDomainIntegrator(BlockBilinearFormIntegrator *bfi);
+
+   /// Adds new Trace Integrator. Assumes ownership of @a bfi.
+   void AddTraceIntegrator(BlockBilinearFormIntegrator *bfi);
+
+   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
+   void operator=(const double a)
+   {
+      if (mat != NULL) { *mat = a; }
+      if (mat_e != NULL) { *mat_e = a; }
+   }
+
+   /// Assembles the form i.e. sums over all domain integrators.
+   void Assemble(int skip_zeros = 1);
+
+
+   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
+                                 Vector &b, OperatorHandle &A, Vector &X,
+                                 Vector &B, int copy_interior = 0);
+
+   /** @brief Form the linear system A X = B, corresponding to this bilinear
+       form and the linear form @a b(.). */
+   /** Version of the method FormLinearSystem() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
+                         OpType &A, Vector &X, Vector &B,
+                         int copy_interior = 0)
+   {
+      OperatorHandle Ah;
+      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
+                                 OperatorHandle &A);
+
+   /// Form the linear system matrix A, see FormLinearSystem() for details.
+   /** Version of the method FormSystemMatrix() where the system matrix is
+       returned in the variable @a A, of type OpType, holding a *reference* to
+       the system matrix (created with the method OpType::MakeRef()). The
+       reference will be invalidated when SetOperatorType(), Update(), or the
+       destructor is called. */
+   template <typename OpType>
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
+   {
+      OperatorHandle Ah;
+      FormSystemMatrix(ess_tdof_list, Ah);
+      OpType *A_ptr = Ah.Is<OpType>();
+      MFEM_VERIFY(A_ptr, "invalid OpType used");
+      A.MakeRef(*A_ptr);
+   }
+
+   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
+
+
+   void ComputeElementMatrices();
+
+   /// Free the memory used by the element matrices.
+   void FreeElementMatrices()
+   { delete element_matrices; element_matrices = NULL; }
+
+   /// Compute the element matrix of the given element
+   /** The element matrix is computed by calling the domain integrators
+       or the one stored internally by a prior call of ComputeElementMatrices()
+       is returned when available.
+   */
+   void ComputeElementMatrix(int i, DenseMatrix &elmat);
+
+   /// Eliminate essential boundary DOFs from the system.
+   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
+       the essential part of the boundary. By default, the diagonal at the
+       essential DOFs is set to 1.0. This behavior is controlled by the argument
+       @a dpolicy. */
+   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                             const Vector &sol, Vector &rhs,
+                             DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /// Eliminate essential boundary DOFs from the system matrix.
+   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
+                             DiagonalPolicy dpolicy = DIAG_ONE);
+   /// Perform elimination and set the diagonal entry to the given value
+   void EliminateEssentialBCDiag(const Array<int> &bdr_attr_is_ess,
+                                 double value);
+
+   /// Eliminate the given @a vdofs.
+   /** NOTE: here, @a vdofs is a list of DOFs from all the fespaces
+       In this case the eliminations are applied to the internal \f$ M \f$
+       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
+   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
+                       DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /// Eliminate the given @a vdofs (all the fespaces), storing the eliminated part internally in \f$ M_e \f$.
+   /** This method works in conjunction with EliminateVDofsInRHS() and allows
+       elimination of boundary conditions in multiple right-hand sides. In this
+       method, @a vdofs is a list of DOFs. */
+   void EliminateVDofs(const Array<int> &vdofs,
+                       DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /** @brief Similar to
+       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
+       but here @a ess_dofs is a marker (boolean) array on all vector-dofs
+       (@a ess_dofs[i] < 0 is true). */
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
+                                     Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
+
+   /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
+       here @a ess_dofs is a marker (boolean) array on all vector-dofs
+       (@a ess_dofs[i] < 0 is true). */
+   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs,
+                                     DiagonalPolicy dpolicy = DIAG_ONE);
+   /// Perform elimination and set the diagonal entry to the given value
+   void EliminateEssentialBCFromDofsDiag(const Array<int> &ess_dofs,
+                                         double value);
+
+   /** @brief Use the stored eliminated part of the matrix (see
+       EliminateVDofs(const Array<int> &, DiagonalPolicy)) to modify the r.h.s.
+       @a b; @a vdofs is a list of DOFs (non-directional, i.e. >= 0). */
+   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
+                            Vector &b);
+
+
+   /// Sets diagonal policy used upon construction of the linear system.
+   /** Policies include:
+
+       - DIAG_ZERO (Set the diagonal values to zero)
+       - DIAG_ONE  (Set the diagonal values to one)
+       - DIAG_KEEP (Keep the diagonal values)
+   */
+   void SetDiagonalPolicy(DiagonalPolicy policy)
+   {
+      diag_policy = policy;
+   }
+
+   /// Destroys bilinear form.
+   virtual ~BlockBilinearForm();
+
+};
+
+} // namespace mfem
+
+#endif
@@ -0,0 +1,136 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+void BlockBilinearFormIntegrator::AssembleElementMatrix(
+   const Array<const FiniteElement *> &el,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   mfem_error ("BlockBilinearFormIntegrator::AssembleElementMatrix\n"
+               "   is not implemented for this class.");
+}
+
+void BlockLinearFormIntegrator::AssembleRHSElementVect(
+   const Array<const FiniteElement *> &el,
+   ElementTransformation &Trans,
+   Vector &elvect)
+{
+   mfem_error ("BlockLinearFormIntegrator::AssembleElementVector\n"
+               "   is not implemented for this class.");
+}
+
+/** Given a particular Finite Element computes the element vector */
+void TestBlockBilinearFormIntegrator::AssembleElementMatrix
+(const Array<const FiniteElement *> &el,
+ ElementTransformation &Trans,
+ DenseMatrix &elmat)
+{
+   int nd = 0;
+   int nblocks = el.Size();
+   Array<int> offsets(nblocks+1);
+   offsets[0] = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      nd += el[i]->GetDof();
+      offsets[i+1] = el[i]->GetDof();
+   }
+   offsets.PartialSum();
+   elmat.SetSize(nd);
+   elmat = 0.0;
+   DenseMatrix dmat;
+
+   if (blfis.NumRows())
+   {
+      // Get the matrices directly from the existing BilinearFormIntegrators
+      for (int i = 0; i<nblocks; i++)
+      {
+         // mfem::out << "i = " << i << std::endl;
+         int offset_i = offsets[i];
+         const FiniteElement * fe_i = el[i];
+         for (int j = 0; j<nblocks; j++)
+         {
+            // mfem::out << "j = " << j << std::endl;
+            BilinearFormIntegrator * blfi = blfis(i,j);
+            if (!blfi) { continue; }
+            if (j == i)
+            {
+               blfi->AssembleElementMatrix(*fe_i,Trans,dmat);
+               // mfem::out << "j 1 = " << j << std::endl;
+               elmat.SetSubMatrix(offset_i,dmat);
+            }
+            else
+            {
+               const FiniteElement * fe_j = el[j];
+               blfi->AssembleElementMatrix2(*fe_j,*fe_i,Trans,dmat);
+               // mfem::out << "j 2 = " << j << std::endl;
+               int offset_j = offsets[j];
+               elmat.SetSubMatrix(offset_i,offset_j,dmat);
+            }
+         }
+      }
+      return;
+   }
+
+   // else compute the matrices
+   elmat = 25.0;
+   // TODO
+
+}
+
+/** Given a particular Finite Element computes the element vector */
+void TestBlockLinearFormIntegrator::AssembleRHSElementVect
+(const Array<const FiniteElement *> &el,
+ ElementTransformation &Trans,
+ Vector &elvector)
+{
+   int nd = 0;
+   int nblocks = el.Size();
+   Array<int> offsets(nblocks+1);
+   offsets[0] = 0;
+   for (int i = 0; i<nblocks; i++)
+   {
+      nd += el[i]->GetDof();
+      offsets[i+1] = el[i]->GetDof();
+   }
+   offsets.PartialSum();
+   elvector.SetSize(nd);
+   elvector = 0.0;
+   Vector subvector;
+
+   if (lfis.Size())
+   {
+      // Get the matrices directly from the existing BilinearFormIntegrators
+      for (int i = 0; i<nblocks; i++)
+      {
+         int offset = offsets[i];
+         const FiniteElement * fe_i = el[i];
+         LinearFormIntegrator * lfi = lfis[i];
+         if (!lfi)
+         {
+            continue;
+         }
+         lfi->AssembleRHSElementVect(*fe_i,Trans,subvector);
+         elvector.SetVector(subvector,offset);
+      }
+      return;
+   }
+
+   // else, compute the block linear form integrator
+   // elvector = 1.0;
+   // TODO
+}
+
+} // namespace mfem
@@ -0,0 +1,147 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BLOCKINTEG
+#define MFEM_BLOCKINTEG
+
+#include "../config/config.hpp"
+#include "fe.hpp"
+#include "coefficient.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+/** The abstract base class BlockBilinearFormIntegrator is
+ a generalization of the BilinearFormIntegrator class suitable
+ for block formulations. */
+class BlockBilinearFormIntegrator
+{
+protected:
+   const IntegrationRule *IntRule;
+   BlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : IntRule(ir) { }
+public:
+
+
+   /// Given a particular Finite Element computes the element matrix elmat.
+   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+
+   virtual ~BlockBilinearFormIntegrator() { }
+};
+
+/** The abstract base class BlockBilinearFormIntegrator is
+ a generalization of the BilinearFormIntegrator class suitable
+ for block formulations. */
+class BlockLinearFormIntegrator
+{
+protected:
+   const IntegrationRule *IntRule;
+   BlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : IntRule(ir) { }
+
+public:
+   /// Given a particular Finite Element computes the element matrix elmat.
+   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
+                                       ElementTransformation &Trans,
+                                       Vector &elvect);
+
+   virtual ~BlockLinearFormIntegrator() { }
+};
+
+
+
+
+class TestBlockBilinearFormIntegrator: public BlockBilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   Array<const FiniteElementSpace * > fespaces;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+
+   Array2D<BilinearFormIntegrator *> blfis;
+
+
+public:
+
+   TestBlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : BlockBilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a mass integrator with coefficient q
+   TestBlockBilinearFormIntegrator(Coefficient &q,
+                                   const IntegrationRule *ir = NULL)
+      : BlockBilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL)   { }
+
+
+   TestBlockBilinearFormIntegrator(Array2D<BilinearFormIntegrator *> blfis_)
+      : BlockBilinearFormIntegrator(NULL), blfis(blfis_) { }
+
+   void SetIntegrators(Array2D<BilinearFormIntegrator *> blfis_)
+   {
+      blfis = blfis_;
+   }
+
+
+   /** Given a particular Finite Element computes the element matrix
+       elmat. */
+   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
+                                      ElementTransformation &Trans,
+                                      DenseMatrix &elmat);
+
+   virtual ~TestBlockBilinearFormIntegrator() { }
+
+};
+
+/** Class for local vector assembly */
+class TestBlockLinearFormIntegrator: public BlockLinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+   Array<const FiniteElementSpace * > fespaces;
+   const DofToQuad *maps;         ///< Not owned
+   const GeometricFactors *geom;  ///< Not owned
+   int dim, ne, nq, dofs1D, quad1D;
+   Array<LinearFormIntegrator *> lfis;
+
+public:
+
+   TestBlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
+      : BlockLinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
+
+   /// Construct a test linear integrator with coefficient q
+   TestBlockLinearFormIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
+      : BlockLinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL) { }
+
+
+   TestBlockLinearFormIntegrator(Array<LinearFormIntegrator *> lfis_)
+      : BlockLinearFormIntegrator(NULL), lfis(lfis_) { }
+
+   void SetIntegrators(Array<LinearFormIntegrator *> lfis_)
+   {
+      lfis = lfis_;
+   }
+
+   /** Given a particular Finite Element computes the element vector */
+   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
+                                       ElementTransformation &Trans,
+                                       Vector &elvector);
+
+
+};
+
+
+} // namespace mfem
+
+
+#endif
@@ -0,0 +1,123 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "fem.hpp"
+
+namespace mfem
+{
+
+BlockLinearForm::BlockLinearForm(Array<FiniteElementSpace * > & fespaces_) :
+   Vector(0), fespaces(fespaces_)
+{
+   int s = 0;
+   int nblocks = fespaces.Size();
+   for (int i =0; i<nblocks; i++)
+   {
+      s += fespaces[i]->GetVSize();
+   }
+   // mfem::out << "size = " << size << std::endl;
+
+   SetSize(s);
+
+}
+
+
+void BlockLinearForm::AddDomainIntegrator(BlockLinearFormIntegrator *lfi)
+{
+   domain_integs.Append(lfi);
+}
+
+void BlockLinearForm::Assemble()
+{
+   ElementTransformation *eltrans;
+   DofTransformation *doftrans;
+   Mesh *mesh = fespaces[0] -> GetMesh();
+   Vector subvect,elvect, *elvect_p;
+
+   int nblocks = fespaces.Size();
+   Array<const FiniteElement *> fe(nblocks);
+   Array<int> offsetvdofs;
+   Array<int> elementblockoffsets(nblocks+1);
+   elementblockoffsets[0] = 0;
+   Array<int> blockoffsets(nblocks+1);
+   blockoffsets[0] = 0;
+   for (int i =0; i<nblocks; i++)
+   {
+      blockoffsets[i+1] = fespaces[i]->GetVSize();
+   }
+   blockoffsets.PartialSum();
+
+   Vector::operator=(0.0);
+
+   if (domain_integs.Size())
+   {
+      // loop through elements
+      for (int i = 0; i < mesh -> GetNE(); i++)
+      {
+         elvect.SetSize(0);
+         for (int k = 0; k < domain_integs.Size(); k++)
+         {
+            for (int j = 0; j<nblocks; j++)
+            {
+               fe[j] = fespaces[j]->GetFE(i);
+               elementblockoffsets[j+1] = fe[j]->GetDof();
+            }
+            elementblockoffsets.PartialSum();
+            eltrans = mesh->GetElementTransformation(i);
+
+            domain_integs[k]->AssembleRHSElementVect(fe, *eltrans, elemvect);
+            if (elvect.Size() == 0)
+            {
+               elvect = elemvect;
+            }
+            else
+            {
+               elvect += elemvect;
+            }
+         }
+         if (elvect.Size() == 0)
+         {
+            continue;
+         }
+         else
+         {
+            elvect_p = &elvect;
+         }
+
+         double *data = elvect_p->GetData();
+
+         for (int j = 0; j<nblocks; j++)
+         {
+            doftrans = fespaces[j]->GetElementVDofs(i, vdofs);
+            int offset = blockoffsets[j];
+            offsetvdofs.SetSize(vdofs.Size());
+            for (int l = 0; l<vdofs.Size(); l++)
+            {
+               offsetvdofs[l] = vdofs[l]<0 ? -offset + vdofs[l]
+                                :  offset + vdofs[l];
+            }
+            int jbeg = elementblockoffsets[j];
+            int jend = elementblockoffsets[j+1]-1;
+            subvect.SetSize(jend-jbeg+1);
+            subvect.SetData(&data[jbeg]);
+
+            if (doftrans)
+            {
+               doftrans->TransformDual(subvect);
+            }
+            AddElementVector(offsetvdofs,subvect);
+         }
+      }
+   }
+}
+
+
+} // name space mfem
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Socratis Petrides	d322228504	testing dpg residual with cholesky factors	2022-03-22 14:13:56 -07:00
Socratis Petrides	2cc8c18250	fix repo check	2022-03-22 12:08:56 -07:00
Socratis Petrides	48c54c0bbc	fix repo-check	2022-03-22 12:05:22 -07:00
Socratis Petrides	aa9a4887f7	change to 2022	2022-03-22 10:24:38 -07:00
Socratis Petrides	757369beb3	Merge branch 'master' into dpg-dev	2022-03-22 10:19:35 -07:00
Socratis Petrides	6967342813	minor	2022-03-03 13:06:29 -08:00
Socratis Petrides	ba94547c4d	fixing residual computation	2022-03-01 19:20:44 -08:00
Socratis Petrides	58f7aacce9	fix acoustics graph norm	2022-02-25 17:45:49 -08:00
psocratis	66e0c79b3f	small valgrind fix in the parallel case	2022-02-11 09:50:52 -08:00
psocratis	8b5f9f7720	resolving conflicts	2022-02-09 19:50:00 -08:00
psocratis	98739f8c09	fix valgrind complaints in the serial case	2022-02-09 19:41:48 -08:00
Socratis Petrides	84129a1f83	finished Block-sc in the parallel case, uw_dpgp passes initial tests	2022-02-09 17:04:01 -08:00
Socratis Petrides	b6ea025795	cleanup, started on par sc case	2022-02-02 20:18:19 -08:00
Socratis Petrides	d528e9d63e	filling blockstaticcond destructor	2022-02-01 10:19:19 -08:00
Socratis Petrides	d18f11e6c7	blockstatic-cond passes tests for primal-dpg	2022-01-31 17:56:37 -08:00
Socratis Petrides	efeb400b27	Block sc FormSystemMatrix. Started on ReduceSystem	2022-01-28 20:01:07 -08:00
Socratis Petrides	1451c3c483	small bug-fix in assembly if doftrans is not null	2022-01-27 17:33:04 -08:00
Socratis Petrides	f12d46bdf9	Shur complement assembly and conforming assembly	2022-01-27 17:31:55 -08:00
Socratis Petrides	6dfb809f97	changing GetSubMatrix to const	2022-01-27 17:30:38 -08:00
Socratis Petrides	ef8785b852	computing indices for interior/interface local dofs for block-static-cond	2022-01-26 18:24:56 -08:00
Socratis Petrides	efd182ec4c	visualize mesh in amr for the l-shape problem	2022-01-26 18:23:12 -08:00
Socratis Petrides	e7856b982a	style	2022-01-26 10:59:38 -08:00
Socratis Petrides	4f4d1f9d2d	adding function signatures in blockstaticcond	2022-01-26 10:59:19 -08:00
Socratis Petrides	1a89d5af02	started on static condensation for DPG (block) systems	2022-01-25 19:45:40 -08:00
Socratis Petrides	c71b02a915	fix print format	2022-01-21 14:37:21 -08:00
Socratis Petrides	5576f3653e	style	2022-01-21 13:45:29 -08:00
Socratis Petrides	7f316dd07b	changing strong_dpg to use vdim>1 test space	2022-01-21 13:45:00 -08:00
Socratis Petrides	19b5ae6b9a	parallel examples acoustics uw_dpg	2022-01-21 13:44:17 -08:00
Socratis Petrides	94cd8859b2	adding adjoint graphnorm in uw_dpg for acoustics	2022-01-21 13:43:56 -08:00
Socratis Petrides	8ea8b97fc8	adding vdim>1 for test spaces in normalequations	2022-01-21 13:43:00 -08:00
Socratis Petrides	f9abc77026	clean up	2022-01-20 16:45:43 -08:00
Socratis Petrides	68e3f9aeb9	adding traditional fosls formulation for acoustics (real)	2022-01-20 16:34:51 -08:00
Socratis Petrides	08f018c299	adding uw dpg formulation for acoustics (real)	2022-01-20 16:33:58 -08:00
Socratis Petrides	69e57546d4	adding 'strong' dpg formulation for acoustics	2022-01-20 16:33:35 -08:00
Socratis Petrides	433a29c9e3	fixing comment in diffusion fosls	2022-01-20 16:32:47 -08:00
Socratis Petrides	92ee7099a4	reorganizing examples	2022-01-19 11:05:16 -08:00
Socratis Petrides	808c9cd2c4	Merge branch 'master' into dpg-dev	2022-01-19 10:10:05 -08:00
Socratis Petrides	f3cd0a10e7	minor	2022-01-17 11:30:23 -08:00
Socratis Petrides	e6bb8c8976	small bugfix in updating mesh dependent coefficient	2022-01-05 18:09:49 -08:00
Socratis Petrides	4718577810	bug-fix serial AMR in normalequations P/R Mult in RHS and Sol vector	2021-12-30 19:59:14 -08:00
Socratis Petrides	9e520183c8	Merge branch 'master' into dpg-dev	2021-12-30 09:26:52 -08:00
Socratis Petrides	95a40d7377	clean up convection-diffusion	2021-12-22 14:40:11 -08:00
Socratis Petrides	036846bcd6	adding parallel convection-diffusion with AMR example	2021-12-21 17:59:56 -08:00
Socratis Petrides	6c2968156f	Merge branch 'master' into dpg-dev	2021-12-21 10:01:43 -08:00
Socratis Petrides	9ba650f2fd	adding Erikson Johnson problem for convection-diffusion UW-DPG	2021-12-17 16:34:22 -08:00
Socratis Petrides	ee6cb39ee0	Merge branch 'pncmesh-getessvdof-fix' into dpg-dev	2021-12-16 09:46:14 -08:00
Socratis Petrides	1701a6e84c	Merge branch 'master' into dpg-dev	2021-12-16 09:45:02 -08:00
Socratis Petrides	f500e5220a	Parallel AMR for lshape	2021-12-15 18:28:31 -08:00
Socratis Petrides	e078288ecb	title fix	2021-12-15 12:40:42 -08:00
Socratis Petrides	e46c7ac0e7	fix sample run and title in the example	2021-12-15 12:39:33 -08:00
Socratis Petrides	c2b133f3c8	minor visualization edits	2021-12-15 12:37:46 -08:00
Socratis Petrides	373379fbdd	lshape mesh	2021-12-15 12:20:41 -08:00
Socratis Petrides	bfc3835a1d	adding AMR l-shape example in diffusion_uwdpg	2021-12-15 12:20:08 -08:00
Socratis Petrides	ab8578a728	add element residual computation to NormalEquations	2021-12-15 12:19:30 -08:00
Socratis Petrides	fdaaacfdf3	Started residual based error estimator. Done some refactoring	2021-12-13 17:36:52 -08:00
Socratis Petrides	5d7ebbb51a	fixed small size bug in DenseMatrix::GetSubMatrix	2021-12-13 17:35:26 -08:00
Socratis Petrides	fe0821413a	adding parallel test for uw diffusion	2021-12-09 15:16:49 -08:00
Socratis Petrides	695a5adcb2	ParNormalEquations cleanup	2021-12-09 15:16:15 -08:00
Socratis Petrides	b2dfc76f1a	minor changes in serial example	2021-12-09 15:13:54 -08:00
Socratis Petrides	76c700b41e	adding BlockOperator and BlockMatrix in OpType	2021-12-09 15:13:05 -08:00
Socratis Petrides	dc3f10fa63	Passing first tests in parallel for ParNormalEquations	2021-12-08 17:36:34 -08:00
Socratis Petrides	420bb73d6c	minor change in normal equations constructor	2021-12-08 17:36:11 -08:00
Socratis Petrides	13648768ae	removing unused code	2021-12-08 17:34:00 -08:00
Socratis Petrides	7bfda36bca	Starting ParNormalEquations -> adding class methods signatures	2021-12-07 18:48:24 -08:00
Socratis Petrides	46fc3ce9c2	remove not used code	2021-12-07 18:47:20 -08:00
Socratis Petrides	b3086ce754	Merge branch 'master' into dpg-dev	2021-12-07 13:05:01 -08:00
Socratis Petrides	d1312d354a	minor bug-fix in the adjoint graph norm	2021-12-03 20:04:49 -08:00
Socratis Petrides	16fcc3f970	Fixing P and R null diagonal blocks to act as Identity for L2 Space and AMR	2021-12-03 20:04:23 -08:00
Socratis Petrides	a6caab3bed	PartMult and AddPartMult for BlockMatrix	2021-12-03 19:53:27 -08:00
Socratis Petrides	172b5bdcc9	Modifying DPG examples to use block-diagonal preconditioners	2021-12-02 17:25:03 -08:00
Socratis Petrides	72f83c132d	Changing normalequations to use only BlockMatrix	2021-12-02 17:24:25 -08:00
Socratis Petrides	911e394d80	adding EliminateRowCols with saving Ae to BlockMatrix class	2021-12-02 17:22:36 -08:00
Socratis Petrides	4ff29e40c9	Refactoring NormalEquations assemble to use BlockMatrix	2021-12-01 19:00:07 -08:00
Socratis Petrides	dac7808875	style	2021-11-30 18:23:31 -08:00
Socratis Petrides	246fd8e6fe	started element residual calculation for AMR	2021-11-30 18:23:09 -08:00
Socratis Petrides	5e243e01f8	clean up	2021-11-30 18:22:31 -08:00
Socratis Petrides	56b83ea538	UW-DPG with AMR: fixing Block Prolongation/Restriction Operators	2021-11-30 16:41:21 -08:00
Socratis Petrides	cd7371896a	removing no longer needed test	2021-11-29 17:30:01 -08:00
Socratis Petrides	3fdd3e2450	style	2021-11-29 17:28:42 -08:00
Socratis Petrides	303ccefd83	completing the descructor	2021-11-29 17:28:17 -08:00
Socratis Petrides	48bcc332f7	fix piola map bug in NormalTraceIntegrator	2021-11-29 17:19:05 -08:00
Socratis Petrides	0c23d23874	new diffusion uw-dpg with adjoint graph norm	2021-11-29 17:17:54 -08:00
Socratis Petrides	e76cef225d	UW-DPG for diffusion works on quad meshes	2021-11-28 15:03:34 -08:00
Socratis Petrides	6f49201e62	Ultraweak-DPG for poisson prototype	2021-11-27 14:23:24 -08:00
Socratis Petrides	1cb75f7730	adding diffusion UW-DPG	2021-11-24 16:56:15 -08:00
Socratis Petrides	ef14d682c5	bug-fix ElementTrasformation	2021-11-24 16:55:45 -08:00
Socratis Petrides	54eb78e0df	adding block primal test	2021-11-23 17:13:22 -08:00
Socratis Petrides	cd1a3a4fc7	fixing minor bug in B^T G^-1 l	2021-11-23 17:12:02 -08:00
Socratis Petrides	ffb50c2416	example tests cleanup	2021-11-23 10:05:44 -08:00
Socratis Petrides	f568ca3c6e	bug fix on accumulating domain integrators	2021-11-23 10:03:13 -08:00
Socratis Petrides	c7fb058051	adding NormalTraceIntegrator and AssembleElementMatrix2 for DivDivIntegrator	2021-11-23 09:58:11 -08:00
Socratis Petrides	a4d87d936b	Adding NormalEquations Assembly for multiple spaces and integrators. Tested succesfully for primal DPG	2021-11-19 19:52:27 -08:00
Socratis Petrides	f26bd4bea1	generalizing NormalEquations assembly for multiple fespaces and integrators	2021-11-18 17:20:30 -08:00
Socratis Petrides	6c001c3f9a	adding example test for primal DPG	2021-11-15 17:57:09 -08:00
Socratis Petrides	82573af316	new Primal DPG reproduces ex8	2021-11-15 17:56:16 -08:00
Socratis Petrides	4bfb2d62d9	first primal dpg test runs	2021-11-15 13:41:57 -08:00
Socratis Petrides	0bf7a715db	minor cleanup	2021-11-12 20:04:01 -08:00
Socratis Petrides	ad68c8f245	Fixing doc complaints	2021-11-12 18:52:48 -08:00
Socratis Petrides	0b430fa5f6	Started NormalEquationsWeakFormulation::Assemble	2021-11-12 18:49:06 -08:00
Socratis Petrides	238d1921c7	adding TraceIntegrator and AssembleTraceFaceMatrix	2021-11-12 18:48:05 -08:00
Socratis Petrides	a25783e3b4	Simplifying P and R in BlockBilinearForm	2021-11-04 09:43:40 -07:00
Socratis Petrides	3aa8d49ca8	started implementation of trace_elem_integrators for DPG	2021-11-03 18:29:08 -07:00
Socratis Petrides	4ed6a4a014	fixing minor bug in reference BlockFOSLS case	2021-11-03 15:11:20 -07:00
Socratis Petrides	08f910343b	AMR for reference Block FOSLS	2021-11-03 11:16:59 -07:00
Socratis Petrides	9fb0340206	fixing minor bug in P/R for AMR with blockforms	2021-11-03 09:03:18 -07:00
Socratis Petrides	db40e8125e	Setting Prolongation/Restriction for BlockBilinearForm. Tested for 2D AMR	2021-11-02 18:56:07 -07:00
Socratis Petrides	3e46fce65f	simplifying BlockBilinearFrom::Assembly()	2021-11-02 17:53:50 -07:00
Socratis Petrides	f649ed6b62	investigating possible bug in high order 3D runs wrt to essential BC elimination (example 1 hexa mesh)	2021-11-01 17:34:59 -07:00
Socratis Petrides	a1a3a51d86	simplifying offset calculation wrt neg orientation	2021-11-01 17:34:05 -07:00
Socratis Petrides	6728fc7eda	make style	2021-11-01 16:44:38 -07:00
Socratis Petrides	71e9d4e1d9	poisson_fosls - reference implementation	2021-11-01 16:44:00 -07:00
Socratis Petrides	7feb560f9b	testing blockbilinearform with FOSLS poisson	2021-11-01 16:43:28 -07:00
Socratis Petrides	aa9eadfcdf	fixing orientation (sign) bug in blockbilinearform assemble	2021-11-01 16:42:59 -07:00
Socratis Petrides	82122f05cf	testing block integrator by borrowing (bi)linearIntegrators	2021-11-01 16:42:20 -07:00
Socratis Petrides	2d64fa2162	fix bug in blocklinearform assemble	2021-11-01 16:41:21 -07:00
Socratis Petrides	8041734755	adding DenseMatrix::SetSubMatrix functions	2021-11-01 16:40:24 -07:00
Socratis Petrides	523891f05f	adding BlockLinearForm (assemble) with testblocklinearinteg	2021-10-29 18:32:31 -07:00
Socratis Petrides	b3fc46b6f2	Merge branch 'master' into dpg-dev	2021-10-29 16:43:38 -07:00
Socratis Petrides	9271494723	Merge branch 'master' into dpg-dev	2021-10-27 08:45:53 -07:00
Socratis Petrides	f2dbcecb92	debugging BlockBilinearForm::Assemble()	2021-10-25 16:47:32 -07:00
Socratis Petrides	f2af9748c4	fixing offset computation in BlockBilinearForm::Assemble()	2021-10-22 19:39:51 -07:00
Socratis Petrides	9813dd7722	fix doxygen complaint	2021-10-22 19:28:52 -07:00
Socratis Petrides	3806e68ed1	adding new classes (block(bi)linearForms(integ) in support for DPG methods	2021-10-22 17:58:52 -07:00