make style

update linear form move ctor/assign to correctly move recently added LinearFormExtension member ext
Merge branch 'master' into move-nlf-dev
2022-06-30 12:33:11 -06:00 · 2022-06-30 12:30:45 -06:00 · 2022-06-30 12:27:48 -06:00 · 2022-06-27 16:11:02 -07:00 · 2022-06-23 12:40:15 -07:00 · 2022-06-17 11:06:24 -07:00
299 changed files with 20176 additions and 12664 deletions
@@ -1,10 +1,13 @@
 version: '{build}'

 # https://www.appveyor.com/docs/build-environment/#build-worker-images
-image: Visual Studio 2017
+image: Visual Studio 2019

 install:

+# Start from outside clone directory
+- cd ..
+
 # Install MS-MPI
 - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
 - MSMpiSetup.exe -unattend
@@ -15,6 +18,11 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

+# Set MSMPI environment variables needed for CMake detection
+- set MSMPI_LIB32=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86
+- set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
+- set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
+
 # Install METIS, use a mirror because the original source server is not always
 # up. Original url:
 #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
@@ -25,20 +33,24 @@ install:
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
 - cmake --build build
+- set METIS_PATH=%cd%
 - cd ..

 # Install hypre
 - ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- cmake -H. -Bbuild
 - cmake --build build
 - cmake --build build --target install
 - cd ../..

+# Return to clone directory
+- cd %APPVEYOR_BUILD_FOLDER%
+
 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -0,0 +1,81 @@
+name: Build Deploy Container
+
+on:
+
+  # Always have a base image ready to go - this is a nightly build
+  schedule:
+    - cron: 0 3 * * *
+
+  # Allow manual trigger of a build
+  workflow_dispatch:
+
+  # On push to main we build and deploy images
+  push:
+    branches:
+      - master
+
+  # Publish packages on release
+  release:
+    types: [published]
+
+jobs:
+  build:
+    if: github.repository == 'mfem/mfem' # Don't run in forks
+    permissions:
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+
+        # Dockerfiles to build, a matrix supports future expanded builds
+        container: [["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu-base"]]
+
+    runs-on: ubuntu-latest
+    name: Build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Make Space For Build
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+
+       # It's easier to reference named variables than indexes of the matrix
+      - name: Set Environment
+        env:
+          dockerfile: ${{ matrix.container[0] }}
+          uri: ${{ matrix.container[1] }}
+        run: |
+          echo "dockerfile=$dockerfile" >> $GITHUB_ENV
+          echo "uri=$uri" >> $GITHUB_ENV
+
+      - name: Pull previous layers for cache
+        run: docker pull ${uri}:latest || echo "No container to pull"
+
+      - name: Build Container
+        run: |
+           container=$uri:latest
+           docker build -f ${dockerfile} -t ${container} .
+           echo "container=$container" >> $GITHUB_ENV
+
+      - name: GHCR Login
+        if: (github.event_name != 'pull_request')
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Deploy
+        if: (github.event_name != 'pull_request')
+        run: |
+            docker push ${container}
+
+      - name: Tag and Push Release
+        if: (github.event_name == 'release')
+        run: |
+            tag=${GITHUB_REF#refs/tags/}
+            echo "Tagging and releasing ${uri}:${tag}"
+            docker tag ${uri}:latest ${uri}:${tag}
+            docker push ${uri}:${tag}
@@ -27,6 +27,7 @@ on:
      - master
      - next
  pull_request:
+  workflow_dispatch:

 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -46,11 +47,18 @@ jobs:
  builds-and-tests:
    strategy:
      matrix:
-        os: [ubuntu-18.04, macos-10.15]
+        os: [ubuntu-20.04, macos-10.15, windows-2022]
        target: [dbg, opt]
        mpi: [seq, par]
-        build-system: [make]
+        build-system: [make, cmake]
        hypre-target: [int32]
+        exclude:
+          - os: ubuntu-20.04
+            build-system: cmake
+          - os: macos-10.15
+            build-system: cmake
+          - os: windows-2022
+            build-system: make
        # 'include' allows us to:
        # - Add a variable to all jobs without creating a new matrix dimension.
        #   Codecov is defined that way.
@@ -64,13 +72,15 @@ jobs:
            codecov: NO
          - target: opt
            codecov: YES
-          - os: ubuntu-18.04
+          - os: windows-2022
+            codecov: NO
+          - os: ubuntu-20.04
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
-          - os: ubuntu-18.04
+          - os: ubuntu-20.04
            target: opt
            codecov: NO
            mpi: par
@@ -102,13 +112,13 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
      run: |
        sudo apt-get install lcov

@@ -129,6 +139,10 @@ jobs:
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install lcov

+    - name: get MPI (Windows)
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      uses: mpi4py/setup-mpi@v1.0.3
+
    # Get Hypre through cache, or build it.
    # Install will only run on cache miss.
    - name: cache hypre
@@ -137,36 +151,67 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
+      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
        target: ${{ matrix.hypre-target }}
+        build-system: make
+
+    - name: get hypre (Windows)
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
+      uses: mfem/github-actions/build-hypre@v2.2
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}
+        build-system: cmake

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: install metis
-      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

+    - name: cache vcpkg (Windows)
+      id: vcpkg-cache
+      uses: actions/cache@v3
+      with:
+        path: vcpkg_cache
+        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
+
+    - name: prepare binary cache location
+      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p vcpkg_cache
+
+    - name: install metis (Windows)
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      env:
+        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+      run: |
+        vcpkg install metis --triplet=x64-windows-static
+
    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v2.1
+      uses: mfem/github-actions/build-mfem@v2.2
+      env:
+        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -176,6 +221,8 @@ jobs:
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
        mfem-dir: ${{ env.MFEM_TOP_DIR }}
+        config-options: ${{ env.MFEM_EXTRA_CONFIG }}
+        library-only: ${{ matrix.target == 'dbg' }}

    # Run checks (and only checks) on debug targets
    - name: checks
@@ -193,10 +240,27 @@ jobs:
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

-    - name: cmake unit tests
-      if: matrix.build-system == 'cmake'
+    - name: cmake checks
+      if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
      run: |
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure
+        CTEST_CONFIG="Debug"
+        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
+      shell: bash
+
+    - name: cmake unit tests (Ubuntu 20.04)
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
+      run: |
+        CTEST_CONFIG="Release"
+        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
+        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
+      shell: bash
+
+    - name: cmake tests
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
+      run: |
+        CTEST_CONFIG="Release"
+        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
+      shell: bash

    # Code coverage (process and upload reports)
    - name: codecov
@@ -20,6 +20,7 @@ on:
      - master
      - next
  pull_request:
+  workflow_dispatch:

 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -53,11 +54,11 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +69,18 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.0
+      uses: mfem/github-actions/build-mfem@v2.2
      with:
        os: ${{ runner.os }}
        target: opt
@@ -16,16 +16,28 @@ permissions:

 on:
  push:
+  pull_request:
+  workflow_dispatch:
+
+# This workflow is run on pushes to any branch in the MFEM repo (with or without
+# PRs), as well as on updates to PRs from forks. In particular, we do not
+# duplicate work by running on both pushes and updates to local PRs. We do that
+# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
+# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
+# github.repository").

 jobs:
  file-headers-check:
    runs-on: ubuntu-18.04
-
+    if: |
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.9.0
      with:
        access_token: ${{ github.token }}
+
    - name: checkout mfem
      uses: actions/checkout@v2

@@ -49,7 +61,10 @@ jobs:
      continue-on-error: true

    - name: wrap-up
-      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
+      if: |
+        steps.copyright.outcome != 'success' ||
+        steps.license.outcome != 'success' ||
+        steps.release.outcome != 'success'
      run: |
        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
          echo "copyright check failed, unroll log for details"
@@ -64,7 +79,9 @@ jobs:

  code-style:
    runs-on: ubuntu-18.04
-
+    if: |
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -79,7 +96,9 @@ jobs:

  documentation:
    runs-on: ubuntu-18.04
-
+    if: |
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -94,9 +113,12 @@ jobs:
        ./runtest documentation

  branch-history:
-    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
+    if: |
+      github.ref != 'refs/heads/next' &&
+      github.ref != 'refs/heads/master' &&
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    runs-on: ubuntu-18.04
-
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -240,6 +240,7 @@ miniapps/navier/navier_kovasznay_vs
 miniapps/navier/navier_tgv
 miniapps/navier/navier_shear
 miniapps/navier/navier_3dfoc
+miniapps/navier/navier_turbchan
 miniapps/navier/tgv_out*.txt
 miniapps/navier/*_output

@@ -260,12 +261,14 @@ miniapps/performance/sol.*

 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
+miniapps/shifted/ParaViewLSF
 miniapps/shifted/extrapolate
 miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
 miniapps/shifted/ParaViewDiffusion
+miniapps/shifted/lsf_integral

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -45,5 +45,5 @@ variables:
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
    # Next script uses 'THREADS': leaving it empty --> it uses 'make all -j'
-    - lalloc 1 -W 30 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - lalloc 1 -W 45 -q pdebug --atsdisable tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
  needs: [setup]
@@ -52,4 +52,4 @@ variables:
    - echo ${JOBID}
    - echo ${MFEM_DATA_DIR}
    - echo ${SPEC}
-    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 30 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
+    - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -t 45 -N 1 tests/gitlab/build_and_test --spec "${SPEC}" --data-dir "${MFEM_DATA_DIR}" --data
@@ -23,7 +23,7 @@ allocate_resource:
  stage: allocate_resource
  script:
    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=mi60 --time=30 --no-shell --job-name=${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --partition=mi60 --time=45 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h
  needs: [setup]

@@ -23,7 +23,7 @@ allocate_resource:
  stage: allocate_resource
  script:
    - echo ${ALLOC_NAME}
-    - salloc --exclusive --nodes=1 --partition=pdebug --time=30 --no-shell --job-name=${ALLOC_NAME}
+    - salloc --exclusive --nodes=1 --partition=pdebug --time=45 --no-shell --job-name=${ALLOC_NAME}
  timeout: 6h

 # GitLab jobs for the Quartz machine at LLNL
@@ -8,6 +8,42 @@
                               https://mfem.org


+Version 4.4.1 (development)
+===========================
+- Added example for body-fitted volumetric and shape integration using the 
+  Algoim library.
+
+- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
+  spatial Gaussian white noise.
+
+- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
+  See fem/estimators.hpp.
+
+- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
+  which provides parallel non-conforming, non-matching, variational, volumetric
+  mesh information transfer. With ParMortarAssember, fields can be exchanged
+  between arbitrarily distributed and unrelated finite element meshes in a
+  variationally consistent way.
+
+- Added full assembly and device support for several LinearForm integrators:
+  * DomainLF: (f, v)
+  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
+  * DomainLFGrad: (f, grad(v))
+  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
+
+- Add a new example code, Example 33/33p, to demonstrate the solution of
+  spectral fractional PDEs with MFEM.
+
+- Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
+
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration.
+
+- Added Windows 2022 CI testing with GitHub actions.
+
+- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
+
 Version 4.4, released on March 21, 2022
 =======================================

@@ -128,6 +164,9 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
  formatting. See the "make style" target.

+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -51,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.4.0)
+set(${PROJECT_NAME}_VERSION 4.4.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -268,7 +268,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
  if(APPLE)
    # On macOS, the compiler needs additional help to find the <omp.h> header.
    # See issue #2642 for more information.
-    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
+    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
  endif(APPLE)
 endif()

@@ -379,6 +379,21 @@ if (MFEM_USE_PUMI)
  endif()
 endif()

+# Moonolith
+if(MFEM_USE_MOONOLITH)
+  find_package(ParMoonolith REQUIRED)
+  if(ParMoonolith_FOUND)
+    get_target_property(
+      MOONOLITH_INCLUDE_DIRS ParMoonolith::par_moonolith
+      INTERFACE_INCLUDE_DIRECTORIES)
+    set(MOONOLITH_FOUND TRUE)
+    set(MOONOLITH_LIBRARIES ParMoonolith::par_moonolith)
+    message(
+      STATUS
+      "MOONOLITH_LIBRARIES=${MOONOLITH_LIBRARIES}, MOONOLITH_INCLUDE_DIRS=${MOONOLITH_INCLUDE_DIRS}")
+  endif()
+endif()
+
 # HiOp optimizer
 if (MFEM_USE_HIOP)
  find_package(HIOP REQUIRED)
@@ -416,6 +431,11 @@ if (MFEM_USE_CALIPER)
  find_package(Caliper REQUIRED)
 endif()

+# Algoim
+if (MFEM_USE_ALGOIM)
+  find_package(Algoim REQUIRED)
+endif()
+
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
@@ -459,7 +479,7 @@ set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE)
+    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -474,7 +494,6 @@ endforeach(TPL)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
-include_directories(${TPL_INCLUDE_DIRS})

 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
@@ -496,6 +515,11 @@ message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
 set(SOURCES "")
 set(HEADERS "")
 set(MFEM_SOURCE_DIRS general linalg mesh fem)
+
+if(MFEM_USE_MOONOLITH)
+  set(MFEM_SOURCE_DIRS ${MFEM_SOURCE_DIRS} fem/moonolith)
+endif()
+
 foreach(DIR IN LISTS MFEM_SOURCE_DIRS)
  add_subdirectory(${DIR})
 endforeach()
@@ -526,6 +550,15 @@ target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
  target_link_libraries(mfem PRIVATE ws2_32)
 endif()
+if (MSVC)
+  target_compile_options(mfem PUBLIC "/wd4819")
+endif()
+message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
+target_include_directories(mfem
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    ${TPL_INCLUDE_DIRS})
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")

@@ -654,6 +687,10 @@ set(INSTALL_LIB_DIR lib
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
  CACHE PATH "Relative path for installing cmake config files.")

+target_include_directories(mfem
+  PUBLIC
+    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
+
 # The 'install' target will not depend on 'all'.
 # set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)

@@ -687,7 +724,7 @@ endif()
 if (MFEM_USE_CEED)
  install(DIRECTORY ${MFEM_SOURCE_DIRS}
    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/*.h")
+    FILES_MATCHING PATTERN "fem/ceed/integrators/*/*.h")
 endif()

 # Install ${HEADERS}
@@ -718,14 +755,8 @@ export(TARGETS ${PROJECT_NAME}
 # TODO: How do we register the install-tree? Replacing the build-tree?
 export(PACKAGE ${PROJECT_NAME})

-# Extract the include directories required to use MFEM
-get_target_property(MFEM_TPL_INCLUDE_DIRS mfem INCLUDE_DIRECTORIES)
-if (NOT MFEM_TPL_INCLUDE_DIRS)
-  set(MFEM_TPL_INCLUDE_DIRS "")
-endif()
-
 # This is the build-tree version
-set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${MFEM_TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${PROJECT_BINARY_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}/MFEMConfig.cmake
@@ -733,7 +764,7 @@ configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR)

 # This is the version that will be installed
-set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${MFEM_TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${INSTALL_LIB_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/MFEMConfig.cmake
@@ -119,6 +119,7 @@ The MFEM source code has the following structure:
  │   ├── ceed
  │   ├── fe
  │   ├── qinterp
+  │   ├── moonolith
  │   └── tmop
  ├── general
  ├── linalg
@@ -471,6 +471,14 @@ MFEM_USE_CODIPACK = YES/NO
  Enable automatic differentiation using the CoDiPack library.
  www.scicomp.uni-kl.de/codi/

+MFEM_USE_ALGOIM = YES/NO
+  Enable the usage of Algoim - a collection of high-order accurate numerical 
+  methods and C++ algorithms for working with implicitly-defined geometry and 
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
+  provides interface to Algoim v1. Thus, to check out the specific state use: 
+  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
+  https://algoim.github.io
+
 MFEM_USE_ADFORWARD = YES/NO
  Enable forward mode for AD packages. This option is valid
  only if the AD package supports two modes (backward/forward).
@@ -520,6 +528,16 @@ MFEM_USE_MKL_CPARDISO = YES/NO
   MFEM_USE_LAPACK=YES, verify that the MKL LAPACK libraries are used. The
   OpenMP capabilities are disabled at link time.

+MFEM_USE_MOONOLITH = YES/NO
+   Enables the ParMoonolith interface for parallel non-conforming, non-matching,
+   variational, volumetric mesh information transfer. It requires the variable
+   MOONOLITH_DIR=<path to installation> to be defined in the environment in
+   order to be used with the Makefile. Makefile users are also required to
+   install moonolith using the command `make install_all`, see
+   https://bitbucket.org/zulianp/par_moonolith for details.
+   Although Moonolith is an MPI-based library, both serial (MFEM_USE_MPI=NO) and
+   parallel (MFEM_USE_MPI=YES) versions of MFEM are supported.
+
 MFEM_USE_CALIPER = YES/NO
   Enables the interface to Caliper. Caliper is a library to integrate
   performance profiling capabilities into applications. To use Caliper,
@@ -728,6 +746,22 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.7.

+- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
+  headers so it just needs to be downloaded at the same level as MFEM. Download
+  the specific version we use as:
+  "git clone https://github.com/algoim/algoim.git;
+   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
+  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
+  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
+  "ln -s blitz-1.0.2 blitz".
+  Build Blitz using CMake as:
+  "cmake . -DCMAKE_INSTALL_PREFIX=.; make lib; make install"
+  URL: https://github.com/blitzpp/blitz/archive/refs/tags/1.0.2.tar.gz
+  Options: BLITZ_OPT, BLITZ_LIB
+  Versions: BLITZ = 1.0.2
+
+
+
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
@@ -759,6 +793,11 @@ The specific libraries and their options are:
  Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
  Versions: RAJA >= 0.14.0.

+- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
+  URL: https://bitbucket.org/zulianp/par_moonolith
+  Options: MOONOLITH_DIR
+  Versions: MOONOLITH >= 1.1.0.
+
 - Caliper (optional), used when MFEM_USE_CALIPER = YES.
  URL: https://github.com/LLNL/Caliper
  Options: CALIPER_DIR
@@ -932,6 +971,7 @@ MFEM_USE_CEED
 MFEM_USE_RAJA
 MFEM_USE_UMPIRE
 MFEM_USE_SIDRE
+MFEM_USE_MOONOLITH
 MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
@@ -990,6 +1030,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - RAJA
 - UMPIRE
 - AXOM - Used when MFEM_USE_SIDRE is enabled
+ - MOONOLITH
 - CALIPER
 - FMS
 - BENCHMARK
@@ -7,21 +7,24 @@

                               https://mfem.org

-MFEM is a modular parallel C++ library for finite element methods. Its goal is
-to enable high-performance scalable finite element discretization research and
-application development on a wide variety of platforms, ranging from laptops to
-supercomputers.
+[MFEM](https://mfem.org) is a modular parallel C++ library for finite element
+methods. Its goal is to enable high-performance scalable finite element
+discretization research and application development on a wide variety of
+platforms, ranging from laptops to supercomputers.

 We welcome contributions and feedback from the community. Please see the file
-CONTRIBUTING.md for additional details about our development process.
+[CONTRIBUTING.md](CONTRIBUTING.md) for additional details about our development
+process.

-* For building instructions, see the file INSTALL, or type "make help".
+* For building instructions, see the file [INSTALL](INSTALL), or type "make help".

-* Copyright and licensing information can be found in files LICENSE and NOTICE.
+* Copyright and licensing information can be found in files [LICENSE](LICENSE) and [NOTICE](NOTICE).

 * The best starting point for new users interested in MFEM's features is to
  review the examples and miniapps at https://mfem.org/examples.

+* Instructions for learning with Docker are in [config/docker](config/docker).
+
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support
@@ -58,12 +61,16 @@ solvers from the hypre library. Comprehensive support for other external
 packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
 additional linear and nonlinear solvers, preconditioners, time integrators, etc.

-For examples of using MFEM, see the examples/ and miniapps/ directories, as well
-as the OpenGL visualization tool GLVis which is available at https://glvis.org.
+For examples of using MFEM, see the [examples/](examples) and [miniapps/](miniapps)
+directories, as well as the OpenGL visualization tool GLVis which is available
+at https://glvis.org.
+
+## License

 MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See LICENSE and NOTICE for details.
+must be made under this license. See [LICENSE](LICENSE) and [NOTICE](NOTICE) for
+details.

-SPDX-License-Identifier: BSD-3-Clause
-LLNL Release Number: LLNL-CODE-806117
+SPDX-License-Identifier: BSD-3-Clause <br>
+LLNL Release Number: LLNL-CODE-806117 <br>
 DOI: 10.11578/dc.20171025.1248
@@ -54,9 +54,11 @@ set(MFEM_USE_CEED @MFEM_USE_CEED@)
 set(MFEM_USE_UMPIRE @MFEM_USE_UMPIRE@)
 set(MFEM_USE_SIMD @MFEM_USE_SIMD@)
 set(MFEM_USE_ADIOS2 @MFEM_USE_ADIOS2@)
+set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
 set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
 set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
+set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)

@@ -128,6 +128,9 @@
 // Enable MFEM functionality based on the PUMI library
 #cmakedefine MFEM_USE_PUMI

+// Enable MFEM functionality based on the Moonolith library
+#cmakedefine MFEM_USE_MOONOLITH
+
 // Enable MFEM functionality based on the HiOp library
 #cmakedefine MFEM_USE_HIOP

@@ -157,6 +160,9 @@
 // Enable MFEM functionality based on the Caliper library
 #cmakedefine MFEM_USE_CALIPER

+// Enable MFEM functionality based on the Algoim library
+#cmakedefine MFEM_USE_ALGOIM
+
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - ALGOIM_FOUND
+#   - ALGOIM_LIBRARIES
+#   - ALGOIM_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(Algoim ALGOIM ALGOIM_DIR
+       	"include" "algoim_quad.hpp"
+       	"" ""
+	"Paths to headers required by Algoim."
+	"Libraries required by Algoim.")
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - BLITZ_FOUND
+#   - BLITZ_LIBRARIES
+#   - BLITZ_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(Blitz BLITZ BLITZ_DIR
+       	"include" "blitz/blitz.h"
+       	"lib" "blitz"
+  	"Paths to headers required by Blitz."
+       	"Libraries required by Blitz.")
@@ -15,5 +15,5 @@
 #   - GSLIB_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-mfem_find_package(gslib GSLIB GSLIB_DIR "include" gslib.h "lib" gs
+mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs
  "Paths to headers required by GSLIB." "Libraries required by GSLIB.")
@@ -893,7 +893,8 @@ function(mfem_export_mk_files)
      MFEM_USE_CONDUIT MFEM_USE_PUMI MFEM_USE_HIOP MFEM_USE_GSLIB MFEM_USE_CUDA
      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
-      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG)
+      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -977,9 +978,11 @@ function(mfem_export_mk_files)
      string(REGEX REPLACE "^SCOREC::" "" libname ${pumilib})
      string(FIND "${pumilib}" ".a" staticlib)
      string(FIND "${pumilib}" ".so" sharedlib)
+      string(FIND "${pumilib}" ".dylib" dynamiclib)
      find_library(lib ${libname} PATHS ${PUMI_DIR}/lib NO_DEFAULT_PATH)
      if (NOT "${sharedlib}" MATCHES "-1" OR
-          NOT "${staticlib}" MATCHES "-1"   )
+          NOT "${staticlib}" MATCHES "-1" OR
+          NOT "${dynamiclib}" MATCHES "-1"  )
        set(MFEM_EXT_LIBS "${pumilib} ${MFEM_EXT_LIBS}")
      elseif (NOT "${lib}" MATCHES "lib-NOTFOUND")
        set(MFEM_EXT_LIBS "${lib} ${MFEM_EXT_LIBS}")
@@ -994,7 +997,7 @@ function(mfem_export_mk_files)
  foreach(lib ${TPL_LIBRARIES})
    get_filename_component(suffix ${lib} EXT)
    # handle interfaces (e.g., SCOREC::apf)
-    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*")
+    if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
    elseif (TARGET "${lib}")
      mfem_get_target_options(${lib} CompileOpts LinkOpts)
      # Removing duplicates may lead to issues:
@@ -138,6 +138,9 @@
 // Enable MFEM functionality based on the PUMI library
 // #define MFEM_USE_PUMI

+// Enable Moonolith-based general interpolation between finite element spaces.
+// #define MFEM_USE_MOONOLITH
+
 // Enable MFEM functionality based on the HIOP library.
 // #define MFEM_USE_HIOP

@@ -164,6 +167,9 @@
 // Enable functionality based on the Caliper library.
 // #define MFEM_USE_CALIPER

+// Enable functionality based on the Algoim library.
+// #define MFEM_USE_ALGOIM
+
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE

@@ -58,6 +58,7 @@ MFEM_USE_UMPIRE        = @MFEM_USE_UMPIRE@
 MFEM_USE_SIMD          = @MFEM_USE_SIMD@
 MFEM_USE_ADIOS2        = @MFEM_USE_ADIOS2@
 MFEM_USE_MKL_CPARDISO  = @MFEM_USE_MKL_CPARDISO@
+MFEM_USE_MOONOLITH     = @MFEM_USE_MOONOLITH@
 MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
@@ -58,6 +58,7 @@ option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
+option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
 option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
@@ -236,6 +237,11 @@ set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
+set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
+set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
+set(ALGOIM_REQUIRED_PACKAGES "BLITZ" CACHE STRING
+    "Packages that ALGOIM depends on.")
+
 set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
    "Path to Google Benchmark")

@@ -153,10 +153,12 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_CALIPER       = NO
+MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
 MFEM_USE_MKL_CPARDISO  = NO
+MFEM_USE_MOONOLITH     = NO
 MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
@@ -387,6 +389,11 @@ ifeq ($(SLEPC_FOUND),YES)
      $(subst $(CXX_XLINKER),$(XLINKER),$(SLEPC_DEP))
 endif

+ifeq ($(MFEM_USE_MOONOLITH),YES)
+  include $(MOONOLITH_DIR)/config/moonolith-config.makefile
+  MOONOLITH_LIB=$(MOONOLITH_LIBRARIES)
+endif
+
 # MPFR library configuration
 MPFR_OPT =
 MPFR_LIB = -lmpfr
@@ -463,6 +470,16 @@ CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
 CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper

+# BLITZ library configuration
+BLITZ_DIR = @MFEM_DIR@/../blitz
+BLITZ_OPT = -I$(BLITZ_DIR)/include
+BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
+
+# ALGOIM library configuration
+ALGOIM_DIR = @MFEM_DIR@/../algoim
+ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT)
+ALGOIM_LIB = $(BLITZ_LIB)
+
 # BENCHMARK library configuration
 BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
 BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
@@ -0,0 +1,30 @@
+FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
+
+# docker build -t ghcr.io/mfem/mfem-ubuntu-base .
+
+RUN apt-get update && \
+    apt-get install -y unzip gfortran && \
+    spack compiler find && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev
+
+# /code is the working directory for code
+WORKDIR /code
+COPY . /code
+
+# This is for a spack environment/view to install from there
+WORKDIR /opt/mfem-env
+RUN . /opt/spack/share/spack/setup-env.sh && \
+    spack env create -d . && \
+    echo "  concretization: together" >> spack.yaml && \
+    spack env activate . && \
+    spack develop --path /code mfem@master+examples+miniapps && \
+    spack add mfem@master+examples+miniapps && \
+    spack install
+
+# ensure mfem always on various paths
+RUN cd /opt/mfem-env && \
+    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+# The user will see the view on shell into the container
+WORKDIR /opt/mfem-env/.spack-env/view/
+ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
@@ -0,0 +1,130 @@
+# mfem Docker
+
+We provide a [Dockerfile](Dockerfile) to build an ubuntu base image. You can use
+this image for a demo of using mfem! 🎉️
+
+Updated containers are built and deployed on merges to the main branch and releases.
+If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
+
+### Usage
+
+Here is how to build the container. Note that we build so it belongs to the same
+namespace as the repository here. "ghcr.io" means "GitHub Container Registry" and
+is the [GitHub packages](https://github.com/features/packages) registry that supports
+ Docker images and other OCI artifacts. From the root of the repository:
+
+```bash
+$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base .
+```
+
+or this directory:
+
+```bash
+$ docker build -f Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base ../../
+```
+
+### Shell
+
+To shell into a container (here is an example with ubuntu):
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
+```
+
+Off the bat, you can see mfem libraries are in your path so you can jump into development:
+
+```bash
+env | grep mfem
+```
+```bash
+PKG_CONFIG_PATH=/opt/mfem-env/.spack-env/view/lib/pkgconfig:/opt/mfem-env/.spack-env/view/share/pkgconfig:/opt/mfem-env/.spack-env/view/lib64/pkgconfig
+PWD=/opt/mfem-env
+MANPATH=/opt/mfem-env/.spack-env/view/share/man:/opt/mfem-env/.spack-env/view/man:
+CMAKE_PREFIX_PATH=/opt/mfem-env/.spack-env/view
+SPACK_ENV=/opt/mfem-env
+ACLOCAL_PATH=/opt/mfem-env/.spack-env/view/share/aclocal
+LD_LIBRARY_PATH=/opt/mfem-env/.spack-env/view/lib:/opt/mfem-env/.spack-env/view/lib64
+PATH=/opt/mfem-env/.spack-env/view/bin:/opt/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+```
+
+#### Examples and MiniApps
+
+If you want to develop a tool that _uses_ mfem, you can find the built libraries in:
+
+```
+$ ls /opt/mfem-env/.spack-env/view/
+bin  etc  include  lib  libexec  sbin  share  var
+```
+
+And yes, this is the working directory when you shell into the container!
+You can find the examples here:
+
+
+```bash
+cd share/mfem/examples
+```
+```bash
+$ ./ex0
+Options used:
+   --mesh ../data/star.mesh
+   --order 1
+Number of unknowns: 101
+   Iteration :   0  (B r, r) = 0.184259
+   Iteration :   1  (B r, r) = 0.102754
+   Iteration :   2  (B r, r) = 0.00558141
+   Iteration :   3  (B r, r) = 1.5247e-05
+   Iteration :   4  (B r, r) = 1.13807e-07
+   Iteration :   5  (B r, r) = 6.27231e-09
+   Iteration :   6  (B r, r) = 3.76268e-11
+   Iteration :   7  (B r, r) = 6.07423e-13
+   Iteration :   8  (B r, r) = 4.10615e-15
+Average reduction factor = 0.140201
+```
+
+Try running a few, and look at the associated .cpp file for the source code!
+You can also explore the "mini apps," also in share/mfem, but under miniapps.
+
+```bash
+# This is run from the examples directory
+$ cd ../miniapps
+```
+```bash
+$ ls
+CMakeLists.txt  common            meshing  nurbs        shifted  toys
+adjoint         electromagnetics  mtop     parelag      solvers
+autodiff        gslib             navier   performance  tools
+```
+
+And an example in "toys"
+
+```bash
+cd toys
+```
+```bash
+$ ./automata -no-vis
+Options used:
+   --num-steps 16
+   --rule 90
+   --no-visualization
+
+Rule:
+ 111 110 101 100 011 010 001 000
+  0   1   0   1   1   0   1   0
+
+Applying rule...done.
+```
+
+Have fun!
+
+
+#### Your own App
+If you want to develop with your own code base
+(and mfem as is in the container) you can bind to somewhere else in the container (e.g., src)
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
+```
+
+In the above, we can pretend your project is in the present working directory (PWD) and we are
+binding to source. You can then use the mfem in the container for development, and if you
+want to distribute your library or app in a container, you can use the mfem container as the base.
@@ -0,0 +1,56 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geomety Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+
+dimension
+3
+
+elements
+4
+1 5 0 1 4 3 6 7 10 9
+1 6 4 1 5 10 7 11
+1 7 11 7 1 5 8
+1 4 2 5 1 8
+
+boundary
+14
+1 3 0 3 4 1
+1 3 6 7 10 9
+1 3 0 6 9 3
+1 3 0 1 7 6
+1 3 3 9 10 4
+1 2 1 5 2
+1 2 1 4 5
+1 2 1 8 7
+1 2 1 2 8
+1 2 2 5 8
+1 2 5 11 8
+1 2 7 8 11
+1 2 7 11 10
+1 3 4 10 11 5
+
+vertices
+12
+3
+0 0 0
+1 0 0
+2 0 0
+0 1 0
+1 1 0
+2 1 0
+0 0 1
+1 0 1
+2 0 1
+0 1 1
+1 1 1
+2 1 1
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.

-PROJECT_NUMBER         = v4.4.0
+PROJECT_NUMBER         = v4.4.1

 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -765,11 +765,14 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                         @MFEM_SOURCE_DIR@/linalg \
                         @MFEM_SOURCE_DIR@/mesh \
                         @MFEM_SOURCE_DIR@/fem \
+                         @MFEM_SOURCE_DIR@/fem/moonolith \
                         @MFEM_SOURCE_DIR@/fem/fe \
+                         @MFEM_SOURCE_DIR@/fem/lor \
                         @MFEM_SOURCE_DIR@/examples \
                         @MFEM_SOURCE_DIR@/examples/caliper \
                         @MFEM_SOURCE_DIR@/examples/amgx \
                         @MFEM_SOURCE_DIR@/examples/ginkgo \
+                         @MFEM_SOURCE_DIR@/examples/moonolith \
                         @MFEM_SOURCE_DIR@/examples/hiop \
                         @MFEM_SOURCE_DIR@/examples/petsc \
                         @MFEM_SOURCE_DIR@/examples/pumi \
@@ -103,6 +103,8 @@ namespace mfem {
 * - <a class="el" href="ex31_8cpp_source.html">Example 31</a>: Nedelec H(curl) FEM for the definite anisotropic Maxwell problem
 * - <a class="el" href="ex31p_8cpp_source.html">Example 31p</a>: parallel Nedelec H(curl) FEM for the definite anisotropic Maxwell problem
 * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
+ * - <a class="el" href="ex33_8cpp_source.html">Example 33</a>: nodal H1 FEM for the fractional Laplacian problem
+ * - <a class="el" href="ex33p_8cpp_source.html">Example 33p</a>: parallel nodal H1 FEM for the fractional Laplacian problem
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -39,6 +39,7 @@ list(APPEND ALL_EXE_SRCS
  ex29.cpp
  ex30.cpp
  ex31.cpp
+  ex33.cpp
  )

 if (MFEM_USE_MPI)
@@ -75,6 +76,7 @@ if (MFEM_USE_MPI)
    ex30p.cpp
    ex31p.cpp
    ex32p.cpp
+    ex33p.cpp
    )
 endif()

@@ -199,3 +201,7 @@ endif()
 if (MFEM_USE_SUPERLU)
  add_subdirectory(superlu)
 endif()
+
+if(MFEM_USE_MOONOLITH)
+  add_subdirectory(moonolith)
+endif()
@@ -1,294 +0,0 @@
-//                 MFEM FOSLS acoustics Example
-//
-// Compile with: make fosls
-//
-// Definite/Indefinite Helmholtz
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// FOSLS:
-//       minimize  1/2(||∇p - ω u||^2 + ||-∇⋅u ± ω p - f||^2)
-
-// (p,u) ∈ H^1(Ω) × H(div,Ω)
-// -------------------------------------------------------------------
-// |   |            p           |            u             |   RHS    | 
-// -------------------------------------------------------------------
-// | q |  (∇ p,∇ q) + ω^2(p,q)  | ∓ ω (∇⋅u,q) - ω (u, ∇ q) | ± ω(f,q) |
-// |   |                        |                          |          |
-// | v | ∓ ω (p,∇⋅v) - ω (∇ p,v)|  (∇⋅u,∇⋅v) + ω^2 (u,v)   | -(f,∇⋅v) |
-
-// where (q,v) ∈ H^1(Ω) × H(div,Ω)
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int sr = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&sr, "-sr", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-   for (int i = 0; i < sr; i++ )
-   {
-      mesh.UniformRefinement();
-   }
-
-   FiniteElementCollection *H1fec = new H1_FECollection(order, dim);
-   FiniteElementCollection *RTfec = new RT_FECollection(order-1, dim);
-   FiniteElementSpace * H1fes = new FiniteElementSpace(&mesh, H1fec);
-   FiniteElementSpace * RTfes = new FiniteElementSpace(&mesh, RTfec);
-
-   Array<FiniteElementSpace *> fespaces(2);
-   fespaces[0] = H1fes;
-   fespaces[1] = RTfes;
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   BlockBilinearForm a(fespaces);
-   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
-   cout << "H1 fespace = " << H1fes->GetTrueVSize() << endl;
-   cout << "RT fespace = " << RTfes->GetTrueVSize() << endl;
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient negomeg(-omega);
-   ConstantCoefficient omeg2(omega*omega);
-
-
-   Array2D<BilinearFormIntegrator * > blfi(2,2);
-
-   // blfi(0,0) = (∇ p,∇ q) + ω^2(p,q)
-   SumIntegrator * integ00 = new SumIntegrator();
-   integ00->AddIntegrator(new DiffusionIntegrator(one));
-   integ00->AddIntegrator(new MassIntegrator(omeg2));
-   blfi(0,0) = integ00;
-
-   // blfi(0,1) = ∓ ω (∇⋅u,q) - ω (u, ∇ q)
-   SumIntegrator * integ01 = new SumIntegrator();
-#ifdef DEFINITE
-   // -ω (∇⋅u,q)
-   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(negomeg));
-#else   
-   // ω (∇⋅u,q)
-   integ01->AddIntegrator(new MixedScalarDivergenceIntegrator(omeg));
-#endif
-   // - ω (u, ∇ q)
-   integ01->AddIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg));
-   blfi(0,1) = integ01;
-
-   // blfi(1,0) = ∓ ω (p,∇⋅v) - ω (∇ p,v)
-   SumIntegrator * integ10 = new SumIntegrator();
-#ifdef DEFINITE
-   // - ω (p,∇⋅v)
-   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(omeg));
-#else
-   // ω (p,∇⋅v)
-   integ10->AddIntegrator(new MixedScalarWeakGradientIntegrator(negomeg));
-#endif
-   // - ω (∇ p,v)
-   integ10->AddIntegrator(new MixedVectorGradientIntegrator(negomeg));
-   blfi(1,0) = integ10;
-
-   // blfi(1,1) = (∇⋅u,∇⋅v) + ω^2 (u,v)
-   SumIntegrator * integ11 = new SumIntegrator();
-   integ11->AddIntegrator(new DivDivIntegrator(one));
-   integ11->AddIntegrator(new VectorFEMassIntegrator(omeg2));
-   blfi(1,1) = integ11;
-
-
-   BlockLinearForm b(fespaces);
-   Array<LinearFormIntegrator * > lfi(2);
-   // ± ω (f,q)
-   FunctionCoefficient f_rhs(rhs_func);
-#ifdef DEFINITE  
-   ProductCoefficient w_f(omeg,f_rhs);
-#else   
-   ProductCoefficient w_f(negomeg,f_rhs);
-#endif   
-   // lfi[0] = new DomainLFIntegrator(w_f);
-   lfi[0] = new DomainLFIntegrator(w_f);
-
-
-   //  -(f,∇⋅v)
-   ProductCoefficient neg_f(negone,f_rhs);
-   // lfi[1] = new VectorFEDomainLFDivIntegrator(f_rhs);
-   lfi[1] = new VectorFEDomainLFDivIntegrator(neg_f);
-
-   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
-   integ->SetIntegrators(blfi);
-   a.AddDomainIntegrator(integ);
-   a.Assemble();
-
-   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
-   lininteg->SetIntegrators(lfi);
-   b.AddDomainIntegrator(lininteg);
-   b.Assemble();
-
-   int size = 0;
-   for (int i = 0; i<fespaces.Size(); i++)
-   {
-      size += fespaces[i]->GetVSize();
-   }
-
-   Vector x(size);
-   x = 0.0;
-   FunctionCoefficient p_ex(p_exact);
-   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
-   VectorFunctionCoefficient u_ex(dim,u_exact);
-   FunctionCoefficient divu_ex(divu_exact);
-   GridFunction p_gf, u_gf;   
-   GridFunction pex_gf(H1fes);   
-
-   p_gf.MakeRef(H1fes,x,0);
-   // p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
-   p_gf.ProjectCoefficient(p_ex);
-   pex_gf.ProjectCoefficient(p_ex);
-
-   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
-   u_gf = 0.;
-
-
-   OperatorPtr A;
-   Vector X,B;
-   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
-
-   GSSmoother M((SparseMatrix&)(*A));
-   CGSolver cg;
-   cg.SetRelTol(1e-10);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   a.RecoverFEMSolution(X,b,x);
-
-   p_gf.MakeRef(H1fes,x,0);
-   u_gf.MakeRef(RTfes,x,H1fes->GetVSize());
-
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << p_gf <<
-                "window_title 'Numerical p' "
-                << flush;
-      // socketstream sols_sock(vishost, visport);
-      // sols_sock.precision(8);
-      // sols_sock << "solution\n" << mesh << u_gf <<
-               //  "window_title 'Numerical sigma' "
-               //  << flush;
-
-      socketstream solex_sock(vishost, visport);
-      solex_sock.precision(8);
-      solex_sock << "solution\n" << mesh << pex_gf <<
-                    "window_title 'Exact p' "
-                    << flush;          
-   }
-
-   return 0;
-}
-
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/acoustics,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = fosls uw_dpg  strong_dpg
-PAR_EXAMPLES = uw_dpgp
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
@@ -1,271 +0,0 @@
-//                 MFEM DPG_strong acoustics Example
-//
-// Compile with: make strong_dpg
-//
-// Definite/Indefinite Helmholtz
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// Strong DPG formulation
-// (p,u) ∈ H^1(Ω) × H(div,Ω)
-// 
-//   (∇ p, v) - ω (u,v) = 0,     in Ω, ∀ v ∈ (L^2)^dim
-//  -(∇⋅u, q) ± ω (p,q) = (f,q), in Ω, ∀ q ∈  L^2 
-//                    p = p_0, in ∂Ω
-// 
-// ------------------------------------
-// |   |      p    |     u    |  RHS  | 
-// ------------------------------------
-// | q | ± ω (p,q) | -(∇⋅u,q) | (f,q) |
-// |   |           |          |       |
-// | v | (∇ p, v)  | -ω (u,v) |       |
-
-// where (q,v) ∈ L^2  × (L^2)^dim
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");                  
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   for (int i = 0; i < ref; i++ )
-   {
-      mesh.UniformRefinement();
-   }
-
-   // Define spaces
-   // H1 space for p
-   FiniteElementCollection *p_fec = new H1_FECollection(order, dim);
-   FiniteElementSpace * p_fes = new FiniteElementSpace(&mesh, p_fec);
-
-   // H(div) for u
-   FiniteElementCollection *u_fec = new RT_FECollection(order-1, dim);
-   FiniteElementSpace * u_fes = new FiniteElementSpace(&mesh, u_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new L2_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new L2_FECollection(test_order-1, dim);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->SetTestFECollVdim(1,dim);
-   
-   a->StoreMatrices(true);
-
-//  ± ω (p, q)
-#ifdef DEFINITE
-//  ω (p, q)
-   a->AddTrialIntegrator(new MassIntegrator(omeg),0,0);
-#else
-// -ω (p, q)
-   a->AddTrialIntegrator(new MassIntegrator(negomeg),0,0);
-#endif
-
-// -(∇⋅u, q) 
-   a->AddTrialIntegrator(new MixedScalarDivergenceIntegrator(negone),1,0);
-
-// -ω (u,v)
-   a->AddTrialIntegrator(new VectorFEMassIntegrator(negomeg),1,1);
-
-// (∇ p, v)
-   a->AddTrialIntegrator(new GradientIntegrator(one),0,1);
-
-// (v,δv)
-   a->AddTestIntegrator(new VectorMassIntegrator(one),1,1);
-    
-// (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-   Array<int> ess_tdof_list;
-   Array<int> ess_bdr;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      p_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   FunctionCoefficient p_ex(p_exact);
-   VectorFunctionCoefficient gradp_ex(dim,gradp_exact);
-   VectorFunctionCoefficient u_ex(dim,u_exact);
-   FunctionCoefficient divu_ex(divu_exact);
-   GridFunction p_gf, u_gf;   
-   GridFunction pex_gf(p_fes);   
-
-   Array<int> offsets(3);
-   offsets[0] = 0;
-   offsets[1] = p_fes->GetVSize();
-   offsets[2] = u_fes->GetVSize();
-   offsets.PartialSum();
-   BlockVector x(offsets);
-   x = 0.0;
-
-   p_gf.MakeRef(p_fes,x.GetBlock(0));
-   p_gf.ProjectBdrCoefficient(p_ex,ess_bdr);
-
-   u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-   a->Assemble();
-
-   OperatorPtr Ah;
-   Vector X,B;
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = Ah.As<BlockMatrix>();
-
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-   delete M;
-
-   a->RecoverFEMSolution(X,x);
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream p_out;
-      socketstream u_out;
-      p_out.open(vishost, visport);
-      u_out.open(vishost, visport);
-      p_out.precision(8);
-      p_out << "solution\n" << mesh << p_gf <<
-               "window_title 'Numerical p' "
-               << flush;
-
-      u_out.precision(8);
-      u_out << "solution\n" << mesh << u_gf <<
-            "window_title 'Numerical flux' "
-            << flush;
-   }
-
-   return 0;
-}
-
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
@@ -1,546 +0,0 @@
-//                   MFEM Ultraweak DPG acoustics example
-//
-// Compile with: make uw_dpg
-//
-// ./uw_dpg -m ../../../data/inline-quad.mesh -rnum 40 -theta 0.7 -prob 1 -graph-norm -ref 40 -o 3
-
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂ = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := -u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p);
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-double divu_exact(const Vector &x);
-double hatp_exact(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-enum prob_type
-{
-   plane_wave,
-   gaussian_beam  
-};
-
-prob_type prob;
-
-int main(int argc, char *argv[])
-{
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-   int iprob = 0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");    
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: plane wave, 1: Gaussian beam");                                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");                               
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 1) { iprob = 0; }
-   prob = (prob_type)iprob;
-
-
-   omega = 2.0 * M_PI * rnum;
-
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *p_fes = new FiniteElementSpace(&mesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatp_fes = new FiniteElementSpace(&mesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-
-   // ± ω (p,q)
-#ifdef DEFINITE
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
-#else
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
-#endif   
-
-// (u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
-
-// - ω (u,v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-// test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -ω (∇q,δv)
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
-      // -ω (v,δq)
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
-
-#ifdef DEFINITE
-      // - ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // - ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
-#else
-      // ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
-      // ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-#endif
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-
-   FunctionCoefficient hatpex(hatp_exact);
-   FunctionCoefficient pex(p_exact);
-   VectorFunctionCoefficient uex(dim,u_exact);
-   Array<int> elements_to_refine;
-   GridFunction hatp_gf;
-
-
-   socketstream p_out;
-   // socketstream u_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out.open(vishost, visport);
-      // u_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-   // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
-      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      for (int i=0; i<A->NumRowBlocks(); i++)
-      {
-         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-      }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-8);
-      cg.SetMaxIter(20000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-      
-      GridFunction p_gf;
-      p_gf.MakeRef(p_fes,x.GetBlock(0));
-
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-      GridFunction pex_gf(p_fes);
-      GridFunction uex_gf(u_fes);
-      pex_gf.ProjectCoefficient(pex);
-      uex_gf.ProjectCoefficient(uex);
-
-
-      // Error
-      int dofs = X.Size();
-      double p_err = p_gf.ComputeL2Error(pex);
-      double p_norm = uex_gf.ComputeL2Error(zero);
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = u_gf.ComputeL2Error(vzero);
-
-      double L2Error = sqrt(p_err*p_err + u_err*u_err);
-      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = residual;
-      dof0 = dofs;
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::endl;
-
-      if (visualization)
-      {
-         p_out.precision(8);
-         p_out << "solution\n" << mesh << p_gf <<
-                  "window_title 'Numerical presure' "
-                  << flush;
-
-         // u_out.precision(8);
-         // u_out << "solution\n" << mesh << u_gf <<
-         //       "window_title 'Numerical velocity' "
-         //       << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   double p, d2p;
-   Vector dp;
-   acoustics_solution(x,p,dp,d2p);
-   return p;
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   double p, d2p;
-   acoustics_solution(x,p,u,d2p);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   double p, d2p;
-   Vector dp;
-   acoustics_solution(x,p,dp,d2p);
-   return d2p/omega;
-}
-
-double hatp_exact(const Vector & X)
-{
-   return p_exact(X);
-}
-
-void hatu_exact(const Vector & X, Vector & hatu)
-{
-   u_exact(X,hatu);
-   hatu *= -1.;
-}
-
-void acoustics_solution(const Vector & X, double & p, Vector & dp, double & d2p)
-{
-   dp.SetSize(X.Size());
-   switch (prob)
-   {
-   case plane_wave:
-   {
-      p = sin(omega*X.Sum());
-      dp = omega * cos(omega * X.Sum());
-      d2p = -dim * omega * omega * sin(omega*X.Sum());
-   }
-      break;
-   default:
-   {
-      double rk = omega;
-      double alpha = 45 * M_PI/180.;
-      double sina = sin(alpha); 
-      double cosa = cos(alpha);
-      // shift the origin
-      double xprim=X(0) + 0.1; 
-      double yprim=X(1) + 0.1;
-
-      double  x = xprim*sina - yprim*cosa;
-      double  y = xprim*cosa + yprim*sina;
-      double  dxdxprim = sina, dxdyprim = -cosa;
-      double  dydxprim = cosa, dydyprim =  sina;
-      //wavelength
-      double rl = 2.*M_PI/rk;
-
-      // beam waist radius
-      double w0 = 0.05;
-
-      // function w
-      double fact = rl/M_PI/(w0*w0);
-      double aux = 1. + (fact*y)*(fact*y);
-
-      double w = w0*sqrt(aux);
-      double dwdy = w0*fact*fact*y/sqrt(aux);
-      double d2wdydy = w0*fact*fact*(1. - (fact*y)*(fact*y)/aux)/sqrt(aux);
-
-      double phi0 = atan(fact*y);
-      double dphi0dy = cos(phi0)*cos(phi0)*fact;
-      double d2phi0dydy = -2.*cos(phi0)*sin(phi0)*fact*dphi0dy;
-
-      double r = y + 1./y/(fact*fact);
-      double drdy = 1. - 1./(y*y)/(fact*fact);
-      double d2rdydy = 2./(y*y*y)/(fact*fact);
-
-      // pressure
-      complex<double> zi = complex<double>(0., 1.);
-      complex<double> ze = - x*x/(w*w) - zi*rk*y - zi * M_PI * x * x/rl/r + zi*phi0/2.;
-
-      complex<double> zdedx = -2.*x/(w*w) - 2.*zi*M_PI*x/rl/r;
-      complex<double> zdedy = 2.*x*x/(w*w*w)*dwdy - zi*rk + zi*M_PI*x*x/rl/(r*r)*drdy + zi*dphi0dy/2.;
-      complex<double> zd2edxdx = -2./(w*w) - 2.*zi*M_PI/rl/r;
-      complex<double> zd2edxdy = 4.*x/(w*w*w)*dwdy + 2.*zi*M_PI*x/rl/(r*r)*drdy;
-      complex<double> zd2edydx = zd2edxdy;
-      complex<double> zd2edydy = -6.*x*x/(w*w*w*w)*dwdy*dwdy + 2.*x*x/(w*w*w)*d2wdydy - 2.*zi*M_PI*x*x/rl/(r*r*r)*drdy*drdy
-                              + zi*M_PI*x*x/rl/(r*r)*d2rdydy + zi/2.*d2phi0dydy;
-
-      double pf = pow(2.0/M_PI/(w*w),0.25);
-      double dpfdy = -pow(2./M_PI/(w*w),-0.75)/M_PI/(w*w*w)*dwdy;
-      double d2pfdydy = -1./M_PI*pow(2./M_PI,-0.75)*(-1.5*pow(w,-2.5)
-                        *dwdy*dwdy + pow(w,-1.5)*d2wdydy);
-
-
-      complex<double> zp = pf*exp(ze);
-      complex<double> zdpdx = zp*zdedx;
-      complex<double> zdpdy = dpfdy*exp(ze)+zp*zdedy;
-      complex<double> zd2pdxdx = zdpdx*zdedx + zp*zd2edxdx;
-      complex<double> zd2pdxdy = zdpdy*zdedx + zp*zd2edxdy;
-      complex<double> zd2pdydx = dpfdy*exp(ze)*zdedx + zdpdx*zdedy + zp*zd2edydx;
-      complex<double> zd2pdydy = d2pfdydy*exp(ze) + dpfdy*exp(ze)*zdedy + zdpdy*zdedy + zp*zd2edydy;
-
-      p = zp.real();
-      dp[0] = (zdpdx*dxdxprim + zdpdy*dydxprim).real();
-      dp[1] = (zdpdx*dxdyprim + zdpdy*dydyprim).real();
-
-      d2p = (  (zd2pdxdx*dxdxprim + zd2pdydx*dydxprim)*dxdxprim + (zd2pdxdy*dxdxprim + zd2pdydy*dydxprim)*dydxprim
-            + (zd2pdxdx*dxdyprim + zd2pdydx*dydyprim)*dxdyprim + (zd2pdxdy*dxdyprim + zd2pdydy*dydyprim)*dydyprim ).real();
-   }
-   break;
-   }
-
-}
@@ -1,507 +0,0 @@
-//                   MFEM Ultraweak DPG MPI acoustics (Helmholtz) example
-//
-// Compile with: make uw_dpgp
-//
-//     - Δ p ± ω^2 p = f̃ ,   in Ω
-//                 p = p_0, on ∂Ω
-//
-// First Order System
-
-//   ∇ p - ω u = 0, in Ω
-// - ∇⋅u ± ω p = f, in Ω
-//           p = p_0, in ∂Ω
-// where f:=f̃/ω 
-// 
-// UW-DPG:
-// 
-// p ∈ L^2(Ω), u ∈ (L^2(Ω))^dim 
-// p̂ ∈ H^1/2(Ω), û ∈ H^-1/2(Ω)  
-// -(p,  ∇⋅v) - ω (u , v) + < p̂, v⋅n> = 0,      ∀ v ∈ H(div,Ω)      
-//  (u , ∇ q) ± ω (p , q) + < û, q >  = (f,q)   ∀ q ∈ H^1(Ω)
-//                                  p̂ = p_0     on ∂Ω 
-
-// Note: 
-// p̂ := p on Γ_h (skeleton)
-// û := -u on Γ_h  
-
-// -------------------------------------------------------------
-// |   |     p     |     u     |    p̂      |    û    |  RHS    |
-// -------------------------------------------------------------
-// | v | -(p, ∇⋅v) | - ω (u,v) | < p̂, v⋅n> |         |         |
-// |   |           |           |           |         |         |
-// | q | ± ω (p,q) | (u , ∇ q) |           | < û,q > |  (f,q)  |  
-
-// where (q,v) ∈  H^1(Ω) × H(div,Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// #define DEFINITE
-
-double p_exact(const Vector &x);
-void u_exact(const Vector &x, Vector & u);
-double rhs_func(const Vector &x);
-void gradp_exact(const Vector &x, Vector &gradu);
-double divu_exact(const Vector &x);
-double d2_exact(const Vector &x);
-double hatp_exact(const Vector & X);
-void hatu_exact(const Vector & X, Vector & hatu);
-
-int dim;
-double omega;
-
-int main(int argc, char *argv[])
-{
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   bool visualization = true;
-   double rnum=1.0;
-   int ref = 1;
-   double theta = 0.0;
-   bool adjoint_graph_norm = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&rnum, "-rnum", "--number_of_wavelenths",
-                  "Number of wavelengths");      
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                    
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");                                
-   args.AddOption(&ref, "-ref", "--serial_ref",
-                  "Number of serial refinements.");  
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   omega = 2.0 * M_PI * rnum;
-
-   Mesh mesh(mesh_file, 1, 1);
-   dim = mesh.Dimension();
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for p
-   FiniteElementCollection *p_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *p_fes = new ParFiniteElementSpace(&pmesh,p_fec);
-
-   // Vector L2 space for u 
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec, dim); 
-
-   // H^1/2 space for p̂  
-   FiniteElementCollection * hatp_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatp_fes = new ParFiniteElementSpace(&pmesh,hatp_fec);
-
-   // H^-1/2 space for û  
-   FiniteElementCollection * hatu_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * q_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * v_fec = new RT_FECollection(test_order-1, dim);
-
-
-   Array<ParFiniteElementSpace * > trial_fes; 
-   trial_fes.Append(p_fes);
-   trial_fes.Append(u_fes);
-   trial_fes.Append(hatp_fes);
-   trial_fes.Append(hatu_fes);
-
-   Array<FiniteElementCollection * > test_fec; 
-   test_fec.Append(q_fec);
-   test_fec.Append(v_fec);
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient omeg(omega);
-   ConstantCoefficient omeg2(omega*omega);
-   ConstantCoefficient negomeg(-omega);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-
-   // Integrators
-
-   // ± ω (p,q)
-#ifdef DEFINITE
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(omeg),0,0);
-#else
-   a->AddTrialIntegrator(new MixedScalarMassIntegrator(negomeg),0,0);
-#endif   
-
-// (u , ∇ q)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-// -(p, ∇⋅v)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,1);
-
-// - ω (u,v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negomeg)),1,1);
-
-// < p̂, v⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-// < û,q >
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   // test integrators 
-
-   //space-induced norm for H(div) × H1
-   // (∇q,∇δq)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-   // (q,δq)
-   a->AddTestIntegrator(new MassIntegrator(one),0,0);
-   // (∇⋅v,∇⋅δv)
-   a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-
-   // additional integrators for the adjoint graph norm
-   if (adjoint_graph_norm)
-   {   
-      // -ω (∇q,δv)
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negomeg),0,1);
-      // -ω (v,δq)
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(omeg),1,0);
-      // ω^2 (v,δv)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(omeg2),1,1);
-
-#ifdef DEFINITE
-      // - ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(negomeg),1,0);
-      // - ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(omeg),0,1);
-#else
-      // ω (∇⋅v,δq)   
-      a->AddTestIntegrator(new VectorFEDivergenceIntegrator(omeg),1,0);
-      // ω (q,∇⋅v)   
-      a->AddTestIntegrator(new MixedScalarWeakGradientIntegrator(negomeg),0,1);
-#endif
-      // ω^2 (q,δq)
-      a->AddTestIntegrator(new MassIntegrator(omeg2),0,0);
-   }   
-
-   // RHS
-   FunctionCoefficient f_rhs(rhs_func);
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f_rhs),0);
-
-
-   FunctionCoefficient hatpex(hatp_exact);
-   FunctionCoefficient pex(p_exact);
-   VectorFunctionCoefficient uex(dim,u_exact);
-   Array<int> elements_to_refine;
-   ParGridFunction hatp_gf;
-
-
-
-
-   socketstream p_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      p_out.open(vishost, visport);
-   }
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   if (myid == 0)
-   {
-      mfem::out << " Refinement |" 
-               << "    Dofs    |" 
-               << "  L2 Error  |" 
-               << " Relative % |" 
-               << "  Rate  |" 
-               << "  Residual  |" 
-               << "  Rate  |" << endl;
-      mfem::out << " --------------------"      
-               <<  "-------------------"    
-               <<  "-------------------"    
-               <<  "-------------------" << endl;   
-   }
-
-
-   for (int i = 0; i<ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatp_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += p_fes->GetTrueVSize() + u_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = p_fes->GetVSize();
-      offsets[2] = u_fes->GetVSize();
-      offsets[3] = hatp_fes->GetVSize();
-      offsets[4] = hatu_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatp_gf.MakeRef(hatp_fes,x.GetBlock(2));
-      hatp_gf.ProjectBdrCoefficient(hatpex,ess_bdr);
-
-      Vector X,B;
-      OperatorPtr Ah;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-
-      HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-      HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(2,2));
-      amg0->SetPrintLevel(0);
-      amg1->SetPrintLevel(0);
-      amg2->SetPrintLevel(0);
-      amg0->SetRelaxType(16);
-      amg1->SetRelaxType(16);
-      amg2->SetRelaxType(16);
-
-      M->SetDiagonalBlock(0,amg0);
-      M->SetDiagonalBlock(1,amg1);
-      M->SetDiagonalBlock(2,amg2);
-      // for (int i = 0; i < 3; i++)
-      // {
-      //    MUMPSSolver * mumps = new MUMPSSolver;
-      //    mumps->SetOperator(A->GetBlock(i,i));
-      //    M->SetDiagonalBlock(i,mumps);
-      // }
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(3,3), hatu_fes);
-      }
-      M->SetDiagonalBlock(3,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-10);
-      cg.SetMaxIter(20000);
-      cg.SetPrintLevel(-1);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      
-      double maxresidual = residuals.Max(); 
-      double globalresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      globalresidual = sqrt(globalresidual);
-
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-
-      ParGridFunction p_gf;
-      p_gf.MakeRef(p_fes,x.GetBlock(0));
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(1));
-
-
-      ParGridFunction pex_gf(p_fes);
-      ParGridFunction uex_gf(u_fes);
-      pex_gf.ProjectCoefficient(pex);
-      uex_gf.ProjectCoefficient(uex);
-
-      int dofs = p_fes->GlobalTrueVSize()
-               + u_fes->GlobalTrueVSize()
-               + hatp_fes->GlobalTrueVSize()
-               + hatu_fes->GlobalTrueVSize();
-
-      double p_err = p_gf.ComputeL2Error(pex);
-      double p_norm = pex_gf.ComputeL2Error(zero);
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = uex_gf.ComputeL2Error(vzero);
-
-      double L2Error = sqrt(p_err*p_err + u_err*u_err);
-      double L2norm = sqrt(p_norm * p_norm + u_norm * u_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/globalresidual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = globalresidual;
-      dof0 = dofs;
-
-         std::ios oldState(nullptr);
-      if (myid == 0)
-      {
-         mfem::out << std::right << std::setw(11) << i << " | " 
-                   << std::setw(10) <<  dof0 << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::scientific <<  err0 << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                   << std::setprecision(2) 
-                   << std::setw(6) << std::fixed << rate_err << " | " 
-                   << std::setprecision(3) 
-                   << std::setw(10) << std::scientific <<  res0 << " | " 
-                   << std::setprecision(2) 
-                   << std::setw(6) << std::fixed << rate_res << " | " 
-                   << std::setprecision(5) 
-                   << std::scientific 
-                   << std::endl;
-      }   
-
-      if (visualization)
-      {
-         p_out << "parallel " << num_procs << " " << myid << "\n";
-         p_out.precision(8);
-         p_out << "solution\n" << pmesh << p_gf <<
-                  "window_title 'Numerical pressure' "
-                  << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      pmesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-   }
-
-   delete a;
-   delete q_fec;
-   delete v_fec;
-   delete hatp_fes;
-   delete hatp_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete u_fec;
-   delete p_fec;
-   delete u_fes;
-   delete p_fes;
-
-   return 0;
-}
-
-double rhs_func(const Vector &x)
-{
-   double p = p_exact(x);
-   double divu = divu_exact(x);
-   // f = - ∇⋅u ± ω p, 
-#ifdef DEFINITE   
-   return -divu + omega * p;
-#else
-   return -divu - omega * p;
-#endif   
-}
-
-double p_exact(const Vector &x)
-{
-   return sin(omega*x.Sum());
-}
-
-void gradp_exact(const Vector &x, Vector &grad)
-{
-   grad.SetSize(x.Size());
-   grad = omega * cos(omega * x.Sum());
-}
-
-void u_exact(const Vector &x, Vector & u)
-{
-   gradp_exact(x,u);
-   u *= 1./omega;
-}
-
-double divu_exact(const Vector &x)
-{
-   return d2_exact(x)/omega;
-}
-
-double d2_exact(const Vector &x)
-{
-   return -dim * omega * omega * sin(omega*x.Sum());
-}
-
-double hatp_exact(const Vector & X)
-{
-   return p_exact(X);
-}
-
-void hatu_exact(const Vector & X, Vector & hatu)
-{
-   u_exact(X,hatu);
-   hatu *= -1.;
-}
@@ -1,59 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/convection-diffusion,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = uw_dpg
-PAR_EXAMPLES = uw_dpgp
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-
-clean-exec:
@@ -1,649 +0,0 @@
-//                     MFEM Ultraweak DPG example
-//
-// Compile with: make uw_dpg
-//
-// sample runs 
-// ./uw_dpg  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
-//     - εΔu + ∇⋅(βu) = f,   in Ω
-//                  u = u_0, on ∂Ω
-
-// First Order System
-
-//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
-//        1/ε σ - ∇u  = 0,   in Ω
-//                  u = u_0, on ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, σ̂ ∈ H^-1/2  
-// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
-//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
-//                                        û = u_0  on ∂Ω 
-
-// Note: 
-// f̂ := βu - σ
-// û := -u
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
-// -------------------------------------------------------------
-// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
-// |   |           |           |           |          |         |
-// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
-
-// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   polynomial,
-   EJ,   
-   general  
-};
-
-enum test_norm_type
-{
-   standard,   
-   adjoint_graph,
-   robust  
-};
-
-prob_type prob;
-test_norm_type test_norm;
-Vector beta;
-double epsilon;
-// Function returns the solution u, and gradient du and the Laplacian d2u
-void solution(const Vector & x, double & u, Vector & du, double & d2u);
-double exact_u(const Vector & X);
-void exact_sigma(const Vector & X, Vector & sigma);
-double exact_hatu(const Vector & X);
-void exact_hatf(const Vector & X, Vector & hatf);
-double f_exact(const Vector & X);
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool visualization = true;
-   int iprob = 0;
-   int itest_norm = 0;
-   double theta = 0.7;
-   epsilon = 1e0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&epsilon, "-eps", "--epsilon",
-                  "Epsilon coefficient");               
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");        
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                             
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: polynomial, 1: EJ ,2: General");           
-   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
-                  " 0: Standard, 1: Adjoint Graph, 2: Robust");       
-   args.AddOption(&beta, "-beta", "--beta",
-                  "Vector Coefficient beta");                                              
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 2) { iprob = 2; }
-   prob = (prob_type)iprob;
-   test_norm = (test_norm_type)itest_norm;
-
-   if (prob == prob_type::EJ)
-   {
-      mesh_file = "../../../data/inline-quad.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   if (beta.Size() == 0)
-   {
-      beta.SetSize(dim);
-      beta[0] = 1.;
-      beta[1] = 0.;
-   }
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatf_fes = new FiniteElementSpace(&mesh,hatf_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient eps(epsilon);
-   ConstantCoefficient eps1(1./epsilon);
-   ConstantCoefficient negeps1(-1./epsilon);
-   ConstantCoefficient eps2(1/(epsilon*epsilon));
-
-   ConstantCoefficient negeps(-epsilon);
-   VectorConstantCoefficient betacoeff(beta);
-   Vector negbeta = beta;
-   negbeta.Neg();
-
-   ConstantCoefficient zero(0.0);
-   Vector vec0(dim); vec0 = 0.;
-   VectorConstantCoefficient vzero(vec0);
-
-
-   DenseMatrix bbt(beta.Size());
-   MultVVt(beta, bbt); 
-   MatrixConstantCoefficient bbtcoeff(bbt);
-
-
-   VectorConstantCoefficient negbetacoeff(negbeta);
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatf_fes);
-   test_fec.Append(v_fec);
-   test_fec.Append(tau_fec);
-
-
-   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
-   FiniteElementSpace *coeff_fes = new FiniteElementSpace(&mesh,coeff_fec);
-   GridFunction c1_gf, c2_gf;
-   GridFunctionCoefficient c1_coeff(&c1_gf);
-   GridFunctionCoefficient c2_coeff(&c2_gf);
-
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //-(βu , ∇v)
-   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
-   
-   // (σ,∇ v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-   // (u ,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
-
-   // 1/ε (σ,τ)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-   // <f̂ ,v> 
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   switch (test_norm)
-   {
-      case standard:
-      {   
-         // (∇v,∇δv)
-         mfem::out << "\n Test norm: Standard" << endl;
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-      }
-      break;
-      case adjoint_graph:
-      {
-         mfem::out << "\n Test norm: Adjoint Graph" << endl;
-         // (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-         // 1/ε^2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
-         // 1/ε (∇v, δτ)
-         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
-         // - (β ⋅ ∇v,∇⋅δτ)  
-         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
-         // 1/ε (τ,∇δv) 
-         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
-         // -(β ∇⋅τ ,∇⋅δv)  
-         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
-      }
-      break;
-      default:
-      {
-         mfem::out << "\n Test norm: Robust" << endl;
-         c1_gf.SetSpace(coeff_fes);
-         c2_gf.SetSpace(coeff_fes);
-         Array<int> dofs;
-         for (int i =0; i < mesh.GetNE(); i++)
-         {
-            double volume = mesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            // double c2 = 1.;
-            coeff_fes->GetElementDofs(i,dofs);
-            c1_gf.SetSubVector(dofs,c1);
-            c2_gf.SetSubVector(dofs,c2);
-         }
-         // c1 (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
-         // ε (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // c2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-      }
-      break;
-   }
-
-
-   FunctionCoefficient f(f_exact);
-   // if (prob != prob_type::EJ)
-   // {
-      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
-   // }
-
-   FunctionCoefficient hatuex(exact_hatu);
-   VectorFunctionCoefficient hatfex(dim,exact_hatf);
-   Array<int> elements_to_refine;
-   FunctionCoefficient uex(exact_u);
-   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
-   GridFunction hatu_gf;
-   GridFunction hatf_gf;
-
-   // socketstream uex_out;
-   socketstream u_out;
-   // socketstream sigma_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      // uex_out.open(vishost, visport);
-      // sigma_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   mfem::out << " Refinement |" 
-             << "    Dofs    |" 
-             << "  L2 Error  |" 
-             << " Relative % |" 
-             << "  Rate  |" 
-             << "  Residual  |" 
-             << "  Rate  |" << endl;
-   mfem::out << " --------------------"      
-             <<  "-------------------"    
-             <<  "-------------------"    
-             <<  "-------------------" << endl;   
-
-
-   for (int i = 0; i<=ref; i++)
-   {
-      a->Assemble();
-
-      Array<int> ess_tdof_list_uhat;
-      Array<int> ess_tdof_list_fhat;
-      Array<int> ess_bdr_uhat;
-      Array<int> ess_bdr_fhat;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr_uhat.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr_fhat.SetSize(mesh.bdr_attributes.Max());
-         // ess_bdr_uhat = 1;
-         // ess_bdr_fhat = 0;
-         ess_bdr_uhat = 0;
-         ess_bdr_fhat = 1;
-         ess_bdr_uhat[1] = 1;
-         ess_bdr_fhat[1] = 0;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
-         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
-      }
-
-      // shift the ess_tdofs
-      int n = ess_tdof_list_uhat.Size();
-      int m = ess_tdof_list_fhat.Size();
-      Array<int> ess_tdof_list(n+m);
-      for (int i = 0; i < n; i++)
-      {
-         ess_tdof_list[i] = ess_tdof_list_uhat[i] 
-                          + u_fes->GetTrueVSize() 
-                          + sigma_fes->GetTrueVSize();
-      }
-      for (int i = 0; i < m; i++)
-      {
-         ess_tdof_list[i+n] = ess_tdof_list_fhat[i] 
-                            + u_fes->GetTrueVSize() 
-                            + sigma_fes->GetTrueVSize()
-                            + hatu_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatf_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-
-      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
-
-      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
-      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      for (int i=0; i<A->NumRowBlocks(); i++)
-      {
-         M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-      }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(200000);
-      cg.SetPrintLevel(0);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }   
-
-      GridFunction uex_gf(u_fes);
-      uex_gf.ProjectCoefficient(uex);
-
-      GridFunction sigmaex_gf(sigma_fes);
-      sigmaex_gf.ProjectCoefficient(sigmaex);
-
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      GridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      int dofs = X.Size();
-      double u_err = u_gf.ComputeL2Error(uex);
-      double u_norm = uex_gf.ComputeL2Error(zero);
-      // mfem::out << "u_err = " << u_err << endl;
-      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
-      double sigma_norm = sigmaex_gf.ComputeL2Error(vzero);
-      // mfem::out << "sigma_err = " << sigma_err << endl;
-      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
-      double L2norm = sqrt(u_norm * u_norm + sigma_norm * sigma_norm);
-
-      double rel_error = L2Error/L2norm;
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/residual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = residual;
-      dof0 = dofs;
-      mfem::out << std::right << std::setw(11) << i << " | " 
-                << std::setw(10) <<  dof0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  err0 << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::fixed <<  rel_error * 100. << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_err << " | " 
-                << std::setprecision(3) 
-                << std::setw(10) << std::scientific <<  res0 << " | " 
-                << std::setprecision(2) 
-                << std::setw(6) << std::fixed << rate_res << " | " 
-                << std::resetiosflags(std::ios::showbase)
-                << std::endl;
-
-
-      if (visualization)
-      {
-         // uex_out.precision(8);
-         // uex_out << "solution\n" << mesh << uex_gf <<
-         //          "window_title 'Exact u' "
-         //          << flush;
-         u_out.precision(8);
-         u_out << "solution\n" << mesh << u_gf <<
-                  "window_title 'Numerical u' "
-                  << flush;
-         // sigma_out.precision(8);
-         // sigma_out << "solution\n" << mesh << sigma_gf <<
-         //       "window_title 'Numerical flux' "
-         //       << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      mesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-      if (test_norm == test_norm_type::robust)
-      {
-         coeff_fes->Update();
-         c1_gf.Update();
-         c2_gf.Update();
-         Array<int> dofs;
-         for (int i = 0; i < mesh.GetNE(); i++)
-         {
-            double volume = mesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            // double c2 = 1.;
-            coeff_fes->GetElementDofs(i,dofs);
-            c1_gf.SetSubVector(dofs,c1);
-            c2_gf.SetSubVector(dofs,c2);
-         }
-      }
-
-   }
-
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatf_fes;
-   delete hatf_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
-
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u)
-{
-   double x = X[0];
-   double y = X[1];
-   double z = 0.;
-   if (X.Size() == 3) z = X[2];
-   du.SetSize(X.Size());
-   du = 0.;
-   d2u = 0.;
-
-   switch(prob)
-   {
-      case polynomial:
-      {
-         int n=2;
-         int m=2;
-         u = pow(x,n)*pow(y,m);
-         du[0] = n * pow(x,n-1) * pow(y,m);
-         du[1] = m * pow(x,n) * pow(y,m-1);
-         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
-             + m * (m-1) * pow(x,n) * pow(y,m-2);
-      }
-      break;
-      case EJ:
-      {
-         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
-         double r1 = (1. + alpha) / (2.*epsilon);
-         double r2 = (1. - alpha) / (2.*epsilon);
-         double denom = exp(-r2) - exp(-r1);
-
-
-         double g1 = exp(r2*(x-1.));
-         double g1_x = r2*g1;
-         double g1_xx = r2*g1_x;
-         double g2 = exp(r1*(x-1.));
-         double g2_x = r1*g2;
-         double g2_xx = r1*g2_x;
-         double g = g1-g2;
-         double g_x = g1_x - g2_x;
-         double g_xx = g1_xx - g2_xx;
-
-
-         u = g * cos(M_PI * y)/denom;
-         double u_x = g_x * cos(M_PI * y)/denom;
-         double u_xx = g_xx * cos(M_PI * y)/denom;
-         double u_y = -M_PI * g * sin(M_PI*y)/denom;
-         double u_yy = -M_PI * M_PI * u;
-         du[0] = u_x;
-         du[1] = u_y;
-         d2u = u_xx + u_yy;
-
-      }
-      break;
-      default:
-      {
-         double alpha = M_PI * (x + y + z);
-         u = sin(alpha);
-         du.SetSize(X.Size());
-         for (int i = 0; i<du.Size(); i++)
-         {
-            du[i] = M_PI * cos(alpha);
-         }
-         d2u = - M_PI*M_PI * u * du.Size();
-      }
-      break;
-   }
-}
-
-
-double exact_u(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return u;
-}
-
-void exact_sigma(const Vector & X, Vector & sigma)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   // σ = ε ∇ u
-   sigma = du;
-   sigma *= epsilon;
-}
-
-double exact_hatu(const Vector & X)
-{
-   return -exact_u(X);
-}
-
-void exact_hatf(const Vector & X, Vector & hatf)
-{
-   Vector sigma;
-   exact_sigma(X,sigma);
-   double u = exact_u(X);
-   hatf.SetSize(X.Size());
-   for (int i = 0; i<hatf.Size(); i++)
-   {
-      hatf[i] = beta[i] * u - sigma[i];
-   }
-}
-
-double f_exact(const Vector & X)
-{
-   // f = - εΔu + ∇⋅(βu)
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-
-   double s = 0;
-   for (int i = 0; i<du.Size(); i++)
-   {
-      s += beta[i] * du[i];
-   }
-   return -epsilon * d2u + s;
-}
@@ -1,698 +0,0 @@
-//                                MFEM Ultraweak DPG example
-//
-// Compile with: make  uw_dpgp
-//
-// sample runs 
-// mpirun -np 6 ./uw_dpgp  -m ../../../data/inline-quad.mesh -o 3 -ref 10 -test-norm 2 -do 1 -prob 1 -eps 1e-4
-//     - εΔu + ∇⋅(βu) = f,   in Ω
-//                  u = u_0, on ∂Ω
-
-// First Order System
-
-//     - ∇⋅σ + ∇⋅(βu) = f,   in Ω
-//        1/ε σ - ∇u  = 0,   in Ω
-//                  u = u_0, on ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, f̂ ∈ H^-1/2  
-// -(βu , ∇v)  + (σ , ∇v)     + < f̂ ,  v  > = (f,v),   ∀ v ∈ H^1(Ω)      
-//   (u , ∇⋅τ) + 1/ε (σ , τ)  + < û , τ⋅n > = 0,       ∀ τ ∈ H(div,Ω)
-//                                        û = u_0  on ∂Ω 
-
-// Note: 
-// f̂ := βu - σ
-// û := -u
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |   û       |     f̂    |  RHS    |
-// -------------------------------------------------------------
-// | v |-(βu , ∇v) | (σ , ∇v)  |           | < f̂ ,v > |  (f,v)  |
-// |   |           |           |           |          |         |
-// | τ | (u ,∇⋅τ)  | 1/ε(σ , τ)|  <û,τ⋅n>  |          |    0    |  
-
-// where (v,τ) ∈  H^1(Ω_h) × H(div,Ω_h)  
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   polynomial,
-   EJ,   
-   general  
-};
-
-enum test_norm_type
-{
-   standard,   
-   adjoint_graph,
-   robust  
-};
-
-prob_type prob;
-test_norm_type test_norm;
-Vector beta;
-double epsilon;
-// Function returns the solution u, and gradient du and the Laplacian d2u
-void solution(const Vector & x, double & u, Vector & du, double & d2u);
-double exact_u(const Vector & X);
-void exact_sigma(const Vector & X, Vector & sigma);
-double exact_hatu(const Vector & X);
-void exact_hatf(const Vector & X, Vector & hatf);
-double f_exact(const Vector & X);
-
-
-int main(int argc, char *argv[])
-{
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool visualization = true;
-   int iprob = 0;
-   int itest_norm = 0;
-   double theta = 0.7;
-   bool static_cond = false;
-   epsilon = 1e0;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&epsilon, "-eps", "--epsilon",
-                  "Epsilon coefficient");               
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");     
-   args.AddOption(&theta, "-theta", "--theta",
-                  "Theta parameter for AMR");                           
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: lshape, 1: General");                         
-   args.AddOption(&itest_norm, "-test-norm", "--test-norm", "Choice of test norm"
-                  " 0: Standard, 1: Adjoint Graph, 2: Robust");     
-   args.AddOption(&beta, "-beta", "--beta",
-                  "Vector Coefficient beta");          
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                            
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   if (iprob > 2) { iprob = 2; }
-   prob = (prob_type)iprob;
-
-   test_norm = (test_norm_type)itest_norm;
-
-   if (prob == prob_type::EJ)
-   {
-      mesh_file = "../../../data/inline-quad.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   if (beta.Size() == 0)
-   {
-      beta.SetSize(dim);
-      beta[0] = 1.;
-      beta[1] = 0.;
-   }
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatf_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatf_fes = new ParFiniteElementSpace(&pmesh,hatf_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-   ConstantCoefficient eps(epsilon);
-   ConstantCoefficient eps1(1./epsilon);
-   ConstantCoefficient negeps1(-1./epsilon);
-   ConstantCoefficient eps2(1/(epsilon*epsilon));
-
-   ConstantCoefficient negeps(-epsilon);
-   VectorConstantCoefficient betacoeff(beta);
-   Vector negbeta = beta;
-   negbeta.Neg();
-
-   DenseMatrix bbt(beta.Size());
-   MultVVt(beta, bbt); 
-   MatrixConstantCoefficient bbtcoeff(bbt);
-
-
-   VectorConstantCoefficient negbetacoeff(negbeta);
-   // Normal equation weak formulation
-   Array<ParFiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatf_fes);
-   test_fec.Append(v_fec);
-   test_fec.Append(tau_fec);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //-(βu , ∇v)
-   a->AddTrialIntegrator(new MixedScalarWeakDivergenceIntegrator(betacoeff),0,0);
-   
-   // (σ,∇ v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,0);
-
-   // (u ,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(negone),0,1);
-
-   // 1/ε (σ,τ)
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(eps1)),1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,1);
-
-   // <f̂ ,v> 
-   a->AddTrialIntegrator(new TraceIntegrator,3,0);
-
-
-   FiniteElementCollection *coeff_fec = new L2_FECollection(0,dim);
-   ParFiniteElementSpace *coeff_fes = new ParFiniteElementSpace(&pmesh,coeff_fec);
-   ParGridFunction c1_gf, c2_gf;
-   GridFunctionCoefficient c1_coeff(&c1_gf);
-   GridFunctionCoefficient c2_coeff(&c2_gf);
-
-   switch (test_norm)
-   {
-      case standard:
-      {   
-         if (myid == 0)
-         {
-            mfem::out << "\n Test norm: Standard" << endl;
-         }
-         // (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-      }
-      break;
-      case adjoint_graph:
-      {
-         if (myid == 0)
-         {
-            mfem::out << "\n Test norm: Adjoint Graph" << endl;
-         }
-         // (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(one),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(one),0,0);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-         // (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(one),1,1);
-         // 1/ε^2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(eps2),1,1);
-         // 1/ε (∇v, δτ)
-         a->AddTestIntegrator(new MixedVectorGradientIntegrator(eps1),0,1);
-         // - (β ⋅ ∇v,∇⋅δτ)  
-         a->AddTestIntegrator(new MixedGradDivIntegrator(betacoeff),0,1);
-         // 1/ε (τ,∇δv) 
-         a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(negeps1),1,0);
-         // -(β ∇⋅τ ,∇⋅δv)  
-         a->AddTestIntegrator(new MixedDivGradIntegrator(betacoeff),1,0);
-      }
-      break;
-      default:
-      {
-         if (myid == 0)
-         {
-            mfem::out << "\n Test norm: Robust" << endl;
-         }
-         c1_gf.SetSpace(coeff_fes);
-         c2_gf.SetSpace(coeff_fes);
-         Array<int> dofs;
-         for (int i =0; i < pmesh.GetNE(); i++)
-         {
-            double volume = pmesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            coeff_fes->GetElementDofs(i,dofs);
-            c1_gf.SetSubVector(dofs,c1);
-            c2_gf.SetSubVector(dofs,c2);
-         }
-         // c1 (v,δv)
-         a->AddTestIntegrator(new MassIntegrator(c1_coeff),0,0);
-         // ε (∇v,∇δv)
-         a->AddTestIntegrator(new DiffusionIntegrator(eps),0,0);
-         // (β⋅∇v, β⋅∇δv)   
-         a->AddTestIntegrator(new DiffusionIntegrator(bbtcoeff), 0,0);
-         // c2 (τ,δτ)
-         a->AddTestIntegrator(new VectorFEMassIntegrator(c2_coeff),1,1);
-         // (∇⋅τ,∇⋅δτ)
-         a->AddTestIntegrator(new DivDivIntegrator(one),1,1);
-      }
-      break;
-   }
-
-
-   FunctionCoefficient f(f_exact);
-   // if (prob != prob_type::EJ)
-   // {
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(f),0);
-   // }
-
-   FunctionCoefficient hatuex(exact_hatu);
-   VectorFunctionCoefficient hatfex(dim,exact_hatf);
-   Array<int> elements_to_refine;
-   FunctionCoefficient uex(exact_u);
-   VectorFunctionCoefficient sigmaex(dim,exact_sigma);
-
-   ParGridFunction hatu_gf;
-   ParGridFunction hatf_gf;
-
-   // socketstream uex_out;
-   socketstream u_out;
-   // socketstream sigma_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      // uex_out.open(vishost, visport);
-      // sigma_out.open(vishost, visport);
-   }
-
-   double res0 = 0.;
-   double err0 = 0.;
-   int dof0;
-   if (myid == 0)
-   {
-      mfem::out << " Refinement |" 
-               << "    Dofs    |" 
-               << "  L2 Error  |" 
-               << "  Rate  |" 
-               << "  Residual  |" 
-               << "  Rate  |" << endl;
-      mfem::out << " --------------------"      
-               <<  "-------------------"    
-               <<  "-------------------"    
-               <<  "-------------------" << endl;   
-   }
-
-
-   for (int i = 0; i<=ref; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list_uhat;
-      Array<int> ess_tdof_list_fhat;
-      Array<int> ess_bdr_uhat;
-      Array<int> ess_bdr_fhat;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr_uhat.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr_fhat.SetSize(pmesh.bdr_attributes.Max());
-         // ess_bdr_uhat = 1;
-         // ess_bdr_fhat = 0;
-         ess_bdr_uhat = 0;
-         ess_bdr_fhat = 1;
-         ess_bdr_uhat[1] = 1;
-         ess_bdr_fhat[1] = 0;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr_uhat, ess_tdof_list_uhat);
-         hatf_fes->GetEssentialTrueDofs(ess_bdr_fhat, ess_tdof_list_fhat);
-      }
-
-      // shift the ess_tdofs
-      int n = ess_tdof_list_uhat.Size();
-      int m = ess_tdof_list_fhat.Size();
-      Array<int> ess_tdof_list(n+m);
-      for (int i = 0; i < n; i++)
-      {
-         ess_tdof_list[i] = ess_tdof_list_uhat[i] 
-                          + u_fes->GetTrueVSize() 
-                          + sigma_fes->GetTrueVSize();
-      }
-      for (int i = 0; i < m; i++)
-      {
-         ess_tdof_list[i+n] = ess_tdof_list_fhat[i] 
-                            + u_fes->GetTrueVSize() 
-                            + sigma_fes->GetTrueVSize()
-                            + hatu_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatf_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr_uhat);
-
-      hatf_gf.MakeRef(hatf_fes,x.GetBlock(3));
-      hatf_gf.ProjectBdrCoefficientNormal(hatfex,ess_bdr_fhat);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
-      {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M->SetDiagonalBlock(0,amg0);
-         M->SetDiagonalBlock(1,amg1);
-         skip = 2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
-      amg2->SetPrintLevel(0);
-      M->SetDiagonalBlock(skip,amg2);
-
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatf_fes);
-      }
-      M->SetDiagonalBlock(skip+1,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-6);
-      cg.SetMaxIter(200000);
-      cg.SetPrintLevel(0);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      double maxresidual = residuals.Max(); 
-
-      double gresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&gresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      gresidual = sqrt(gresidual);
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      ParGridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      int dofs = u_fes->GlobalTrueVSize()
-               + sigma_fes->GlobalTrueVSize()
-               + hatu_fes->GlobalTrueVSize()
-               + hatf_fes->GlobalTrueVSize();
-
-      double u_err = u_gf.ComputeL2Error(uex);
-      double sigma_err = sigma_gf.ComputeL2Error(sigmaex);
-      double L2Error = sqrt(u_err*u_err + sigma_err*sigma_err);
-
-      double rate_err = (i) ? dim*log(err0/L2Error)/log((double)dof0/dofs) : 0.0;
-      double rate_res = (i) ? dim*log(res0/gresidual)/log((double)dof0/dofs) : 0.0;
-
-      err0 = L2Error;
-      res0 = gresidual;
-      dof0 = dofs;
-
-      if (myid == 0)
-      {
-         mfem::out << std::right << std::setw(11) << i << " | " 
-                  << std::setw(10) <<  dof0 << " | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::scientific <<  err0 << " | " 
-                  << std::setprecision(2) 
-                  << std::setw(6) << std::fixed << rate_err << " | " 
-                  << std::setprecision(3) 
-                  << std::setw(10) << std::scientific <<  res0 << " | " 
-                  << std::setprecision(2) 
-                  << std::setw(6) << std::fixed << rate_res << " | " 
-                  << std::resetiosflags(std::ios::showbase)
-                  << std::endl;
-      }
-
-
-      if (visualization)
-      {
-         // uex_out.precision(8);
-         // uex_out << "parallel " << num_procs << " " << myid << "\n";
-         // uex_out << "solution\n" << pmesh << uex_gf <<
-         //          "window_title 'Exact u' "
-         //          << flush;
-
-         u_out << "parallel " << num_procs << " " << myid << "\n";
-         u_out.precision(8);
-         u_out << "solution\n" << pmesh << u_gf <<
-                  "window_title 'Numerical u' "
-                  << flush;
-
-         // sigma_out << "parallel " << num_procs << " " << myid << "\n";
-         // sigma_out.precision(8);
-         // sigma_out << "solution\n" << pmesh << sigma_gf <<
-         //       "window_title 'Numerical flux' "
-         //       << flush;
-      }
-
-      if (i == ref)
-         break;
-
-      pmesh.GeneralRefinement(elements_to_refine,1,1);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-
-      if (test_norm == test_norm_type::robust)
-      {
-         coeff_fes->Update();
-         c1_gf.Update();
-         c2_gf.Update();
-         Array<int> edofs;
-         for (int i = 0; i < pmesh.GetNE(); i++)
-         {
-            double volume = pmesh.GetElementVolume(i);
-            double c1 = min(epsilon/volume, 1.);
-            double c2 = min(1./epsilon, 1./volume);
-            coeff_fes->GetElementDofs(i,edofs);
-            c1_gf.SetSubVector(edofs,c1);
-            c2_gf.SetSubVector(edofs,c2);
-         }
-      }
-   }
-
-
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatf_fes;
-   delete hatf_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u)
-{
-   double x = X[0];
-   double y = X[1];
-   double z = 0.;
-   if (X.Size() == 3) z = X[2];
-   du.SetSize(X.Size());
-   du = 0.;
-   d2u = 0.;
-
-   switch(prob)
-   {
-      case polynomial:
-      {
-         int n=2;
-         int m=2;
-         u = pow(x,n)*pow(y,m);
-         du[0] = n * pow(x,n-1) * pow(y,m);
-         du[1] = m * pow(x,n) * pow(y,m-1);
-         d2u = n * (n-1) * pow(x,n-2) * pow(y,m)
-             + m * (m-1) * pow(x,n) * pow(y,m-2);
-      }
-      break;
-      case EJ:
-      {
-         double alpha = sqrt(1. + 4. * epsilon * epsilon * M_PI * M_PI);
-         double r1 = (1. + alpha) / (2.*epsilon);
-         double r2 = (1. - alpha) / (2.*epsilon);
-         double denom = exp(-r2) - exp(-r1);
-
-
-         double g1 = exp(r2*(x-1.));
-         double g1_x = r2*g1;
-         double g1_xx = r2*g1_x;
-         double g2 = exp(r1*(x-1.));
-         double g2_x = r1*g2;
-         double g2_xx = r1*g2_x;
-         double g = g1-g2;
-         double g_x = g1_x - g2_x;
-         double g_xx = g1_xx - g2_xx;
-
-
-         u = g * cos(M_PI * y)/denom;
-         double u_x = g_x * cos(M_PI * y)/denom;
-         double u_xx = g_xx * cos(M_PI * y)/denom;
-         double u_y = -M_PI * g * sin(M_PI*y)/denom;
-         double u_yy = -M_PI * M_PI * u;
-         du[0] = u_x;
-         du[1] = u_y;
-         d2u = u_xx + u_yy;
-
-      }
-      break;
-      default:
-      {
-         double alpha = M_PI * (x + y + z);
-         u = sin(alpha);
-         du.SetSize(X.Size());
-         for (int i = 0; i<du.Size(); i++)
-         {
-            du[i] = M_PI * cos(alpha);
-         }
-         d2u = - M_PI*M_PI * u * du.Size();
-      }
-      break;
-   }
-}
-
-
-double exact_u(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return u;
-}
-
-void exact_sigma(const Vector & X, Vector & sigma)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   // σ = ε ∇ u
-   sigma = du;
-   sigma *= epsilon;
-}
-
-double exact_hatu(const Vector & X)
-{
-   return -exact_u(X);
-}
-
-void exact_hatf(const Vector & X, Vector & hatf)
-{
-   Vector sigma;
-   exact_sigma(X,sigma);
-   double u = exact_u(X);
-   hatf.SetSize(X.Size());
-   for (int i = 0; i<hatf.Size(); i++)
-   {
-      hatf[i] = beta[i] * u - sigma[i];
-   }
-}
-
-double f_exact(const Vector & X)
-{
-   // f = - εΔu + ∇⋅(βu)
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-
-   double s = 0;
-   for (int i = 0; i<du.Size(); i++)
-   {
-      s += beta[i] * du[i];
-   }
-   return -epsilon * d2u + s;
-}
@@ -1,203 +0,0 @@
-//                 MFEM Fosls 1
-//
-// Compile with: make blkfosls
-//
-//     - Δ u = f, in Ω
-//         u = 0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// FOSLS:
-//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
-
-
-// -------------------------------------------------
-// |   |    u    |         σ           |    RHS    |
-// -------------------------------------------------
-// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
-// |   |         |                     |           |
-// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
-
-// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   // 5. Define a finite element space on the mesh. Here we use continuous
-   //    Lagrange finite elements of the specified order. If order < 1, we
-   //    instead use an isoparametric/isogeometric space.
-   FiniteElementCollection *fec0 = new H1_FECollection(order, dim);
-   FiniteElementCollection *fec1 = new RT_FECollection(order-1, dim);
-   FiniteElementSpace fespace0(&mesh, fec0);
-   FiniteElementSpace fespace1(&mesh, fec1);
-
-   Array<FiniteElementSpace *> fespaces(2);
-   fespaces[0] = &fespace0;
-   fespaces[1] = &fespace1;
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      fespaces[0]->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   BlockBilinearForm a(fespaces);
-   a.SetDiagonalPolicy(mfem::Operator::DIAG_KEEP);
-
-   cout << "H1 fespace = " << fespace0.GetVSize() << endl;
-   cout << "RT fespace = " << fespace1.GetVSize() << endl;
-
-   FiniteElementCollection *fec2 = new RT_Trace_FECollection(order-1, dim);
-   FiniteElementSpace RT_trace_fes(&mesh, fec2);
-   cout << "RT trace = " << RT_trace_fes.GetVSize() << endl;
-
-   // for (int i = 0; i<mesh.GetNE(); i++)
-   // {
-   //    // const FiniteElement * fe = fespace1.GetFE(i);
-   //    // fespace1.GetTraceElement()
-   //    Array<int> faces, ori;
-   //    mesh.GetElementEdges(i, faces, ori);
-   //    for (int f = 0; f<faces.Size(); f++)
-   //    {
-   //       const FiniteElement * fe_trace = RT_trace_fes.GetFaceElement(faces[f]);
-   //       cout << fe_trace->GetDof() << endl;
-   //       Array<int> face_dofs;
-   //       RT_trace_fes.GetFaceDofs(faces[f],face_dofs);
-   //       cout << "face dofs = " << endl;
-   //       face_dofs.Print();
-   //    }
-
-   //    // cout << fe->GetGeomType() << endl;
-   //    Array<int> vdofs;
-   //    RT_trace_fes.GetElementVDofs(i, vdofs);
-   //    cout << "trace dofs = " << endl;
-   //    vdofs.Print();
-   //    fespace1.GetElementVDofs(i, vdofs);
-   //    cout << "elem dofs = " << endl;
-   //    vdofs.Print();
-   //    cin.get();
-   // }
-
-
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   Array2D<BilinearFormIntegrator * > blfi(2,2);
-   blfi(0,0) = new DiffusionIntegrator(one);
-   blfi(0,1) = new MixedVectorWeakDivergenceIntegrator(one);
-   blfi(1,0) = new MixedVectorGradientIntegrator(negone);
-
-   BilinearFormIntegrator * divdiv = new DivDivIntegrator(one);
-   BilinearFormIntegrator * mass = new VectorFEMassIntegrator(one);
-   SumIntegrator * suminteg = new SumIntegrator();
-   suminteg->AddIntegrator(divdiv);
-   suminteg->AddIntegrator(mass);
-   blfi(1,1) = suminteg;
-
-   TestBlockBilinearFormIntegrator * integ = new TestBlockBilinearFormIntegrator();
-   integ->SetIntegrators(blfi);
-   a.AddDomainIntegrator(integ);
-   a.Assemble();
-
-
-   BlockLinearForm b(fespaces);
-
-   TestBlockLinearFormIntegrator * lininteg = new TestBlockLinearFormIntegrator();
-   Array<LinearFormIntegrator * > lfi(2);
-   lfi[0] = nullptr;
-   lfi[1] = new VectorFEDomainLFDivIntegrator(negone);
-   lininteg->SetIntegrators(lfi);
-   b.AddDomainIntegrator(lininteg);
-   b.Assemble();
-
-
-   // need to implement blkgridfunction later but for now Vector would do
-   int size = 0;
-   for (int i = 0; i<fespaces.Size(); i++)
-   {
-      size += fespaces[i]->GetVSize();
-   }
-
-   Vector x(size);
-   x = 0.0;
-
-   OperatorPtr A;
-   Vector X,B;
-   a.FormLinearSystem(ess_tdof_list,x,b,A,X,B);
-
-   GSSmoother M((SparseMatrix&)(*A));
-   CGSolver cg;
-   cg.SetRelTol(1e-6);
-   cg.SetMaxIter(200);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   a.RecoverFEMSolution(X,b,x);
-
-   GridFunction u_gf, sigma_gf;
-   double *data = x.GetData();
-   u_gf.MakeRef(fespaces[0],&data[0]);
-   sigma_gf.MakeRef(fespaces[1],&data[fespaces[0]->GetVSize()]);
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << u_gf <<
-                "window_title 'Numerical u' "
-                << flush;
-      socketstream sols_sock(vishost, visport);
-      sols_sock.precision(8);
-      sols_sock << "solution\n" << mesh << sigma_gf <<
-                "window_title 'Numerical sigma' "
-                << flush;
-   }
-
-   delete fec0;
-
-   return 0;
-}
@@ -1,223 +0,0 @@
-//                   MFEM Fosls example
-//
-// Compile with: make fosls
-//
-//     - Δ u = f, in Ω
-//         u = 0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// FOSLS:
-//       minimize  1/2(||∇u - σ||^2 + ||∇ ⋅ σ - f||^2)
-
-
-// -------------------------------------------------
-// |   |    u    |         σ           |    RHS    |
-// -------------------------------------------------
-// | v | (∇u,∇v) |      -(σ,∇v)        |     0     |
-// |   |         |                     |           |
-// | τ | -(∇u,τ) | (∇⋅σ, ∇⋅τ) + (σ,τ)  | -(f,∇⋅τ ) |
-
-// where (u,τ) ∈ H^1(Ω) × H(div,Ω)
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   bool visualization = true;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree) or -1 for"
-                  " isoparametric space.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   // 3. Read the mesh from the given mesh file. We can handle triangular,
-   //    quadrilateral, tetrahedral, hexahedral, surface and volume meshes with
-   //    the same code.
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   FiniteElementCollection *H1fec = new H1_FECollection(order,dim);
-   FiniteElementSpace *H1fes = new FiniteElementSpace(&mesh, H1fec);
-
-   FiniteElementCollection *RTfec = new RT_FECollection(order-1,dim);
-   FiniteElementSpace *RTfes = new FiniteElementSpace(&mesh, RTfec);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   // Linear forms
-   LinearForm b_0(H1fes);
-   // (f,∇⋅τ )
-   LinearForm b_1(RTfes);
-   b_1.AddDomainIntegrator(new VectorFEDomainLFDivIntegrator(negone));
-
-   // Bilinear forms
-   // (∇u,∇v)
-   BilinearForm a_00(H1fes);
-   a_00.AddDomainIntegrator(new DiffusionIntegrator(one));
-
-   //   -(σ,∇v)
-   MixedBilinearForm a_01(RTfes, H1fes);
-   a_01.AddDomainIntegrator(new MixedVectorWeakDivergenceIntegrator(
-                               one)); // (-1 is included)
-
-   //    // -(∇u,τ)
-   // MixedBilinearForm()
-   MixedBilinearForm a_10(H1fes, RTfes);
-   a_10.AddDomainIntegrator(new MixedVectorGradientIntegrator(negone));
-
-   // (∇⋅σ, ∇⋅τ) + (σ,τ)
-
-   BilinearForm a_11(RTfes);
-   a_11.AddDomainIntegrator(new DivDivIntegrator(one));
-   a_11.AddDomainIntegrator(new VectorFEMassIntegrator(one));
-
-
-   Array<int> ess_bdr;
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      H1fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   Array<int> block_Toffsets(3);
-   block_Toffsets[0] = 0;
-   block_Toffsets[1] = H1fes->GetTrueVSize();
-   block_Toffsets[2] = RTfes->GetTrueVSize();
-   block_Toffsets.PartialSum();
-
-   Vector rhs_H1(H1fes->GetVSize()); rhs_H1 = 0.;
-   Vector rhs_RT(RTfes->GetVSize()); rhs_RT = 0.;
-
-   Vector x_H1(H1fes->GetVSize()); x_H1 = 0.;
-   Vector x_RT(RTfes->GetVSize()); x_RT = 0.;
-
-
-   Vector RHS_H1(H1fes->GetTrueVSize()); RHS_H1 = 0.0;
-   Vector RHS_RT(RTfes->GetTrueVSize()); RHS_RT = 0.0;
-
-   Vector X_H1(H1fes->GetTrueVSize()); X_H1 = 0.0;
-   Vector X_RT(RTfes->GetTrueVSize()); X_RT = 0.0;
-
-
-   b_0.Update(H1fes,rhs_H1,0);
-   b_0.Assemble();
-
-   b_1.Update(RTfes,rhs_RT,0);
-   b_1.Assemble();
-
-
-   // Assembly and BC
-   a_00.Assemble();
-   SparseMatrix A_00;
-   a_00.FormLinearSystem(ess_tdof_list,x_H1,rhs_H1,
-                         A_00,X_H1,RHS_H1);
-
-   a_01.Assemble();
-   SparseMatrix A_01;
-   Array<int> empty;
-   a_01.FormRectangularSystemMatrix(empty, ess_tdof_list,A_01);
-
-
-   a_10.Assemble();
-   SparseMatrix A_10;
-
-   a_10.FormRectangularLinearSystem(ess_tdof_list,empty,x_H1,rhs_RT,
-                                    A_10,X_H1,RHS_RT);
-
-   a_11.Assemble();
-   SparseMatrix A_11;
-   a_11.FormSystemMatrix(empty,A_11);
-
-
-   BlockMatrix BlockA(block_Toffsets);
-   BlockA.SetBlock(0,0,&A_00);
-   BlockA.SetBlock(0,1,&A_01);
-   BlockA.SetBlock(1,0,&A_10);
-   BlockA.SetBlock(1,1,&A_11);
-
-
-   BlockVector RHS(block_Toffsets);
-   RHS.GetBlock(0) = RHS_H1;
-   RHS.GetBlock(1) = RHS_RT;
-
-   BlockVector X(block_Toffsets);
-   X.GetBlock(0) = X_H1;
-   X.GetBlock(1) = X_RT;
-
-
-   SparseMatrix * A = BlockA.CreateMonolithic();
-
-   GSSmoother M(*A);
-
-   CGSolver cg;
-   cg.SetRelTol(1e-6);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(1);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(RHS, X);
-
-   GridFunction u_gf(H1fes), sigma_gf(RTfes);
-   u_gf = 0.;
-   sigma_gf = 0.;
-
-   const SparseMatrix * P = H1fes->GetConformingProlongation();
-   if (P)
-   {
-      a_00.RecoverFEMSolution(X.GetBlock(0),rhs_H1,u_gf);
-      a_11.RecoverFEMSolution(X.GetBlock(1),rhs_RT,sigma_gf);
-   }
-   else
-   {
-      u_gf.MakeRef(X.GetBlock(0),0);
-      sigma_gf.MakeRef(X.GetBlock(1),0);
-   }
-
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      socketstream solu_sock(vishost, visport);
-      solu_sock.precision(8);
-      solu_sock << "solution\n" << mesh << u_gf <<
-                "window_title 'Numerical u' "
-                << flush;
-      socketstream sols_sock(vishost, visport);
-      sols_sock.precision(8);
-      sols_sock << "solution\n" << mesh << sigma_gf <<
-                "window_title 'Numerical sigma' "
-                << flush;
-   }
-
-   return 0;
-}
@@ -1,61 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/diffusion,)
-CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
-
-MFEM_LIB_FILE = mfem_is_not_built
-include $(CONFIG_MK)
-
-SEQ_EXAMPLES = blkfosls fosls primal_dpg \
-	 				uw_dpg 
-PAR_EXAMPLES = uw_dpgp 
-
-ifeq ($(MFEM_USE_MPI),NO)
-   EXAMPLES = $(SEQ_EXAMPLES)
-else
-   EXAMPLES = $(PAR_EXAMPLES) $(SEQ_EXAMPLES)
-endif
-
-.SUFFIXES:
-.SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
-
-# Remove built-in rule
-%: %.cpp
-
-# Replace the default implicit rule for *.cpp files
-%: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
-	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)
-
-all: $(EXAMPLES) 
-
-MFEM_TESTS = EXAMPLES
-include $(MFEM_TEST_MK)
-
-# Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-%-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
-%-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
-
-clean: clean-build clean-exec 
-
-clean-build:
-	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
-	rm -rf *.dSYM *.TVD.*breakpoints
-	rm -rf ParaView
-
-clean-exec:
@@ -1,179 +0,0 @@
-//                     MFEM primal_dpg example
-//
-// Compile with: make primal_dpg
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../../data/star.mesh";
-   int order = 1;
-   bool static_cond = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");   
-   args.ParseCheck();
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   // mesh.UniformRefinement();
-
-   // 3. Define a finite element space on the mesh. Here we use H1 continuous
-   //    high-order Lagrange finite elements of the given order.
-   H1_FECollection fec(order, mesh.Dimension());
-   FiniteElementSpace H1fes(&mesh, &fec);
-
-   RT_Trace_FECollection trace_fec(order-1, mesh.Dimension());
-   FiniteElementSpace RTtrace_fes(&mesh, &trace_fec);
-
-   int dim = mesh.Dimension();
-   int test_order = order;
-   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
-   {
-      test_order++;
-   }
-
-   test_order++;
-
-   H1_FECollection test_fec(test_order,mesh.Dimension());
-
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fecs; 
-
-   trial_fes.Append(&H1fes);
-   trial_fes.Append(&RTtrace_fes);
-   test_fecs.Append(&test_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
-
-
-   ConstantCoefficient one(1.0);
-   a->AddTrialIntegrator(new DiffusionIntegrator(one),0,0);
-   a->AddTrialIntegrator(new TraceIntegrator,1,0);
-
-   BilinearFormIntegrator * diffusion = new DiffusionIntegrator(one);
-   BilinearFormIntegrator * mass = new MassIntegrator(one);
-   a->AddTestIntegrator(diffusion,0,0);
-   a->AddTestIntegrator(mass,0,0);
-
-   a->AddDomainLFIntegrator(new DomainLFIntegrator(one),0);
-   if (static_cond) { a->EnableStaticCondensation(); }
-   a->Assemble();
-
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      H1fes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   Vector X,B;
-   OperatorPtr Ah;
-
-   int size = H1fes.GetVSize() + RTtrace_fes.GetVSize();
-
-   Vector x(size);
-   x = 0.0;
-
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-6);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   delete M;
-
-
-   a->RecoverFEMSolution(X,x);
-
-   GridFunction u_gf;
-   double *data = x.GetData();
-   u_gf.MakeRef(&H1fes,data);
-
-   GridFunction s_gf;
-   s_gf.MakeRef(&RTtrace_fes,&data[H1fes.GetVSize()]);
-
-
-
-   RT_FECollection RTfec(order-1, mesh.Dimension());
-   FiniteElementSpace RTfes(&mesh, &RTfec);
-
-   GridFunction sigma_gf(&RTfes);
-   sigma_gf = 0.0;
-   for (int i = 0; i<mesh.GetNE(); i++)
-   {
-      Array<int> strace_dofs;
-      Array<int> trace_dofs;
-      Vector dofs;
-      RTtrace_fes.GetElementDofs(i,trace_dofs);
-      strace_dofs.SetSize(trace_dofs.Size());
-      // shift dofs;
-      for (int j = 0; j< trace_dofs.Size(); j++)
-      {
-         int offset = trace_dofs[j] < 0 ? -H1fes.GetVSize() : H1fes.GetVSize();
-         strace_dofs[j] = offset + trace_dofs[j];
-      }
-      x.GetSubVector(strace_dofs, dofs);
-      sigma_gf.SetSubVector(trace_dofs,dofs);
-
-   }
-
-
-
-
-
-   ParaViewDataCollection paraview_dc("DPG_example", &mesh);
-   paraview_dc.SetPrefixPath("ParaView");
-   paraview_dc.SetLevelsOfDetail(order);
-   paraview_dc.SetCycle(0);
-   paraview_dc.SetDataFormat(VTKFormat::BINARY);
-   paraview_dc.SetHighOrderOutput(true);
-   paraview_dc.SetTime(0.0); // set the time
-   paraview_dc.RegisterField("field",&u_gf);
-   paraview_dc.RegisterField("flux",&sigma_gf);
-   // paraview_dc.RegisterField("flux",&s_gf);
-   paraview_dc.Save();
-
-
-
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream solu_sock(vishost, visport);
-   solu_sock.precision(8);
-   solu_sock << "solution\n" << mesh << u_gf <<
-             "window_title 'Numerical u' "
-             << flush;
-
-   socketstream soltrace_sock(vishost, visport);
-   soltrace_sock.precision(8);
-   soltrace_sock << "solution\n" << mesh << sigma_gf <<
-                 "window_title 'Flux sigma_n' "
-                 << flush;
-
-
-
-}
@@ -1,403 +0,0 @@
-//                       MFEM Ultraweak DPG example
-//
-// Compile with: make uw_dpg
-//
-// sample runs 
-// ./uw_dpg -m ../lshape2.mesh -o 2 -ref 20 -graph-norm -do 1 -prob 0
-
-//     - Δ u = f,   in Ω
-//         u = u_0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, σ̂ ∈ H^-1/2  
-// -(u , ∇⋅τ) - (σ , τ) + < û, τ⋅n> = 0,      ∀ τ ∈ H(div,Ω)      
-//  (σ , ∇ v) + < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
-//                                û = 0       on ∂Ω 
-
-// Note: 
-// û := u
-// σ̂ := -σ
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
-// -------------------------------------------------------------
-// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
-// |   |           |           |           |         |         |
-// | v |           |  (σ,∇ v)  |           |  <σ̂,v>  |  (f,v)  |  
-
-// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   lshape,   
-   general  
-};
-
-prob_type prob;
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u);
-
-
-double exact_u(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return u;
-}
-
-void exact_sigma(const Vector & X, Vector & sigma)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   // σ = ∇ u
-   sigma = du;
-}
-
-double exact_hatu(const Vector & X)
-{
-   return exact_u(X);
-}
-
-void exact_hatsigma(const Vector & X, Vector & hatsigma)
-{
-   exact_sigma(X,hatsigma);
-   hatsigma *= -1.;
-}
-
-double f_exact(const Vector & X)
-{
-   double u, d2u;
-   Vector du;
-   solution(X,u,du,d2u);
-   return -d2u;
-}
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool adjoint_graph_norm = false;
-   bool visualization = true;
-   int iprob = 0;
-   bool static_cond = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");               
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space"); 
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: lshape, 1: General");
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                  
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   if (iprob > 1) { iprob = 1; }
-   prob = (prob_type)iprob;
-
-   if (prob == prob_type::lshape)
-   {
-      mesh_file = "../lshape2.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   mesh.UniformRefinement();
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *u_fes = new FiniteElementSpace(&mesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   FiniteElementSpace *sigma_fes = new FiniteElementSpace(&mesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   FiniteElementSpace *hatu_fes = new FiniteElementSpace(&mesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
-   FiniteElementSpace *hatsigma_fes = new FiniteElementSpace(&mesh,hatsigma_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   // Normal equation weak formulation
-   Array<FiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatsigma_fes);
-
-   test_fec.Append(tau_fec);
-   test_fec.Append(v_fec);
-
-   NormalEquations * a = new NormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //  -(u,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
-
-   // -(σ,τ) 
-   a->AddTrialIntegrator(new TransposeIntegrator(new VectorFEMassIntegrator(negone)),1,0);
-
-   // (σ,∇ v)
-   a->AddTrialIntegrator(new TransposeIntegrator(new GradientIntegrator(one)),1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
-
-   //  <σ̂,v> 
-   a->AddTrialIntegrator(new TraceIntegrator,3,1);
-
-   // test integrators (space-induced norm for H(div) × H1)
-   // (∇⋅τ,∇⋅δτ)
-   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
-   // (τ,δτ)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   // (∇v,∇δv)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new MassIntegrator(one),1,1);
-
-   // additional terms for adjoint graph norm
-   if (adjoint_graph_norm)
-   {
-      // -(∇v,δτ) 
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
-      // -(τ,∇δv) 
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
-      // (τ,δτ)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   }
-
-   // RHS
-   FunctionCoefficient f(f_exact);
-   if (prob == prob_type::general)
-   {
-      a->AddDomainLFIntegrator(new DomainLFIntegrator(f),1);
-   }
-
-   FunctionCoefficient hatuex(exact_hatu);
-   Array<int> elements_to_refine;
-   GridFunction hatu_gf;
-
-
-   socketstream u_out;
-   // socketstream sigma_out;
-   socketstream mesh_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      // sigma_out.open(vishost, visport);
-      mesh_out.open(vishost, visport);
-   }
-
-
-   for (int iref = 0; iref<ref; iref++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (mesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-   // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatsigma_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-      hatu_gf.ProjectBdrCoefficient(hatuex,ess_bdr);
-
-      OperatorPtr Ah;
-      Vector X,B;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockMatrix * A = Ah.As<BlockMatrix>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      for (int i=0; i<A->NumRowBlocks(); i++)
-      {
-         M->SetDiagonalBlock(i,new GSSmoother(A->GetBlock(i,i)));
-      }
-
-      CGSolver cg;
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      cout << "Residual = " << residual << endl;
-
-      elements_to_refine.SetSize(0);
-      double max_resid = residuals.Max();
-      double theta = 0.7;
-      for (int iel = 0; iel<mesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * max_resid)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-      
-      GridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      GridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      if (visualization)
-      {
-         u_out.precision(8);
-         string keys = (iref == 0) ? "keys em\n" : "keys";
-         u_out << "solution\n" << mesh << u_gf 
-               << "window_title 'Numerical u' "
-                  << flush;
-
-         // sigma_out.precision(8);
-         // sigma_out << "solution\n" << mesh << sigma_gf <<
-         //       "window_title 'Numerical flux' "
-         //       << flush;
-
-         mesh_out.precision(8);
-         mesh_out << "mesh\n" << mesh 
-                  << keys
-                  << "window_title 'Mesh' "
-                  << flush;
-
-      }
-
-      mesh.GeneralRefinement(elements_to_refine);
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatsigma_fes;
-   delete hatsigma_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete sigma_fes;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
-
-
-void solution(const Vector & X, double & u, Vector & du, double & d2u)
-{
-   double x = X[0];
-   double y = X[1];
-   double z = 0.;
-   if (X.Size() == 3) z = X[2];
-   du.SetSize(X.Size());
-   du = 0.;
-   d2u = 0.;
-
-   switch(prob)
-   {
-      case lshape:
-      {
-         double r = sqrt(x*x + y*y);
-         double alpha = 2./3.;
-         double theta = atan2(y,x);
-         if (theta < 0) theta += 2*M_PI;
-         u = pow(r,alpha) * sin(alpha * theta);
-      }
-      break;
-      default:
-      {
-         double alpha = M_PI * (x + y + z);
-         u = sin(alpha);
-         du.SetSize(X.Size());
-         for (int i = 0; i<du.Size(); i++)
-         {
-            du[i] = M_PI * cos(alpha);
-         }
-         d2u = - M_PI*M_PI * u * du.Size();
-      }
-      break;
-   }
-}
@@ -1,404 +0,0 @@
-//                 MFEM UW DPG parallel example
-//
-// Compile with: make poisson_fosls
-//
-//     - Δ u = f, in Ω
-//         u = 0, on ∂Ω
-
-// First Order System
-
-//   ∇ u - σ = 0, in Ω
-// - ∇⋅σ     = f, in Ω
-//        u  = 0, in ∂Ω
-
-// UW-DPG:
-// 
-// u ∈ L^2(Ω), σ ∈ (L^2(Ω))^dim 
-// û ∈ H^1/2, σ̂ ∈ H^-1/2  
-// -(u , ∇⋅τ) + < û, τ⋅n> - (σ , τ) = 0,      ∀ τ ∈ H(div,Ω)      
-//  (σ , ∇ v) - < σ̂, v  >           = (f,v)   ∀ v ∈ H^1(Ω)
-//            û = 0        on ∂Ω 
-
-// -------------------------------------------------------------
-// |   |     u     |     σ     |    û      |    σ̂    |  RHS    |
-// -------------------------------------------------------------
-// | τ | -(u,∇⋅τ)  |  -(σ,τ)   | < û, τ⋅n> |         |    0    |
-// |   |           |           |           |         |         |
-// | v |           |  (σ,∇ v)  |           | -<σ̂,v>  |  (f,v)  |  
-
-// where (τ,v) ∈  H(div,Ω) × H^1(Ω) 
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-enum prob_type
-{
-   lshape,   
-   general  
-};
-
-prob_type prob;
-
-double exact(const Vector & X)
-{
-   double x = X[0];
-   double y = X[1];
-
-   double r = sqrt(x*x + y*y);
-   double alpha = 2./3.;
-   double theta = atan2(y,x);
-   if (theta < 0) theta += 2*M_PI;
-
-   return pow(r,alpha) * sin(alpha * theta);
-}
-
-void gradexact(const Vector & X, Vector & grad)
-{
-   grad.SetSize(2);
-   double x = X[0];
-   double y = X[1];
-
-   double r = sqrt(x*x + y*y);
-   double alpha = 2./3.;
-   double theta = atan2(y,x);
-   if (theta < 0) theta += 2*M_PI;
-
-   double r_x = x/r;
-   double r_y = y/r;
-   double theta_x = - y / (r*r);
-   double theta_y = x / (r*r);
-   double beta = alpha * pow(r,alpha - 1.);
-   grad[0] = beta*(r_x * sin(alpha*theta) + r * theta_x * cos(alpha*theta));
-   grad[1] = beta*(r_y * sin(alpha*theta) + r * theta_y * cos(alpha*theta));
-}
-
-
-int main(int argc, char *argv[])
-{
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
-
-   // 1. Parse command-line options.
-   const char *mesh_file = "../../../data/inline-quad.mesh";
-   int order = 1;
-   int delta_order = 1;
-   int ref = 1;
-   bool adjoint_graph_norm = false;
-   bool visualization = true;
-   int iprob = 0;
-   bool static_cond = false;
-   double theta = 0.7;
-
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
-   args.AddOption(&delta_order, "-do", "--delta_order",
-                  "Order enrichment for DPG test space.");
-   args.AddOption(&ref, "-ref", "--num_refinements",
-                  "Number of uniform refinements");    
-   args.AddOption(&theta, "-theta", "--theta_factor",
-                  "Refinement factor");                              
-   args.AddOption(&adjoint_graph_norm, "-graph-norm", "--adjoint-graph-norm",
-                  "-no-graph-norm", "--no-adjoint-graph-norm",
-                  "Enable or disable Adjoint Graph Norm on the test space");   
-   args.AddOption(&iprob, "-prob", "--problem", "Problem case"
-                  " 0: lshape, 1: General");       
-   args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
-                  "--no-static-condensation", "Enable static condensation.");                                            
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      return 1;
-   }
-   if (myid == 0)
-   {
-      args.PrintOptions(cout);
-   }
-
-   if (iprob > 1) { iprob = 1; }
-   prob = (prob_type)iprob;
-
-   if (prob == prob_type::lshape)
-   {
-      mesh_file = "../lshape2.mesh";
-   }
-
-   Mesh mesh(mesh_file, 1, 1);
-   int dim = mesh.Dimension();
-
-   mesh.UniformRefinement();
-
-   mesh.EnsureNCMesh();
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // Define spaces
-   // L2 space for u
-   FiniteElementCollection *u_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *u_fes = new ParFiniteElementSpace(&pmesh,u_fec);
-
-   // Vector L2 space for σ 
-   FiniteElementCollection *sigma_fec = new L2_FECollection(order-1,dim);
-   ParFiniteElementSpace *sigma_fes = new ParFiniteElementSpace(&pmesh,sigma_fec, dim); 
-
-   // H^1/2 space for û 
-   FiniteElementCollection * hatu_fec = new H1_Trace_FECollection(order,dim);
-   ParFiniteElementSpace *hatu_fes = new ParFiniteElementSpace(&pmesh,hatu_fec);
-
-   // H^-1/2 space for σ̂ 
-   FiniteElementCollection * hatsigma_fec = new RT_Trace_FECollection(order-1,dim);   
-   ParFiniteElementSpace *hatsigma_fes = new ParFiniteElementSpace(&pmesh,hatsigma_fec);
-
-   // testspace fe collections
-   int test_order = order+delta_order;
-   FiniteElementCollection * tau_fec = new RT_FECollection(test_order-1, dim);
-   FiniteElementCollection * v_fec = new H1_FECollection(test_order, dim);
-
-   // Coefficients
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient negone(-1.0);
-
-   // Normal equation weak formulation
-   Array<ParFiniteElementSpace * > trial_fes; 
-   Array<FiniteElementCollection * > test_fec; 
-
-   trial_fes.Append(u_fes);
-   trial_fes.Append(sigma_fes);
-   trial_fes.Append(hatu_fes);
-   trial_fes.Append(hatsigma_fes);
-
-   test_fec.Append(tau_fec);
-   test_fec.Append(v_fec);
-
-   ParNormalEquations * a = new ParNormalEquations(trial_fes,test_fec);
-   a->StoreMatrices(true);
-
-   //  -(u,∇⋅τ)
-   a->AddTrialIntegrator(new MixedScalarWeakGradientIntegrator(one),0,0);
-
-   // -(σ,τ) 
-   TransposeIntegrator * mass = new TransposeIntegrator(new VectorFEMassIntegrator(negone));
-   a->AddTrialIntegrator(mass,1,0);
-
-   // (σ,∇ v)
-   TransposeIntegrator * grad = new TransposeIntegrator(new GradientIntegrator(one));
-   a->AddTrialIntegrator(grad,1,1);
-
-   //  <û,τ⋅n>
-   a->AddTrialIntegrator(new NormalTraceIntegrator,2,0);
-
-   // -<σ̂,v> (sign is included in σ̂)
-   a->AddTrialIntegrator(new TraceIntegrator,3,1);
-
-   // test integrators (space-induced norm for H(div) × H1)
-   // (∇⋅τ,∇⋅δτ)
-   a->AddTestIntegrator(new DivDivIntegrator(one),0,0);
-   // (τ,δτ)
-   a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   // (∇v,∇δv)
-   a->AddTestIntegrator(new DiffusionIntegrator(one),1,1);
-   // (v,δv)
-   a->AddTestIntegrator(new MassIntegrator(one),1,1);
-
-   // additional terms for adjoint graph norm
-   if (adjoint_graph_norm)
-   {
-      // -(∇v,δτ) 
-      a->AddTestIntegrator(new MixedVectorGradientIntegrator(negone),1,0);
-      // -(τ,∇δv) 
-      a->AddTestIntegrator(new MixedVectorWeakDivergenceIntegrator(one),0,1);
-      // (τ,δτ)
-      a->AddTestIntegrator(new VectorFEMassIntegrator(one),0,0);
-   }
-   // RHS
-   if (prob == prob_type::general)
-   {
-      a->AddDomainLFIntegrator(new DomainLFIntegrator(one),1);
-   }
-
-   FunctionCoefficient uex(exact);
-   Array<int> elements_to_refine;
-   ParGridFunction hatu_gf;
-
-
-   socketstream u_out;
-   socketstream sigma_out;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      u_out.open(vishost, visport);
-      sigma_out.open(vishost, visport);
-   }
-
-   for (int i = 0; i<ref; i++)
-   {
-      if (static_cond) { a->EnableStaticCondensation(); }
-      a->Assemble();
-
-      Array<int> ess_tdof_list;
-      Array<int> ess_bdr;
-      if (pmesh.bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(pmesh.bdr_attributes.Max());
-         ess_bdr = 1;
-         hatu_fes->GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-      }
-
-      // shift the ess_tdofs
-      for (int i = 0; i < ess_tdof_list.Size(); i++)
-      {
-         ess_tdof_list[i] += u_fes->GetTrueVSize() + sigma_fes->GetTrueVSize();
-      }
-
-      Array<int> offsets(5);
-      offsets[0] = 0;
-      offsets[1] = u_fes->GetVSize();
-      offsets[2] = sigma_fes->GetVSize();
-      offsets[3] = hatu_fes->GetVSize();
-      offsets[4] = hatsigma_fes->GetVSize();
-      offsets.PartialSum();
-      BlockVector x(offsets);
-      x = 0.0;
-      if (prob == prob_type::lshape)
-      {
-         hatu_gf.MakeRef(hatu_fes,x.GetBlock(2));
-         hatu_gf.ProjectBdrCoefficient(uex,ess_bdr);
-      }
-
-      Vector X,B;
-      OperatorPtr Ah;
-      a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-      BlockOperator * A = Ah.As<BlockOperator>();
-
-      BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-      M->owns_blocks = 1;
-      int skip = 0;
-      if (!static_cond)
-      {
-         HypreBoomerAMG * amg0 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(0,0));
-         HypreBoomerAMG * amg1 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(1,1));
-         amg0->SetPrintLevel(0);
-         amg1->SetPrintLevel(0);
-         M->SetDiagonalBlock(0,amg0);
-         M->SetDiagonalBlock(1,amg1);
-         skip=2;
-      }
-      HypreBoomerAMG * amg2 = new HypreBoomerAMG((HypreParMatrix &)A->GetBlock(skip,skip));
-      amg2->SetPrintLevel(0);
-      M->SetDiagonalBlock(skip,amg2);
-      HypreSolver * prec;
-      if (dim == 2)
-      {
-         prec = new HypreAMS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
-      }
-      else
-      {
-         prec = new HypreADS((HypreParMatrix &)A->GetBlock(skip+1,skip+1), hatsigma_fes);
-      }
-      M->SetDiagonalBlock(skip+1,prec);
-
-      CGSolver cg(MPI_COMM_WORLD);
-      cg.SetRelTol(1e-12);
-      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(*M);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete M;
-
-      a->RecoverFEMSolution(X,x);
-
-      Vector & residuals = a->ComputeResidual(x);
-
-      double residual = residuals.Norml2();
-      
-      double maxresidual = residuals.Max(); 
-      double globalresidual = residual * residual; 
-
-      MPI_Allreduce(MPI_IN_PLACE,&maxresidual,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD);
-      MPI_Allreduce(MPI_IN_PLACE,&globalresidual,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-
-      globalresidual = sqrt(globalresidual);
-
-      if (myid == 0)
-      {
-         cout << "Global Residual = " << globalresidual << endl;
-      }
-
-      elements_to_refine.SetSize(0);
-      for (int iel = 0; iel<pmesh.GetNE(); iel++)
-      {
-         if (residuals[iel] > theta * maxresidual)
-         {
-            elements_to_refine.Append(iel);
-         }
-      }
-
-      ParGridFunction u_gf;
-      u_gf.MakeRef(u_fes,x.GetBlock(0));
-
-      ParGridFunction sigma_gf;
-      sigma_gf.MakeRef(sigma_fes,x.GetBlock(1));
-
-      if (visualization)
-      {
-         u_out << "parallel " << num_procs << " " << myid << "\n";
-         u_out.precision(8);
-         u_out << "solution\n" << pmesh << u_gf <<
-                  "window_title 'Numerical u' "
-                  << flush;
-
-         sigma_out << "parallel " << num_procs << " " << myid << "\n";
-         sigma_out.precision(8);
-         sigma_out << "solution\n" << pmesh << sigma_gf <<
-                  "window_title 'Numerical flux' "
-                  << flush;
-      }
-
-
-      if (i == ref-1)
-      {
-         break;
-      }
-
-      pmesh.GeneralRefinement(elements_to_refine);
-
-      for (int i =0; i<trial_fes.Size(); i++)
-      {
-         trial_fes[i]->Update(false);
-      }
-      a->Update();
-   }
-
-   delete a;
-   delete tau_fec;
-   delete v_fec;
-   delete hatsigma_fes;
-   delete hatsigma_fec;
-   delete hatu_fes;
-   delete hatu_fec;
-   delete sigma_fec;
-   delete sigma_fes;
-   delete u_fec;
-   delete u_fes;
-
-   return 0;
-}
@@ -1,176 +0,0 @@
-//                                MFEM primal dpg example for grad-dic problem
-//
-// Compile with: make primal_dpg
-//
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-
-using namespace std;
-using namespace mfem;
-
-// Exact solution, F, and r.h.s., f. See below for implementation.
-void F_exact(const Vector &, Vector &);
-void f_exact(const Vector &, Vector &);
-double freq = 1.0, kappa;
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command line options
-   const char *mesh_file = "../../../data/star.mesh";
-   int order = 1;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use.");
-   args.AddOption(&order, "-o", "--order", "Finite element polynomial degree");
-   args.ParseCheck();
-
-   kappa = freq * M_PI;
-
-
-   // 2. Read the mesh from the given mesh file, and refine once uniformly.
-   Mesh mesh(mesh_file);
-   // mesh.UniformRefinement();
-
-   RT_FECollection fec(order-1, mesh.Dimension());
-   FiniteElementSpace RTfes(&mesh, &fec);
-
-   H1_Trace_FECollection trace_fec(order, mesh.Dimension());
-   FiniteElementSpace H1trace_fes(&mesh, &trace_fec);
-
-   int dim = mesh.Dimension();
-   int test_order = order;
-   if (dim == 2 && (order%2 == 0 || (mesh.MeshGenerator() & 2 && order > 1)))
-   {
-      test_order++;
-   }
-
-   test_order++;
-
-   RT_FECollection test_fec(test_order,mesh.Dimension());
-
-   Array<FiniteElementSpace *> trial_fes; 
-   Array<FiniteElementCollection * > test_fecs; 
-   
-   trial_fes.Append(&RTfes);
-   trial_fes.Append(&H1trace_fes);
-   test_fecs.Append(&test_fec);
-
-
-   GridFunction rt_gf(&RTfes);
-   VectorFunctionCoefficient F(dim, F_exact);
-   rt_gf.ProjectCoefficient(F);
-
-   Vector x(RTfes.GetVSize()+H1trace_fes.GetVSize());
-   x = 0.;
-   x.SetVector(rt_gf,0);
-
-
-   ConstantCoefficient alpha(1.0);
-   ConstantCoefficient beta(1.0);
-   NormalEquations * a = new NormalEquations(trial_fes,test_fecs);
-   a->AddTrialIntegrator(new DivDivIntegrator(alpha),0,0);
-   a->AddTrialIntegrator(new VectorFEMassIntegrator(beta),0,0);
-   a->AddTrialIntegrator(new NormalTraceIntegrator,1,0);
-   a->AddTestIntegrator(new DivDivIntegrator(alpha),0,0);
-   a->AddTestIntegrator(new VectorFEMassIntegrator(beta),0,0);
-
-
-   VectorFunctionCoefficient f(dim, f_exact);
-   a->AddDomainLFIntegrator(new VectorFEDomainLFIntegrator(f),0);
-   a->Assemble();
-
-   Array<int> ess_tdof_list;
-   if (mesh.bdr_attributes.Size())
-   {
-      Array<int> ess_bdr(mesh.bdr_attributes.Max());
-      ess_bdr = 1;
-      RTfes.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
-   }
-
-   Vector X,B;
-
-   OperatorPtr Ah;
-   a->FormLinearSystem(ess_tdof_list,x,Ah,X,B);
-
-   BlockMatrix * A = (BlockMatrix *)(Ah.Ptr());
-   BlockDiagonalPreconditioner * M = new BlockDiagonalPreconditioner(A->RowOffsets());
-   M->owns_blocks = 1;
-   for (int i=0; i<A->NumRowBlocks(); i++)
-   {
-      M->SetDiagonalBlock(i,new UMFPackSolver(A->GetBlock(i,i)));
-   }
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(*M);
-   cg.SetOperator(*A);
-   cg.Mult(B, X);
-
-   delete M;
-
-   a->RecoverFEMSolution(X,x);
-
-   // GridFunction u_gf;
-   double *data = x.GetData();
-   rt_gf.MakeRef(&RTfes,data);
-
-   GridFunction exact_gf(&RTfes);
-   exact_gf.ProjectCoefficient(F);
-
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream solu_sock(vishost, visport);
-   solu_sock.precision(8);
-   solu_sock << "solution\n" << mesh << rt_gf <<
-             "window_title 'Numerical u' "
-             << flush;
-
-   socketstream soltrace_sock(vishost, visport);
-   soltrace_sock.precision(8);
-   soltrace_sock << "solution\n" << mesh << exact_gf <<
-                 "window_title 'Exact' "
-                 << flush;
-
-}
-
-
-// The exact solution (for non-surface meshes)
-void F_exact(const Vector &p, Vector &F)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if F is changed to depend on z
-
-   F(0) = cos(kappa*x)*sin(kappa*y);
-   F(1) = cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      F(2) = 0.0;
-   }
-}
-
-// The right hand side
-void f_exact(const Vector &p, Vector &f)
-{
-   int dim = p.Size();
-
-   double x = p(0);
-   double y = p(1);
-   // double z = (dim == 3) ? p(2) : 0.0; // Uncomment if f is changed to depend on z
-
-   double temp = 1 + 2*kappa*kappa;
-
-   f(0) = temp*cos(kappa*x)*sin(kappa*y);
-   f(1) = temp*cos(kappa*y)*sin(kappa*x);
-   if (dim == 3)
-   {
-      f(2) = 0;
-   }
-}
@@ -1,51 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-3
-1 3 0 1 4 3
-1 3 3 4 7 6
-1 3 1 2 5 4
-
-boundary
-8
-1 1 0 1
-1 1 1 2
-1 1 2 5
-2 1 5 4
-2 1 4 7
-1 1 7 6
-1 1 6 3
-1 1 3 0
-
-vertices
-8
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P1
-VDim: 2
-Ordering: 1
-
-1 1
-1 -0
-1 -1
-0 1
-0 -0
-0 -1
-1 1
-1 -0
@@ -305,6 +305,8 @@ int main(int argc, char *argv[])
         vis_w.precision(8);
         visualize(vis_w, mesh, &x, &w, "Elastic energy density", true);
      }
+      cout << "GLVis visualization paused."
+           << " Press space (in the GLVis window) to resume it.\n";
   }

   double ee0 = oper.ElasticEnergy(x.GetTrueVector());
@@ -351,6 +351,11 @@ int main(int argc, char *argv[])
         vis_w.precision(8);
         visualize(vis_w, pmesh, &x_gf, &w_gf, "Elastic energy density", true);
      }
+      if (myid == 0)
+      {
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
   }

   double ee0 = oper.ElasticEnergy(x_gf);
@@ -1,4 +1,4 @@
-//                    MFEM Example 17 - Parallel Version
+//                       MFEM Example 17 - Parallel Version
 //
 // Compile with: make ex17p
 //
@@ -43,7 +43,7 @@
 #include <sstream>
 #include <iostream>

-// Classes FE_Evolution, RiemannSolver, DomainIntegrator and FaceIntegrator
+// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
 // shared between the serial and parallel version of the example.
 #include "ex18.hpp"

@@ -189,7 +189,7 @@ int main(int argc, char *argv[])
   // 7. Set up the nonlinear form corresponding to the DG discretization of the
   //    flux divergence, and assemble the corresponding mass matrix.
   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new DomainIntegrator(dim));
+   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
   Aflux.Assemble();

   NonlinearForm A(&vfes);
@@ -56,27 +56,6 @@ public:
               const Vector &nor, Vector &flux);
 };

-
-// Constant (in time) mixed bilinear form multiplying the flux grid function.
-// The form is (vec(v), grad(w)) where the trial space = vector L2 space (mesh
-// dim) and test space = scalar L2 space.
-class DomainIntegrator : public BilinearFormIntegrator
-{
-private:
-   Vector shape;
-   DenseMatrix flux;
-   DenseMatrix dshapedr;
-   DenseMatrix dshapedx;
-
-public:
-   DomainIntegrator(const int dim);
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Tr,
-                                       DenseMatrix &elmat);
-};
-
 // Interior face term: <F.n(u),[w]>
 class FaceIntegrator : public NonlinearFormIntegrator
 {
@@ -318,60 +297,6 @@ double RiemannSolver::Eval(const Vector &state1, const Vector &state2,
   return maxE;
 }

-// Implementation of class DomainIntegrator
-DomainIntegrator::DomainIntegrator(const int dim) : flux(num_equation, dim) { }
-
-void DomainIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                              const FiniteElement &test_fe,
-                                              ElementTransformation &Tr,
-                                              DenseMatrix &elmat)
-{
-   // Assemble the form (vec(v), grad(w))
-
-   // Trial space = vector L2 space (mesh dim)
-   // Test space  = scalar L2 space
-
-   const int dof_trial = trial_fe.GetDof();
-   const int dof_test = test_fe.GetDof();
-   const int dim = trial_fe.GetDim();
-
-   shape.SetSize(dof_trial);
-   dshapedr.SetSize(dof_test, dim);
-   dshapedx.SetSize(dof_test, dim);
-
-   elmat.SetSize(dof_test, dof_trial * dim);
-   elmat = 0.0;
-
-   const int maxorder = max(trial_fe.GetOrder(), test_fe.GetOrder());
-   const int intorder = 2 * maxorder;
-   const IntegrationRule *ir = &IntRules.Get(trial_fe.GetGeomType(), intorder);
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      // Calculate the shape functions
-      trial_fe.CalcShape(ip, shape);
-      shape *= ip.weight;
-
-      // Compute the physical gradients of the test functions
-      Tr.SetIntPoint(&ip);
-      test_fe.CalcDShape(ip, dshapedr);
-      Mult(dshapedr, Tr.AdjugateJacobian(), dshapedx);
-
-      for (int d = 0; d < dim; d++)
-      {
-         for (int j = 0; j < dof_test; j++)
-         {
-            for (int k = 0; k < dof_trial; k++)
-            {
-               elmat(j, k + d * dof_trial) += shape(k) * dshapedx(j, d);
-            }
-         }
-      }
-   }
-}
-
 // Implementation of class FaceIntegrator
 FaceIntegrator::FaceIntegrator(RiemannSolver &rsolver_, const int dim) :
   rsolver(rsolver_),
@@ -1,4 +1,4 @@
-//                         MFEM Example 18 - Parallel Version
+//                       MFEM Example 18 - Parallel Version
 //
 // Compile with: make ex18
 //
@@ -43,7 +43,7 @@
 #include <sstream>
 #include <iostream>

-// Classes FE_Evolution, RiemannSolver, DomainIntegrator and FaceIntegrator
+// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
 // shared between the serial and parallel version of the example.
 #include "ex18.hpp"

@@ -219,7 +219,7 @@ int main(int argc, char *argv[])
   // 9. Set up the nonlinear form corresponding to the DG discretization of the
   //    flux divergence, and assemble the corresponding mass matrix.
   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new DomainIntegrator(dim));
+   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
   Aflux.Assemble();

   ParNonlinearForm A(&vfes);
@@ -1,4 +1,4 @@
-//                                MFEM Example 21
+//                       MFEM Example 21 - Parallel Version
 //
 // Compile with: make ex21p
 //
@@ -1,4 +1,4 @@
-//                               MFEM Example 22
+//                                MFEM Example 22
 //
 // Compile with: make ex22
 //
@@ -1,4 +1,4 @@
-//                               MFEM Example 24
+//                                MFEM Example 24
 //
 // Compile with: make ex24
 //
@@ -1,4 +1,4 @@
-//                       MFEM Example 27 - Serial Version
+//                                MFEM Example 27
 //
 // Compile with: make ex27
 //
@@ -1,4 +1,4 @@
-//                      MFEM Example 29 - Parallel Version
+//                       MFEM Example 29 - Parallel Version
 //
 // Compile with: make ex29p
 //
@@ -1,4 +1,4 @@
-//                            MFEM Example 30
+//                                MFEM Example 30
 //
 // Compile with: make ex30
 //
@@ -1,4 +1,4 @@
-//                        MFEM Example 30 - Parallel Version
+//                       MFEM Example 30 - Parallel Version
 //
 // Compile with: make ex30p
 //
@@ -45,9 +45,10 @@ int dim;
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../data/inline-quad.mesh";
@@ -119,7 +120,7 @@ int main(int argc, char *argv[])
   }
   ParFiniteElementSpace fespace(&pmesh, fec);
   HYPRE_Int size = fespace.GlobalTrueVSize();
-   if (mpi.Root()) { cout << "Number of H(Curl) unknowns: " << size << endl; }
+   if (Mpi::Root()) { cout << "Number of H(Curl) unknowns: " << size << endl; }

   // 7. Determine the list of true (i.e. parallel conforming) essential
   //    boundary dofs. In this example, the boundary conditions are defined
@@ -181,7 +182,7 @@ int main(int argc, char *argv[])
   // 12. Solve the system AX=B using PCG with the AMS preconditioner from hypre
   if (use_ams)
   {
-      if (mpi.Root())
+      if (Mpi::Root())
      {
         cout << "Size of linear system: "
              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
@@ -199,7 +200,7 @@ int main(int argc, char *argv[])
   else
 #ifdef MFEM_USE_SUPERLU
   {
-      if (mpi.Root())
+      if (Mpi::Root())
      {
         cout << "Size of linear system: "
              << A.As<HypreParMatrix>()->GetGlobalNumRows() << endl;
@@ -212,7 +213,7 @@ int main(int argc, char *argv[])
   }
 #else
   {
-      if (mpi.Root()) { cout << "No solvers available." << endl; }
+      if (Mpi::Root()) { cout << "No solvers available." << endl; }
      return 1;
   }
 #endif
@@ -224,7 +225,7 @@ int main(int argc, char *argv[])
   // 14. Compute and print the H(Curl) norm of the error.
   {
      double error = sol.ComputeHCurlError(&E, &CurlE);
-      if (mpi.Root())
+      if (Mpi::Root())
      {
         cout << "\n|| E_h - E ||_{H(Curl)} = " << error << '\n' << endl;
      }
@@ -41,9 +41,10 @@ double GetScalarMax(const ParGridFunction &x);
 int main(int argc, char *argv[])
 {
   // 1. Initialize MPI.
-   MPI_Session mpi;
-   int num_procs = mpi.WorldSize();
-   int myid = mpi.WorldRank();
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();

   // 2. Parse command-line options.
   const char *mesh_file = "../data/inline-quad.mesh";
@@ -122,7 +123,7 @@ int main(int argc, char *argv[])
   ParFiniteElementSpace fespace_rt(&pmesh, fec_rt);
   HYPRE_Int size_nd = fespace_nd.GlobalTrueVSize();
   HYPRE_Int size_rt = fespace_rt.GlobalTrueVSize();
-   if (mpi.Root())
+   if (Mpi::Root())
   {
      cout << "Number of H(Curl) unknowns: " << size_nd << endl;
      cout << "Number of H(Div) unknowns: " << size_rt << endl;
@@ -164,7 +165,7 @@ int main(int argc, char *argv[])
         // closed surface.
         a.AddDomainIntegrator(new VectorFEMassIntegrator(epsilon));
         shift = 1.0;
-         if (mpi.Root())
+         if (Mpi::Root())
         {
            cout << "Computing eigenvalues shifted by " << shift << endl;
         }
@@ -287,7 +288,7 @@ int main(int argc, char *argv[])

         for (int i=0; i<nev; i++)
         {
-            if (mpi.Root())
+            if (Mpi::Root())
            {
               cout << "Eigenmode " << i+1 << '/' << nev
                    << ", Lambda = " << eigenvalues[i] - shift << endl;
@@ -408,7 +409,7 @@ int main(int argc, char *argv[])
               MPI_Barrier(MPI_COMM_WORLD);
            }
            char c;
-            if (mpi.Root())
+            if (Mpi::Root())
            {
               cout << "press (q)uit or (c)ontinue --> " << flush;
               cin >> c;
@@ -461,7 +462,7 @@ int main(int argc, char *argv[])

         for (int i=0; i<nev; i++)
         {
-            if (mpi.Root())
+            if (Mpi::Root())
            {
               cout << "Eigenmode " << i+1 << '/' << nev
                    << ", Lambda = " << eigenvalues[i] - shift << endl;
@@ -565,7 +566,7 @@ int main(int argc, char *argv[])
               MPI_Barrier(MPI_COMM_WORLD);
            }
            char c;
-            if (mpi.Root())
+            if (Mpi::Root())
            {
               cout << "press (q)uit or (c)ontinue --> " << flush;
               cin >> c;
@@ -591,7 +592,7 @@ int main(int argc, char *argv[])

         for (int i=0; i<nev; i++)
         {
-            if (mpi.Root())
+            if (Mpi::Root())
            {
               cout << "Eigenmode " << i+1 << '/' << nev
                    << ", Lambda = " << eigenvalues[i] - shift << endl;
@@ -620,7 +621,7 @@ int main(int argc, char *argv[])
            MPI_Barrier(MPI_COMM_WORLD);

            char c;
-            if (mpi.Root())
+            if (Mpi::Root())
            {
               cout << "press (q)uit or (c)ontinue --> " << flush;
               cin >> c;
@@ -0,0 +1,404 @@
+//                                MFEM Example 33
+//
+// Compile with: make ex33
+//
+// Sample runs:  ex33 -m ../data/square-disc.mesh -alpha 0.33 -o 2
+//               ex33 -m ../data/square-disc.mesh -alpha 4.5 -o 3
+//               ex33 -m ../data/star.mesh -alpha 1.4 -o 3
+//               ex33 -m ../data/star.mesh -alpha 0.99 -o 3
+//               ex33 -m ../data/inline-quad.mesh -alpha 0.5 -o 3
+//               ex33 -m ../data/amr-quad.mesh -alpha 1.5 -o 3
+//               ex33 -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
+//               ex33 -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
+//               ex33 -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
+//               ex33 -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
+//
+// Verification runs:
+//    ex33 -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
+//    ex33 -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
+//    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
+//    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
+//
+//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
+//        for all alpha.
+//
+// Description:
+//
+//  In this example we solve the following fractional PDE with MFEM:
+//
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
+//
+//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
+//  N is given by floor(α). By doing so, we obtain
+//
+//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
+//
+//  We first compute the right hand side by solving the integer order PDE
+//
+//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
+//
+//  The remaining FPDE is then given by
+//
+//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
+//
+//  We rely on a rational approximation [2] of the normal linear operator
+//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
+//  and (a-N) in (0,1). We approximate the operator
+//
+//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//
+//  where I is the L2-identity operator and the coefficients c_i and d_i
+//  are generated offline to a prescribed accuracy in a pre-processing step.
+//  We use the triple-A algorithm [1] to generate the rational approximation
+//  that this partial fractional expansion derives from. We then solve M+1
+//  independent integer-order PDEs,
+//
+//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
+//
+//  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
+//
+//    u ≈ Σ_{i=0}^M u_i.
+//
+//  (If alpha is an integer, we stop after the first PDE was solved.)
+//
+// References:
+//
+// [1] Nakatsukasa, Y., Sète, O., & Trefethen, L. N. (2018). The AAA algorithm
+//     for rational approximation. SIAM Journal on Scientific Computing, 40(3),
+//     A1494-A1522.
+//
+// [2] Harizanov, S., Lazarov, R., Margenov, S., Marinov, P., & Pasciak, J.
+//     (2020). Analysis of numerical methods for spectral fractional elliptic
+//     equations based on the best uniform rational approximation. Journal of
+//     Computational Physics, 408, 109285.
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <math.h>
+#include <string>
+
+#include "ex33.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int num_refs = 3;
+   double alpha = 0.5;
+   bool visualization = true;
+   bool verification = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&num_refs, "-r", "--refs",
+                  "Number of uniform refinements");
+   args.AddOption(&alpha, "-alpha", "--alpha",
+                  "Fractional exponent");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
+                  "--no-verification",
+                  "Use sinusoidal function (f) for analytic comparison.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   args.PrintOptions(cout);
+
+   Array<double> coeffs, poles;
+   int progress_steps = 1;
+
+   // 2. Compute the rational expansion coefficients that define the
+   //    integer-order PDEs.
+   const int power_of_laplace = floor(alpha);
+   double exponent_to_approximate = alpha - power_of_laplace;
+   bool integer_order = false;
+   // Check if alpha is an integer or not.
+   if (abs(exponent_to_approximate) > 1e-12)
+   {
+      mfem::out << "Approximating the fractional exponent "
+                << exponent_to_approximate
+                << endl;
+      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
+                                          poles);
+
+      // If the example is build without LAPACK, the exponent_to_approximate
+      // might be modified by the function call above.
+      alpha = exponent_to_approximate + power_of_laplace;
+   }
+   else
+   {
+      integer_order = true;
+      mfem::out << "Treating integer order PDE." << endl;
+   }
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 4. Refine the mesh to increase the resolution.
+   for (int i = 0; i < num_refs; i++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 5. Define a finite element space on the mesh.
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec);
+   cout << "Number of finite element unknowns: "
+        << fespace.GetTrueVSize() << endl;
+
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   Array<int> ess_tdof_list;
+   if (mesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(mesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 7. Define diffusion coefficient, load, and solution GridFunction.
+   auto func = [&alpha](const Vector &x)
+   {
+      double val = 1.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         val *= sin(M_PI*x(i));
+      }
+      return pow(x.Size()*pow(M_PI,2), alpha) * val;
+   };
+   FunctionCoefficient f(func);
+   ConstantCoefficient one(1.0);
+   GridFunction u(&fespace);
+   GridFunction x(&fespace);
+   GridFunction g(&fespace);
+   u = 0.0;
+   x = 0.0;
+   g = 0.0;
+
+   // 8. Prepare for visualization.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   // 9. Set up the linear form b(.) for integer-order PDE solves.
+   LinearForm b(&fespace);
+   if (verification)
+   {
+      // This statement is only relevant for the verification of the code. It
+      // uses a different f such that an analytic solution is known and easy
+      // to compare with the numerical one. The FPDE becomes:
+      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
+      // -> u(x,y) = sin(\pi x) sin(\pi y)
+      b.AddDomainIntegrator(new DomainLFIntegrator(f));
+   }
+   else
+   {
+      b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   }
+   b.Assemble();
+
+   // ------------------------------------------------------------------------
+   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
+   // ------------------------------------------------------------------------
+
+   if (power_of_laplace > 0)
+   {
+      // 10.1 Compute Stiffnes Matrix
+      BilinearForm k(&fespace);
+      k.AddDomainIntegrator(new DiffusionIntegrator(one));
+      k.Assemble();
+
+      // 10.2 Compute Mass Matrix
+      BilinearForm m(&fespace);
+      m.AddDomainIntegrator(new MassIntegrator(one));
+      m.Assemble();
+      SparseMatrix mass;
+      Array<int> empty;
+      m.FormSystemMatrix(empty, mass);
+
+      // 10.3 Form the system of equations
+      Vector B, X;
+      OperatorPtr Op;
+      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
+      GSSmoother M((SparseMatrix&)(*Op));
+
+      mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
+                << " ( f ) " << endl;
+      for (int i = 0; i < power_of_laplace; i++)
+      {
+         // 10.4 Solve the linear system Op X = B (N times).
+         PCG(*Op, M, B, X, 3, 300, 1e-12, 0.0);
+
+         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
+         if (i == power_of_laplace - 1)
+         {
+            // Needed for visualization and solution verification.
+            k.RecoverFEMSolution(X, b, g);
+            if (integer_order && verification)
+            {
+               // For an integer order PDE, g is also our solution u.
+               u+=g;
+            }
+            if (visualization)
+            {
+               socketstream fout;
+               ostringstream oss_f;
+               fout.open(vishost, visport);
+               fout.precision(8);
+               oss_f.str(""); oss_f.clear();
+               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
+                     << power_of_laplace
+                     << " g = f";
+               fout << "solution\n" << mesh << g
+                    << "window_title '" << oss_f.str() << "'" << flush;
+            }
+         }
+
+         // 10.6 Prepare for next iteration (primal / dual space)
+         mass.Mult(X, B);
+         X.SetSubVectorComplement(ess_tdof_list,0.0);
+      }
+
+      // 10.7 Extract solution for the next step. The b now corresponds to the
+      //      function g in the PDE.
+      const SparseMatrix * R = fespace.GetRestrictionMatrix();
+      if (R)
+      {
+         R->MultTranspose(B,b);
+      }
+      else
+      {
+         b = B;
+      }
+   }
+
+   // ------------------------------------------------------------------------
+   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
+   //     up the solutions.
+   // ------------------------------------------------------------------------
+   if (!integer_order)
+   {
+      // Setup visualization.
+      socketstream xout, uout;
+      ostringstream oss_x, oss_u;
+      if (visualization)
+      {
+         xout.open(vishost, visport);
+         xout.precision(8);
+         uout.open(vishost, visport);
+         uout.precision(8);
+      }
+      // Iterate over all expansion coefficient that contribute to the
+      // solution.
+      for (int i = 0; i < coeffs.Size(); i++)
+      {
+         mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                   << " u = " << coeffs[i] << " g " << endl;
+
+
+         // 11.1 Reset GridFunction for integer-order PDE solve.
+         x = 0.0;
+
+         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
+         BilinearForm a(&fespace);
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         ConstantCoefficient d_i(-poles[i]);
+         a.AddDomainIntegrator(new MassIntegrator(d_i));
+         a.Assemble();
+
+         // 11.3 Assemble the bilinear form and the corresponding linear system.
+         OperatorPtr A;
+         Vector B, X;
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+         // 11.4 Solve the linear system A X = B.
+         GSSmoother M((SparseMatrix&)(*A));
+
+         PCG(*A, M, B, X, 3, 300, 1e-12, 0.0);
+
+         // 11.5 Recover the solution as a finite element grid function.
+         a.RecoverFEMSolution(X, b, x);
+
+         // 11.6 Accumulate integer-order PDE solutions.
+         x *= coeffs[i];
+         u += x;
+
+         // 11.7 Send fractional PDE solution to a GLVis server.
+         if (visualization)
+         {
+            oss_x.str(""); oss_x.clear();
+            oss_x << "Step " << progress_steps
+                  << ": Solution of PDE -Δ u + " << -poles[i]
+                  << " u = " << coeffs[i] << " g";
+            xout << "solution\n" << mesh << x
+                 << "window_title '" << oss_x.str() << "'" << flush;
+
+            oss_u.str(""); oss_u.clear();
+            oss_u << "Step " << progress_steps + 1
+                  << ": Solution of fractional PDE (-Δ)^" << alpha
+                  << " u = f";
+            uout << "solution\n" << mesh << u
+                 << "window_title '" << oss_u.str() << "'"
+                 << flush;
+         }
+      }
+   }
+
+   // ------------------------------------------------------------------------
+   // 12. (optional) Verify the solution.
+   // ------------------------------------------------------------------------
+   if (verification)
+   {
+      auto solution = [] (const Vector &x)
+      {
+         double val = 1.0;
+         for (int i=0; i<x.Size(); i++)
+         {
+            val *= sin(M_PI*x(i));
+         }
+         return val;
+      };
+      FunctionCoefficient sol(solution);
+      double l2_error = u.ComputeL2Error(sol);
+
+      string analytic_solution,expected_mesh;
+      switch (dim)
+      {
+         case 1:
+            analytic_solution = "sin(π x)";
+            expected_mesh = "inline_segment.mesh";
+            break;
+         case 2:
+            analytic_solution = "sin(π x) sin(π y)";
+            expected_mesh = "inline_quad.mesh";
+            break;
+         default:
+            analytic_solution = "sin(π x) sin(π y) sin(π z)";
+            expected_mesh = "inline_hex.mesh";
+            break;
+      }
+
+      mfem::out << "\n" << string(80,'=')
+                << "\n\nSolution Verification in "<< dim << "D \n\n"
+                << "Analytic solution : " << analytic_solution << "\n"
+                << "Expected mesh     : " << expected_mesh <<"\n"
+                << "Your mesh         : " << mesh_file << "\n"
+                << "L2 error          : " << l2_error << "\n\n"
+                << string(80,'=') << endl;
+   }
+
+   return 0;
+}
@@ -0,0 +1,401 @@
+//                  MFEM Example 33 - Serial/Parallel Shared Code
+//                      (Implementation of the AAA algorithm)
+//
+//  Here, we implement the triple-A algorithm [1] for the rational approximation
+//  of complex-valued functions,
+//
+//          p(z)/q(z) ≈ f(z).
+//
+//  In this file, we always assume f(z) = z^{-α}. The triple-A algorithm
+//  provides a robust, accurate approximation in rational barycentric form.
+//  This representation must be transformed into a partial fraction
+//  representation in order to be used to solve a spectral FPDE.
+//
+//  More specifically, we first expand the numerator in terms of the zeros of
+//  the rational approximation,
+//
+//          p(z) ∝ Π_i (z - z_i),
+//
+//  and expand the denominator in terms of the poles of the rational
+//  approximation,
+//
+//          q(z) ∝ Π_i (z - p_i).
+//
+//  We then use these zeros and poles to derive the partial fraction expansion
+//
+//          f(z) ≈ p(z)/q(z) = Σ_i c_i / (z - p_i).
+//
+//  [1] Nakatsukasa, Y., Sète, O., & Trefethen, L. N. (2018). The AAA algorithm
+//      for rational approximation. SIAM Journal on Scientific Computing, 40(3),
+//      A1494-A1522.
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <string>
+
+using namespace std;
+using namespace mfem;
+
+/** RationalApproximation_AAA: compute the rational approximation (RA) of data
+    @a val [in] at the set of points @a pt [in].
+
+    @param[in]  val        Vector of data values
+    @param[in]  pt         Vector of sample points
+    @param[in]  tol        Relative tolerance
+    @param[in]  max_order  Maximum number of terms (order) of the RA
+    @param[out] z          Support points of the RA in rational barycentric form
+    @param[out] f          Data values at support points @a z
+    @param[out] w          Weights of the RA in rational barycentric form
+
+    See pg. A1501 of Nakatsukasa et al. [1]. */
+void RationalApproximation_AAA(const Vector &val, const Vector &pt,
+                               Array<double> &z, Array<double> &f, Vector &w,
+                               double tol, int max_order)
+{
+
+   // number of sample points
+   int size = val.Size();
+   MFEM_VERIFY(pt.Size() == size, "size mismatch");
+
+   // Initializations
+   Array<int> J(size);
+   for (int i = 0; i < size; i++) { J[i] = i; }
+   z.SetSize(0);
+   f.SetSize(0);
+
+   DenseMatrix C, Ctemp, A, Am;
+   // auxiliary arrays and vectors
+   Vector f_vec;
+   Array<double> c_i;
+
+   // mean of the value vector
+   Vector R(val.Size());
+   double mean_val = val.Sum()/size;
+
+   for (int i = 0; i<R.Size(); i++) { R(i) = mean_val; }
+
+   for (int k = 0; k < max_order; k++)
+   {
+      // select next support point
+      int idx = 0;
+      double tmp_max = 0;
+      for (int j = 0; j < size; j++)
+      {
+         double tmp = abs(val(j)-R(j));
+         if (tmp > tmp_max)
+         {
+            tmp_max = tmp;
+            idx = j;
+         }
+      }
+
+      // Append support points and data values
+      z.Append(pt(idx));
+      f.Append(val(idx));
+
+      // Update index vector
+      J.DeleteFirst(idx);
+
+      // next column in Cauchy matrix
+      Array<double> C_tmp(size);
+      for (int j = 0; j < size; j++)
+      {
+         C_tmp[j] = 1.0/(pt(j)-pt(idx));
+      }
+      c_i.Append(C_tmp);
+      int h_C = C_tmp.Size();
+      int w_C = k+1;
+      C.UseExternalData(c_i.GetData(),h_C,w_C);
+
+      Ctemp = C;
+
+      f_vec.SetDataAndSize(f.GetData(),f.Size());
+      Ctemp.InvLeftScaling(val);
+      Ctemp.RightScaling(f_vec);
+
+      A.SetSize(C.Height(), C.Width());
+      Add(C,Ctemp,-1.0,A);
+      A.LeftScaling(val);
+
+      int h_Am = J.Size();
+      int w_Am = A.Width();
+      Am.SetSize(h_Am,w_Am);
+      for (int i = 0; i<h_Am; i++)
+      {
+         int ii = J[i];
+         for (int j = 0; j<w_Am; j++)
+         {
+            Am(i,j) = A(ii,j);
+         }
+      }
+
+#ifdef MFEM_USE_LAPACK
+      DenseMatrixSVD svd(Am,false,true);
+      svd.Eval(Am);
+      DenseMatrix &v = svd.RightSingularvectors();
+      v.GetRow(k,w);
+#else
+      mfem_error("Compiled without LAPACK");
+#endif
+
+      // N = C*(w.*f); D = C*w; % numerator and denominator
+      Vector aux(w);
+      aux *= f_vec;
+      Vector N(C.Height()); // Numerator
+      C.Mult(aux,N);
+      Vector D(C.Height()); // Denominator
+      C.Mult(w,D);
+
+      R = val;
+      for (int i = 0; i<J.Size(); i++)
+      {
+         int ii = J[i];
+         R(ii) = N(ii)/D(ii);
+      }
+
+      Vector verr(val);
+      verr-=R;
+
+      if (verr.Normlinf() <= tol*val.Normlinf()) { break; }
+   }
+}
+
+/** ComputePolesAndZeros: compute the @a poles [out] and @a zeros [out] of the
+    rational function f(z) = C p(z)/q(z) from its ration barycentric form.
+
+    @param[in]  z      Support points in rational barycentric form
+    @param[in]  f      Data values at support points @a z
+    @param[in]  w      Weights in rational barycentric form
+    @param[out] poles  Array of poles (roots of p(z))
+    @param[out] zeros  Array of zeros (roots of q(z))
+    @param[out] scale  Scaling constant in f(z) = C p(z)/q(z)
+
+    See pg. A1501 of Nakatsukasa et al. [1]. */
+void ComputePolesAndZeros(const Vector &z, const Vector &f, const Vector &w,
+                          Array<double> & poles, Array<double> & zeros, double &scale)
+{
+   // Initialization
+   poles.SetSize(0);
+   zeros.SetSize(0);
+
+   // Compute the poles
+   int m = w.Size();
+   DenseMatrix B(m+1); B = 0.;
+   DenseMatrix E(m+1); E = 0.;
+   for (int i = 1; i<=m; i++)
+   {
+      B(i,i) = 1.;
+      E(0,i) = w(i-1);
+      E(i,0) = 1.;
+      E(i,i) = z(i-1);
+   }
+
+#ifdef MFEM_USE_LAPACK
+   DenseMatrixGeneralizedEigensystem eig1(E,B);
+   eig1.Eval();
+   Vector & evalues = eig1.EigenvaluesRealPart();
+   for (int i = 0; i<evalues.Size(); i++)
+   {
+      if (IsFinite(evalues(i)))
+      {
+         poles.Append(evalues(i));
+      }
+   }
+#else
+   mfem_error("Compiled without LAPACK");
+#endif
+   // compute the zeros
+   B = 0.;
+   E = 0.;
+   for (int i = 1; i<=m; i++)
+   {
+      B(i,i) = 1.;
+      E(0,i) = w(i-1) * f(i-1);
+      E(i,0) = 1.;
+      E(i,i) = z(i-1);
+   }
+
+#ifdef MFEM_USE_LAPACK
+   DenseMatrixGeneralizedEigensystem eig2(E,B);
+   eig2.Eval();
+   evalues = eig2.EigenvaluesRealPart();
+   for (int i = 0; i<evalues.Size(); i++)
+   {
+      if (IsFinite(evalues(i)))
+      {
+         zeros.Append(evalues(i));
+      }
+   }
+#else
+   mfem_error("Compiled without LAPACK");
+#endif
+
+   scale = w * f / w.Sum();
+}
+
+/** PartialFractionExpansion: compute the partial fraction expansion of the
+    rational function f(z) = Σ_i c_i / (z - p_i) from its @a poles [in] and
+    @a zeros [in].
+
+    @param[in]  poles   Array of poles (same as p_i above)
+    @param[in]  zeros   Array of zeros
+    @param[in]  scale   Scaling constant
+    @param[out] coeffs  Coefficients c_i */
+void PartialFractionExpansion(double scale, Array<double> & poles,
+                              Array<double> & zeros, Array<double> & coeffs)
+{
+   int psize = poles.Size();
+   int zsize = zeros.Size();
+   coeffs.SetSize(psize);
+   coeffs = scale;
+
+   // Note: C p(z)/q(z) = Σ_i c_i / (z - p_i) results in an system of equations
+   // where the N unknowns are the coefficients c_i. After multiplying the
+   // system with q(z), the coefficients c_i can be computed analytically by
+   // choosing N values for z. Choosing z_j = = p_j diagonalizes the system and
+   // one can obtain an analytic form for the c_i coefficients. The result is
+   // implemented in the code block below.
+
+   for (int i=0; i<psize; i++)
+   {
+      double tmp_numer=1.0;
+      for (int j=0; j<zsize; j++)
+      {
+         tmp_numer *= poles[i]-zeros[j];
+      }
+
+      double tmp_denom=1.0;
+      for (int k=0; k<psize; k++)
+      {
+         if (k != i) { tmp_denom *= poles[i]-poles[k]; }
+      }
+      coeffs[i] *= tmp_numer / tmp_denom;
+   }
+}
+
+
+/** ComputePartialFractionApproximation: compute a rational approximation (RA)
+    in partial fraction form, e.g., f(z) ≈ Σ_i c_i / (z - p_i), from sampled
+    values of the function f(z) = z^{-a}, 0 < a < 1.
+
+    @param[in]  alpha         Exponent a in f(z) = z^-a
+    @param[in] lmax, npoints  f(z) is uniformly sampled @a npoints times in the
+                              interval [ 0, @a lmax ]
+    @param[in]  tol           Relative tolerance
+    @param[in]  max_order     Maximum number of terms (order) of the RA
+    @param[out] coeffs        Coefficients c_i
+    @param[out] poles         Poles p_i
+
+    NOTES: When MFEM is not built with LAPACK support, only @a alpha = 0.33,
+           0.5, and 0.99 are possible. In this case, if @a alpha != 0.33 and
+           @a alpha != 0.99, then @a alpha = 0.5 is used by default.
+
+   See pg. A1501 of Nakatsukasa et al. [1]. */
+void ComputePartialFractionApproximation(double & alpha,
+                                         Array<double> & coeffs, Array<double> & poles,
+                                         double lmax = 1000.,
+                                         double tol=1e-10, int npoints = 1000,
+                                         int max_order = 100)
+{
+   MFEM_VERIFY(alpha < 1., "alpha must be less than 1");
+   MFEM_VERIFY(alpha > 0., "alpha must be greater than 0");
+   MFEM_VERIFY(npoints > 2, "npoints must be greater than 2");
+   MFEM_VERIFY(lmax > 0,  "lmin must be greater than 0");
+   MFEM_VERIFY(tol > 0,  "tol must be greater than 0");
+
+   bool print_warning = true;
+#ifdef MFEM_USE_MPI
+   if ((Mpi::IsInitialized() && !Mpi::Root())) { print_warning = false; }
+#endif
+
+#ifndef MFEM_USE_LAPACK
+   if (print_warning)
+   {
+      mfem::out
+            << "\n" << string(80, '=')
+            << "\nMFEM is compiled without LAPACK."
+            << "\nUsing precomputed values for PartialFractionApproximation."
+            << "\nOnly alpha = 0.33, 0.5, and 0.99 are available."
+            << "\nThe default is alpha = 0.5.\n" << string(80, '=') << "\n"
+            << endl;
+   }
+   const double eps = std::numeric_limits<double>::epsilon();
+
+   if (abs(alpha - 0.33) < eps)
+   {
+      coeffs = Array<double> ({1.821898e+03, 9.101221e+01, 2.650611e+01,
+                               1.174937e+01, 6.140444e+00, 3.441713e+00,
+                               1.985735e+00, 1.162634e+00, 6.891560e-01,
+                               4.111574e-01, 2.298736e-01});
+      poles = Array<double> ({-4.155583e+04, -2.956285e+03, -8.331715e+02,
+                              -3.139332e+02, -1.303448e+02, -5.563385e+01,
+                              -2.356255e+01, -9.595516e+00, -3.552160e+00,
+                              -1.032136e+00, -1.241480e-01});
+   }
+   else if (abs(alpha - 0.99) < eps)
+   {
+      coeffs = Array<double>({2.919591e-02, 1.419750e-02, 1.065798e-02,
+                              9.395094e-03, 8.915329e-03, 8.822991e-03,
+                              9.058247e-03, 9.814521e-03, 1.180396e-02,
+                              1.834554e-02, 9.840482e-01});
+      poles = Array<double> ({-1.069683e+04, -1.769370e+03, -5.718374e+02,
+                              -2.242095e+02, -9.419132e+01, -4.031012e+01,
+                              -1.701525e+01, -6.810088e+00, -2.382810e+00,
+                              -5.700059e-01, -1.384324e-03});
+   }
+   else
+   {
+      if (abs(alpha - 0.5) > eps && print_warning)
+      {
+         alpha = 0.5;
+      }
+      coeffs = Array<double>({2.290262e+02, 2.641819e+01, 1.005566e+01,
+                              5.390411e+00, 3.340725e+00, 2.211205e+00,
+                              1.508883e+00, 1.049474e+00, 7.462709e-01,
+                              5.482686e-01, 4.232510e-01, 3.578967e-01});
+      poles = Array<double>({-3.168211e+04, -3.236077e+03, -9.868287e+02,
+                             -3.945597e+02, -1.738889e+02, -7.925178e+01,
+                             -3.624992e+01, -1.629196e+01, -6.982956e+00,
+                             -2.679984e+00, -7.782607e-01, -7.649166e-02});
+   }
+
+   if (print_warning)
+   {
+      mfem::out << "=> Using precomputed values for alpha = "
+                << alpha << "\n" << std::endl;
+   }
+
+
+   return;
+#endif
+
+   Vector x(npoints);
+   Vector val(npoints);
+   double dx = lmax / (double)(npoints-1);
+   for (int i = 0; i<npoints; i++)
+   {
+      x(i) = dx * (double)i;
+      val(i) = pow(x(i),1.-alpha);
+   }
+
+   // Apply triple-A algorithm to f(x) = x^{1-a}
+   Array<double> z, f;
+   Vector w;
+   RationalApproximation_AAA(val,x,z,f,w,tol,max_order);
+
+   Vector vecz, vecf;
+   vecz.SetDataAndSize(z.GetData(), z.Size());
+   vecf.SetDataAndSize(f.GetData(), f.Size());
+
+   // Compute poles and zeros for RA of f(x) = x^{1-a}
+   double scale;
+   Array<double> zeros;
+   ComputePolesAndZeros(vecz, vecf, w, poles, zeros, scale);
+
+   // Remove the zero at x=0, thus, delivering a RA for f(x) = x^{-a}
+   zeros.DeleteFirst(0.0);
+
+   // Compute partial fraction approximation of f(x) = x^{-a}
+   PartialFractionExpansion(scale, poles, zeros, coeffs);
+}
@@ -0,0 +1,441 @@
+//                       MFEM Example 33 - Parallel Version
+//
+// Compile with: make ex33p
+//
+// Sample runs:  mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 0.33 -o 2
+//               mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 4.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 1.4 -o 3
+//               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 0.99 -o 3
+//               mpirun -np 4 ex33p -m ../data/inline-quad.mesh -alpha 0.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/amr-quad.mesh -alpha 1.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3 -r 2
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
+//               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
+//               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
+//
+// Verification runs:
+//    mpirun -np 4 ex33p -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
+
+//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
+//        for all alpha.
+//
+// Description:
+//
+//  In this example we solve the following fractional PDE with MFEM:
+//
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
+//
+//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
+//  N is given by floor(α). By doing so, we obtain
+//
+//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
+//
+//  We first compute the right hand side by solving the integer order PDE
+//
+//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
+//
+//  The remaining FPDE is then given by
+//
+//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
+//
+//  We rely on a rational approximation [2] of the normal linear operator
+//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
+//  and (a-N) in (0,1). We approximate the operator
+//
+//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//
+//  where I is the L2-identity operator and the coefficients c_i and d_i
+//  are generated offline to a prescribed accuracy in a pre-processing step.
+//  We use the triple-A algorithm [1] to generate the rational approximation
+//  that this partial fractional expansion derives from. We then solve M+1
+//  independent integer-order PDEs,
+//
+//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
+//
+//  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
+//
+//    u ≈ Σ_{i=0}^M u_i.
+//
+//  (If alpha is an integer, we stop after the first PDE was solved.)
+//
+// References:
+//
+// [1] Nakatsukasa, Y., Sète, O., & Trefethen, L. N. (2018). The AAA algorithm
+//     for rational approximation. SIAM Journal on Scientific Computing, 40(3),
+//     A1494-A1522.
+//
+// [2] Harizanov, S., Lazarov, R., Margenov, S., Marinov, P., & Pasciak, J.
+//     (2020). Analysis of numerical methods for spectral fractional elliptic
+//     equations based on the best uniform rational approximation. Journal of
+//     Computational Physics, 408, 109285.
+//
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <math.h>
+#include <string>
+
+#include "ex33.hpp"
+
+using namespace std;
+using namespace mfem;
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI.
+   Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   const char *mesh_file = "../data/star.mesh";
+   int order = 1;
+   int num_refs = 3;
+   double alpha = 0.5;
+   bool visualization = true;
+   bool verification = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&order, "-o", "--order",
+                  "Finite element order (polynomial degree) or -1 for"
+                  " isoparametric space.");
+   args.AddOption(&num_refs, "-r", "--refs",
+                  "Number of uniform refinements");
+   args.AddOption(&alpha, "-alpha", "--alpha",
+                  "Fractional exponent");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
+                  "--no-verification",
+                  "Use sinusoidal function (f) for analytic comparison.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(cout);
+      return 1;
+   }
+   if (Mpi::Root())
+   {
+      args.PrintOptions(cout);
+   }
+
+   Array<double> coeffs, poles;
+   int progress_steps = 1;
+
+   // 2. Compute the rational expansion coefficients that define the
+   //    integer-order PDEs.
+   const int power_of_laplace = floor(alpha);
+   double exponent_to_approximate = alpha - power_of_laplace;
+   bool integer_order = false;
+   // Check if alpha is an integer or not.
+   if (abs(exponent_to_approximate) > 1e-12)
+   {
+      if (Mpi::Root())
+      {
+         mfem::out << "Approximating the fractional exponent "
+                   << exponent_to_approximate
+                   << endl;
+      }
+      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
+                                          poles);
+
+      // If the example is build without LAPACK, the exponent_to_approximate
+      // might be modified by the function call above.
+      alpha = exponent_to_approximate + power_of_laplace;
+   }
+   else
+   {
+      integer_order = true;
+      if (Mpi::Root())
+      {
+         mfem::out << "Treating integer order PDE." << endl;
+      }
+   }
+
+   // 3. Read the mesh from the given mesh file.
+   Mesh mesh(mesh_file, 1, 1);
+   int dim = mesh.Dimension();
+
+   // 4. Refine the mesh to increase the resolution.
+   for (int i = 0; i < num_refs; i++)
+   {
+      mesh.UniformRefinement();
+   }
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 5. Define a finite element space on the mesh.
+   H1_FECollection fec(order, dim);
+   ParFiniteElementSpace fespace(&pmesh, &fec);
+   if (Mpi::Root())
+   {
+      cout << "Number of finite element unknowns: "
+           << fespace.GetTrueVSize() << endl;
+   }
+
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   Array<int> ess_tdof_list;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> ess_bdr(pmesh.bdr_attributes.Max());
+      ess_bdr = 1;
+      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
+   }
+
+   // 7. Define diffusion coefficient, load, and solution GridFunction.
+   auto func = [&alpha](const Vector &x)
+   {
+      double val = 1.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         val *= sin(M_PI*x(i));
+      }
+      return pow(x.Size()*pow(M_PI,2), alpha) * val;
+   };
+   FunctionCoefficient f(func);
+   ConstantCoefficient one(1.0);
+   ParGridFunction u(&fespace);
+   ParGridFunction x(&fespace);
+   ParGridFunction g(&fespace);
+   u = 0.0;
+   x = 0.0;
+   g = 0.0;
+
+   // 8. Prepare for visualization.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+
+   // 9. Set up the linear form b(.) for integer-order PDE solves.
+   ParLinearForm b(&fespace);
+   if (verification)
+   {
+      // This statement is only relevant for the verification of the code. It
+      // uses a different f such that an analytic solution is known and easy
+      // to compare with the numerical one. The FPDE becomes:
+      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
+      // -> u(x,y) = sin(\pi x) sin(\pi y)
+      b.AddDomainIntegrator(new DomainLFIntegrator(f));
+   }
+   else
+   {
+      b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   }
+   b.Assemble();
+
+   // ------------------------------------------------------------------------
+   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
+   // ------------------------------------------------------------------------
+
+   if (power_of_laplace > 0)
+   {
+      // 10.1 Compute Stiffnes Matrix
+      ParBilinearForm k(&fespace);
+      k.AddDomainIntegrator(new DiffusionIntegrator(one));
+      k.Assemble();
+
+      // 10.2 Compute Mass Matrix
+      ParBilinearForm m(&fespace);
+      m.AddDomainIntegrator(new MassIntegrator(one));
+      m.Assemble();
+      HypreParMatrix mass;
+      Array<int> empty;
+      m.FormSystemMatrix(empty, mass);
+
+      // 10.3 Form the system of equations
+      Vector B, X;
+      OperatorPtr Op;
+      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
+      HypreBoomerAMG prec;
+      prec.SetPrintLevel(-1);
+      CGSolver cg(MPI_COMM_WORLD);
+      cg.SetRelTol(1e-12);
+      cg.SetMaxIter(2000);
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(prec);
+      cg.SetOperator(*Op);
+
+      if (Mpi::Root())
+      {
+         mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
+                   << " ( f ) " << endl;
+      }
+      for (int i = 0; i < power_of_laplace; i++)
+      {
+         // 10.4 Solve the linear system Op X = B (N times).
+         cg.Mult(B, X);
+         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
+         if (i == power_of_laplace - 1)
+         {
+            // Needed for visualization and solution verification.
+            k.RecoverFEMSolution(X, b, g);
+            if (integer_order && verification)
+            {
+               // For an integer order PDE, g is also our solution u.
+               u+=g;
+            }
+            if (visualization)
+            {
+               socketstream fout;
+               ostringstream oss_f;
+               fout.open(vishost, visport);
+               fout.precision(8);
+               oss_f.str(""); oss_f.clear();
+               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
+                     << power_of_laplace
+                     << " g = f";
+               fout << "parallel " << num_procs << " " << myid << "\n"
+                    << "solution\n" << pmesh << g
+                    << "window_title '" << oss_f.str() << "'" << flush;
+            }
+         }
+
+         // 10.6 Prepare for next iteration (primal / dual space)
+         mass.Mult(X, B);
+         X.SetSubVectorComplement(ess_tdof_list,0.0);
+      }
+
+      // 10.7 Extract solution for the next step. The b now corresponds to the
+      //      function g in the PDE.
+      const SparseMatrix* rm = fespace.GetRestrictionMatrix();
+      rm->MultTranspose(B, b);
+   }
+
+   // ------------------------------------------------------------------------
+   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
+   //     up the solutions.
+   // ------------------------------------------------------------------------
+   if (!integer_order)
+   {
+      // Setup visualization.
+      socketstream xout, uout;
+      ostringstream oss_x, oss_u;
+      if (visualization)
+      {
+         xout.open(vishost, visport);
+         xout.precision(8);
+         uout.open(vishost, visport);
+         uout.precision(8);
+      }
+      // Iterate over all expansion coefficient that contribute to the
+      // solution.
+      for (int i = 0; i < coeffs.Size(); i++)
+      {
+         if (Mpi::Root())
+         {
+            mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                      << " u = " << coeffs[i] << " g " << endl;
+         }
+
+         // 11.1 Reset GridFunction for integer-order PDE solve.
+         x = 0.0;
+
+         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
+         ParBilinearForm a(&fespace);
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         ConstantCoefficient d_i(-poles[i]);
+         a.AddDomainIntegrator(new MassIntegrator(d_i));
+         a.Assemble();
+
+         // 11.3 Assemble the bilinear form and the corresponding linear system.
+         OperatorPtr A;
+         Vector B, X;
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+         // 11.4 Solve the linear system A X = B.
+         HypreBoomerAMG prec;
+         prec.SetPrintLevel(-1);
+
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-12);
+         cg.SetMaxIter(2000);
+         cg.SetPrintLevel(3);
+         cg.SetPreconditioner(prec);
+         cg.SetOperator(*A);
+         cg.Mult(B, X);
+
+         // 11.5 Recover the solution as a finite element grid function.
+         a.RecoverFEMSolution(X, b, x);
+
+         // 11.6 Accumulate integer-order PDE solutions.
+         x *= coeffs[i];
+         u += x;
+
+         // 11.7 Send fractional PDE solution to a GLVis server.
+         if (visualization)
+         {
+            oss_x.str(""); oss_x.clear();
+            oss_x << "Step " << progress_steps
+                  << ": Solution of PDE -Δ u + " << -poles[i]
+                  << " u = " << coeffs[i] << " g";
+            xout << "parallel " << num_procs << " " << myid << "\n"
+                 << "solution\n" << pmesh << x
+                 << "window_title '" << oss_x.str() << "'" << flush;
+
+            oss_u.str(""); oss_u.clear();
+            oss_u << "Step " << progress_steps + 1
+                  << ": Solution of fractional PDE (-Δ)^" << alpha
+                  << " u = f";
+            uout << "parallel " << num_procs << " " << myid << "\n"
+                 << "solution\n" << pmesh << u
+                 << "window_title '" << oss_u.str() << "'"
+                 << flush;
+         }
+      }
+   }
+
+   // ------------------------------------------------------------------------
+   // 12. (optional) Verify the solution.
+   // ------------------------------------------------------------------------
+   if (verification)
+   {
+      auto solution = [] (const Vector &x)
+      {
+         double val = 1.0;
+         for (int i=0; i<x.Size(); i++)
+         {
+            val *= sin(M_PI*x(i));
+         }
+         return val;
+      };
+      FunctionCoefficient sol(solution);
+      double l2_error = u.ComputeL2Error(sol);
+
+      if (Mpi::Root())
+      {
+         string analytic_solution,expected_mesh;
+         switch (dim)
+         {
+            case 1:
+               analytic_solution = "sin(π x)";
+               expected_mesh = "inline_segment.mesh";
+               break;
+            case 2:
+               analytic_solution = "sin(π x) sin(π y)";
+               expected_mesh = "inline_quad.mesh";
+               break;
+            default:
+               analytic_solution = "sin(π x) sin(π y) sin(π z)";
+               expected_mesh = "inline_hex.mesh";
+               break;
+         }
+
+         mfem::out << "\n" << string(80,'=')
+                   << "\n\nSolution Verification in "<< dim << "D \n\n"
+                   << "Analytic solution : " << analytic_solution << "\n"
+                   << "Expected mesh     : " << expected_mesh <<"\n"
+                   << "Your mesh         : " << mesh_file << "\n"
+                   << "L2 error          : " << l2_error << "\n\n"
+                   << string(80,'=') << endl;
+      }
+   }
+
+   return 0;
+}
@@ -55,6 +55,7 @@ int main(int argc, char *argv[])
   bool pa = false;
   const char *device_config = "cpu";
   int max_dofs = 50000;
+   bool LSZZ = false;
   bool visualization = true;

   OptionsParser args(argc, argv);
@@ -68,6 +69,9 @@ int main(int argc, char *argv[])
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&max_dofs, "-md", "--max-dofs",
                  "Stop after reaching this many degrees of freedom.");
+   args.AddOption(&LSZZ, "-ls", "--ls-zz", "-no-ls",
+                  "--no-ls-zz",
+                  "Switch to least-squares ZZ estimator.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -151,15 +155,29 @@ int main(int argc, char *argv[])
   //     recover a smoothed flux (gradient) that is subtracted from the element
   //     flux to get an error indicator. We need to supply the space for the
   //     smoothed flux: an (H1)^sdim (i.e., vector-valued) space is used here.
-   FiniteElementSpace flux_fespace(&mesh, &fec, sdim);
-   ZienkiewiczZhuEstimator estimator(*integ, x, flux_fespace);
-   estimator.SetAnisotropic();
+   ErrorEstimator *estimator{nullptr};
+
+   if (LSZZ)
+   {
+      estimator = new LSZienkiewiczZhuEstimator(*integ, x);
+      if (dim == 3 && mesh.GetElementType(0) != Element::HEXAHEDRON)
+      {
+         dynamic_cast<LSZienkiewiczZhuEstimator *>
+         (estimator)->SetTichonovRegularization();
+      }
+   }
+   else
+   {
+      auto flux_fes = new FiniteElementSpace(&mesh, &fec, sdim);
+      estimator = new ZienkiewiczZhuEstimator(*integ, x, flux_fes);
+      dynamic_cast<ZienkiewiczZhuEstimator *>(estimator)->SetAnisotropic();
+   }

   // 11. A refiner selects and refines elements based on a refinement strategy.
   //     The strategy here is to refine elements with errors larger than a
   //     fraction of the maximum element error. Other strategies are possible.
   //     The refiner will call the given error estimator.
-   ThresholdRefiner refiner(estimator);
+   ThresholdRefiner refiner(*estimator);
   refiner.SetTotalErrorFraction(0.7);

   // 12. The main AMR loop. In each iteration we solve the problem on the
@@ -256,5 +274,6 @@ int main(int argc, char *argv[])
      b.Update();
   }

+   delete estimator;
   return 0;
 }
@@ -72,8 +72,8 @@ int main(int argc, char *argv[])
   //    largest number that gives a final mesh with no more than 10,000
   //    elements.
   {
-      int ref_levels = 1;
-      // (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
+      int ref_levels =
+         (int)floor(log(10000./mesh->GetNE())/log(2.)/dim);
      for (int l = 0; l < ref_levels; l++)
      {
         mesh->UniformRefinement();
@@ -147,8 +147,6 @@ int main(int argc, char *argv[])
   F.AddDomainIntegrator(new DomainLFIntegrator(one));
   F.Assemble();

-
-
   // 7. Set up the mixed bilinear form for the primal trial unknowns, B0,
   //    the mixed bilinear form for the interfacial unknowns, Bhat,
   //    the inverse stiffness matrix on the discontinuous test space, Sinv,
@@ -189,17 +187,10 @@ int main(int argc, char *argv[])
   // 8. Set up the 1x2 block Least Squares DPG operator, B = [B0  Bhat],
   //    the normal equation operator, A = B^t Sinv B, and
   //    the normal equation right-hand-size, b = B^t Sinv F.
-   // BlockOperator B(offsets_test, offsets);
-
-   BlockMatrix B(offsets_test, offsets);
+   BlockOperator B(offsets_test, offsets);
   B.SetBlock(0,0,&matB0);
   B.SetBlock(0,1,&matBhat);
-   SparseMatrix * Bh = B.CreateMonolithic();
-
-   SparseMatrix * A = RAP(*Bh, matSinv, *Bh);
-
-
-   // RAPOperator A(B, matSinv, B);
+   RAPOperator A(B, matSinv, B);
   {
      Vector SinvF(s_test);
      matSinv.Mult(F,SinvF);
@@ -243,20 +234,7 @@ int main(int argc, char *argv[])
   // 10. Solve the normal equation system using the PCG iterative solver.
   //     Check the weighted norm of residual for the DPG least square problem.
   //     Wrap the primal variable in a GridFunction for visualization purposes.
-
-
-   // PCG(*A, P, b, x, 1, 200, 1e-12, 0.0);
-
-   GSSmoother M(*A);
-
-   CGSolver cg;
-   cg.SetRelTol(1e-12);
-   cg.SetMaxIter(2000);
-   cg.SetPrintLevel(3);
-   cg.SetPreconditioner(M);
-   cg.SetOperator(*A);
-   cg.Mult(b, x);
-
+   PCG(A, P, b, x, 1, 200, 1e-12, 0.0);

   {
      Vector LSres(s_test);
@@ -23,10 +23,10 @@ MFEM_LIB_FILE = mfem_is_not_built

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31
+  ex31 ex33
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p
+  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
  ex24p ex25p ex26p
@@ -58,6 +58,9 @@ endif
 ifeq ($(MFEM_USE_SUPERLU),YES)
   SUBDIRS += superlu
 endif
+ifeq ($(MFEM_USE_MOONOLITH),YES)
+   SUBDIRS += moonolith
+endif
 ifeq ($(MFEM_USE_CALIPER),YES)
   SUBDIRS += caliper
 endif
@@ -88,8 +91,11 @@ $(SUBDIRS_TPRINT):

 # Additional dependencies
 ex18: $(SRC)ex18.hpp
+ex33: $(SRC)ex33.hpp
+
 ifeq ($(MFEM_USE_MPI),YES)
 ex18p: $(SRC)ex18.hpp
+ex33p: $(SRC)ex33.hpp
 endif

 MFEM_TESTS = EXAMPLES
@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+set(MOONOLITH_EXAMPLES_SRCS)
+list(APPEND MOONOLITH_EXAMPLES_SRCS ex1.cpp)
+
+if (MFEM_USE_MPI)
+  list(APPEND MOONOLITH_EXAMPLES_SRCS ex1p.cpp ex2p.cpp)
+endif()
+
+# Include the source directory where mfem.hpp and mfem-performance.hpp are.
+include_directories(BEFORE ${PROJECT_BINARY_DIR})
+
+# Add "test_moonolith" target, see below.
+add_custom_target(test_moonolith
+  ${CMAKE_CTEST_COMMAND} -R moonolith USES_TERMINAL)
+
+# Add one executable per cpp file, adding "moonolith_" as prefix. Sets
+# "test_moonolith" as a target that depends on the given examples.
+set(PFX moonolith_)
+add_mfem_examples(MOONOLITH_EXAMPLES_SRCS ${PFX} "" test_moonolith)
+
+# Testing.
+# The MOONOLITH tests can be run separately using the target "test_moonolith"
+# which builds the examples and runs:
+#   ctest -R moonolith
+
+# Add the tests: one test per source file.
+foreach(SRC_FILE ${MOONOLITH_EXAMPLES_SRCS})
+  get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
+  string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
+  string(TOUPPER ${TEST_NAME} UP_TEST_NAME)
+  set(TEST_NAME ${PFX}${TEST_NAME})
+
+  set(THIS_TEST_OPTIONS "-no-vis")
+  list(APPEND THIS_TEST_OPTIONS ${${UP_TEST_NAME}_TEST_OPTS})
+  # message(STATUS "Test ${TEST_NAME} options: ${THIS_TEST_OPTIONS}")
+
+  if (NOT (${TEST_NAME} MATCHES ".*p$"))
+    add_test(NAME ${TEST_NAME}_ser
+      COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
+  else()
+    add_test(NAME ${TEST_NAME}_np=4
+      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} 4
+      ${MPIEXEC_PREFLAGS}
+      $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS}
+      ${MPIEXEC_POSTFLAGS})
+  endif()
+endforeach()
@@ -0,0 +1,18 @@
+                    Finite Element Discretization Library
+                                   __
+                       _ __ ___   / _|  ___  _ __ ___
+                      | '_ ` _ \ | |_  / _ \| '_ ` _ \
+                      | | | | | ||  _||  __/| | | | | |
+                      |_| |_| |_||_|   \___||_| |_| |_|
+
+                               https://mfem.org
+
+This directory contains modifications of the example codes that illustrate the
+use of MFEM features based on ParMoonolith library for variational information
+transfer between non-matching meshes.
+
+To build these examples, make sure that MFEM is configured with the option
+"MFEM_USE_MOONOLITH = YES".
+
+For the parallel version add also the options MFEM_USE_MPI = YES", and
+MFEM_USE_METIS = YES". See the top-level INSTALL file for details.
@@ -0,0 +1,229 @@
+//                         MFEM + Moonolith Example 1
+//
+// Compile with: make ex1
+//
+// Moonolith sample runs:
+//   ex1
+//   ex1 --source_refinements 1 --dest_refinements 2
+//   ex1 --source_refinements 1 --dest_refinements 2 --use_vector_fe
+//   ex1 -s ../../data/inline-hex.mesh -d ../../data/inline-tet.mesh
+//
+// Description:  This example code demonstrates the use of MFEM for transferring
+//               discrete fields from one finite element mesh to another. The
+//               meshes can be of arbitrary shape and completely unrelated with
+//               each other. This feature can be used for implementing immersed
+//               domain methods for fluid-structure interaction or general
+//               multi-physics applications.
+//
+//               This particular example is only for serial runtimes.
+
+#include "example_utils.hpp"
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+int main(int argc, char *argv[])
+{
+   // Init transfer library context
+   InitTransfer(argc, argv);
+
+   const char *source_mesh_file = "../../data/inline-tri.mesh";
+   const char *destination_mesh_file = "../../data/inline-quad.mesh";
+
+   int src_n_refinements = 0;
+   int dest_n_refinements = 0;
+   int source_fe_order = 1;
+   int dest_fe_order = 1;
+   bool visualization = true;
+   bool use_vector_fe = false;
+   bool verbose = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&source_mesh_file, "-s", "--source_mesh",
+                  "Mesh file to use for src.");
+   args.AddOption(&destination_mesh_file, "-d", "--destination_mesh",
+                  "Mesh file to use for dest.");
+   args.AddOption(&src_n_refinements, "-sr", "--source_refinements",
+                  "Number of src refinements");
+   args.AddOption(&dest_n_refinements, "-dr", "--dest_refinements",
+                  "Number of dest refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&source_fe_order, "-so", "--source_fe_order",
+                  "Order of the src finite elements");
+   args.AddOption(&dest_fe_order, "-do", "--dest_fe_order",
+                  "Order of the dest finite elements");
+   args.AddOption(&verbose, "-verb", "--verbose", "--no-verb", "--no-verbose",
+                  "Enable/Disable verbose output");
+   args.AddOption(&use_vector_fe, "-vfe", "--use_vector_fe", "-no-vfe",
+                  "--no-vector_fe", "Use vector finite elements");
+   args.Parse();
+   check_options(args);
+
+   shared_ptr<Mesh> src_mesh, dest_mesh;
+
+   ifstream imesh;
+
+   imesh.open(destination_mesh_file);
+   if (imesh)
+   {
+      dest_mesh = make_shared<Mesh>(imesh, 1, 1);
+      imesh.close();
+   }
+   else
+   {
+      mfem::err << "WARNING: Destination mesh file not found: "
+                << destination_mesh_file << "\n"
+                << "Using default 2D quad mesh.";
+
+      dest_mesh = make_shared<Mesh>(4, 4, Element::QUADRILATERAL);
+   }
+
+   const int dim = dest_mesh->Dimension();
+
+   Vector box_min(dim), box_max(dim), range(dim);
+   dest_mesh->GetBoundingBox(box_min, box_max);
+   range = box_max;
+   range -= box_min;
+
+   imesh.open(source_mesh_file);
+
+   if (imesh)
+   {
+      src_mesh = make_shared<Mesh>(imesh, 1, 1);
+      imesh.close();
+   }
+   else
+   {
+      mfem::err << "WARNING: Source mesh file not found: " << source_mesh_file
+                << "\n"
+                << "Using default box mesh.\n";
+
+      if (dim == 2)
+      {
+         src_mesh =
+            make_shared<Mesh>(4, 4, Element::TRIANGLE, 1, range[0], range[1]);
+      }
+      else if (dim == 3)
+      {
+         src_mesh = make_shared<Mesh>(4, 4, 4, Element::TETRAHEDRON, 1, range[0],
+                                      range[1], range[2]);
+      }
+
+      for (int i = 0; i < src_mesh->GetNV(); ++i)
+      {
+         double *v = src_mesh->GetVertex(i);
+
+         for (int d = 0; d < dim; ++d)
+         {
+            v[d] += box_min[d];
+         }
+      }
+   }
+
+   for (int i = 0; i < src_n_refinements; ++i)
+   {
+      src_mesh->UniformRefinement();
+   }
+
+   for (int i = 0; i < dest_n_refinements; ++i)
+   {
+      dest_mesh->UniformRefinement();
+   }
+
+   shared_ptr<FiniteElementCollection> src_fe_coll, dest_fe_coll;
+
+   if (use_vector_fe)
+   {
+      src_fe_coll =
+         make_shared<RT_FECollection>(source_fe_order, src_mesh->Dimension());
+      dest_fe_coll =
+         make_shared<RT_FECollection>(dest_fe_order, dest_mesh->Dimension());
+   }
+   else
+   {
+      src_fe_coll =
+         make_shared<L2_FECollection>(source_fe_order, src_mesh->Dimension());
+      dest_fe_coll =
+         make_shared<L2_FECollection>(dest_fe_order, dest_mesh->Dimension());
+   }
+
+   auto src_fe =
+      make_shared<FiniteElementSpace>(src_mesh.get(), src_fe_coll.get());
+
+   auto dest_fe =
+      make_shared<FiniteElementSpace>(dest_mesh.get(), dest_fe_coll.get());
+
+   GridFunction src_fun(src_fe.get());
+   GridFunction dest_fun(dest_fe.get());
+   src_fun = 1.0;
+
+   // To be used with standard fe
+   FunctionCoefficient coeff(example_fun);
+
+   // To be used with vector fe
+   VectorFunctionCoefficient vector_coeff(dim, &vector_fun);
+
+   if (use_vector_fe)
+   {
+      src_fun.ProjectCoefficient(vector_coeff);
+      src_fun.Update();
+   }
+   else
+   {
+      src_fun.ProjectCoefficient(coeff);
+      src_fun.Update();
+   }
+
+   dest_fun = 0.0;
+   dest_fun.Update();
+
+   MortarAssembler assembler(src_fe, dest_fe);
+   assembler.SetVerbose(verbose);
+
+   if (use_vector_fe)
+   {
+      assembler.AddMortarIntegrator(make_shared<VectorL2MortarIntegrator>());
+   }
+   else
+   {
+      assembler.AddMortarIntegrator(make_shared<L2MortarIntegrator>());
+   }
+
+   if (assembler.Transfer(src_fun, dest_fun))
+   {
+      if (visualization)
+      {
+         dest_fun.Update();
+
+         double src_err = 0;
+         double dest_err = 0;
+
+         if (use_vector_fe)
+         {
+            src_err = src_fun.ComputeL2Error(vector_coeff);
+            dest_err = dest_fun.ComputeL2Error(vector_coeff);
+         }
+         else
+         {
+            src_err = src_fun.ComputeL2Error(coeff);
+            dest_err = dest_fun.ComputeL2Error(coeff);
+         }
+
+         mfem::out << "l2 error: src: " << src_err << ", dest: " << dest_err
+                   << std::endl;
+
+         plot(*src_mesh, src_fun, "source");
+         plot(*dest_mesh, dest_fun, "destination");
+      }
+   }
+   else
+   {
+      mfem::out << "No intersection -> no transfer!" << std::endl;
+   }
+
+   // Finalize transfer library context
+   return FinalizeTransfer();
+}
@@ -0,0 +1,258 @@
+//                   MFEM + Moonolith Example 1 (parallel version)
+//
+// Compile with: make ex1p
+//
+// Moonolith sample runs:
+//   mpirun -np 4 ex1p
+//   mpirun -np 4 ex1p --source_refinements 1 --dest_refinements 2
+//   mpirun -np 4 ex1p -s ../../data/inline-hex.mesh -d ../../data/inline-tet.mesh
+//
+// Description:  This example code demonstrates the use of MFEM for transferring
+//               discrete fields from one finite element mesh to another. The
+//               meshes can be of arbitrary shape and completely unrelated with
+//               each other. This feature can be used for implementing immersed
+//               domain methods for fluid-structure interaction or general
+//               multi-physics applications.
+//
+//               This particular example is for parallel runtimes. Vector FE is
+//               an experimental feature in parallel.
+
+#include "example_utils.hpp"
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+void destination_transform(const Vector &x, Vector &x_new)
+{
+   x_new = x;
+   // x_new *= 0.5;
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Init(&argc, &argv);
+
+   int num_procs, rank;
+
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   // Init transfer library context, with MPI handled outside the library
+   InitTransfer(argc, argv, MPI_COMM_WORLD);
+
+   const char *source_mesh_file = "../../data/inline-tri.mesh";
+   const char *destination_mesh_file = "../../data/inline-quad.mesh";
+
+   int src_n_refinements = 0;
+   int dest_n_refinements = 0;
+   int source_fe_order = 1;
+   int dest_fe_order = 1;
+   bool visualization = true;
+   bool use_vector_fe = false;
+   bool verbose = false;
+   bool assemble_mass_and_coupling_together = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&source_mesh_file, "-s", "--source_mesh",
+                  "Mesh file to use for src.");
+   args.AddOption(&destination_mesh_file, "-d", "--destination_mesh",
+                  "Mesh file to use for dest.");
+   args.AddOption(&src_n_refinements, "-sr", "--source_refinements",
+                  "Number of src refinements");
+   args.AddOption(&dest_n_refinements, "-dr", "--dest_refinements",
+                  "Number of dest refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&source_fe_order, "-so", "--source_fe_order",
+                  "Order of the src finite elements");
+   args.AddOption(&dest_fe_order, "-do", "--dest_fe_order",
+                  "Order of the dest finite elements");
+   args.AddOption(&verbose, "-verb", "--verbose", "--no-verb", "--no-verbose",
+                  "Enable/Disable verbose output");
+   args.AddOption(&use_vector_fe, "-vfe", "--use_vector_fe", "-no-vfe",
+                  "--no-vector_fe", "Use vector finite elements (Experimental)");
+   args.AddOption(&assemble_mass_and_coupling_together, "-act",
+                  "--assemble_mass_and_coupling_together", "-no-act",
+                  "--no-assemble_mass_and_coupling_together",
+                  "Assemble mass and coupling operators together (better for non-affine elements)");
+   args.Parse();
+   check_options(args);
+
+   shared_ptr<Mesh> src_mesh, dest_mesh;
+
+   ifstream imesh;
+
+   imesh.open(destination_mesh_file);
+   if (imesh)
+   {
+      dest_mesh = make_shared<Mesh>(imesh, 1, 1);
+      imesh.close();
+   }
+   else
+   {
+      if (rank == 0)
+         mfem::err << "WARNING: Destination mesh file not found: "
+                   << destination_mesh_file << "\n"
+                   << "Using default 2D quad mesh.";
+
+      dest_mesh = make_shared<Mesh>(4, 4, Element::QUADRILATERAL);
+   }
+
+   const int dim = dest_mesh->Dimension();
+
+   dest_mesh->Transform(&destination_transform);
+
+   Vector box_min(dim), box_max(dim), range(dim);
+   dest_mesh->GetBoundingBox(box_min, box_max);
+   range = box_max;
+   range -= box_min;
+
+   imesh.open(source_mesh_file);
+
+   if (imesh)
+   {
+      src_mesh = make_shared<Mesh>(imesh, 1, 1);
+      imesh.close();
+   }
+   else
+   {
+      if (rank == 0)
+         mfem::err << "WARNING: Source mesh file not found: " << source_mesh_file
+                   << "\n"
+                   << "Using default box mesh.\n";
+
+      if (dim == 2)
+      {
+         src_mesh =
+            make_shared<Mesh>(4, 4, Element::TRIANGLE, 1, range[0], range[1]);
+      }
+      else if (dim == 3)
+      {
+         src_mesh = make_shared<Mesh>(4, 4, 4, Element::TETRAHEDRON, 1, range[0],
+                                      range[1], range[2]);
+      }
+
+      for (int i = 0; i < src_mesh->GetNV(); ++i)
+      {
+         double *v = src_mesh->GetVertex(i);
+
+         for (int d = 0; d < dim; ++d)
+         {
+            v[d] += box_min[d];
+         }
+      }
+   }
+
+   for (int i = 0; i < src_n_refinements; ++i)
+   {
+      src_mesh->UniformRefinement();
+   }
+
+   for (int i = 0; i < dest_n_refinements; ++i)
+   {
+      dest_mesh->UniformRefinement();
+   }
+
+   auto p_src_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *src_mesh);
+   auto p_dest_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *dest_mesh);
+
+   shared_ptr<FiniteElementCollection> src_fe_coll, dest_fe_coll;
+
+   if (use_vector_fe)
+   {
+      src_fe_coll =
+         make_shared<RT_FECollection>(source_fe_order, src_mesh->Dimension());
+      dest_fe_coll =
+         make_shared<RT_FECollection>(dest_fe_order, dest_mesh->Dimension());
+   }
+   else
+   {
+      src_fe_coll =
+         make_shared<L2_FECollection>(source_fe_order, src_mesh->Dimension());
+      dest_fe_coll =
+         make_shared<L2_FECollection>(dest_fe_order, dest_mesh->Dimension());
+   }
+
+   auto src_fe =
+      make_shared<ParFiniteElementSpace>(p_src_mesh.get(), src_fe_coll.get());
+
+   auto dest_fe =
+      make_shared<ParFiniteElementSpace>(p_dest_mesh.get(), dest_fe_coll.get());
+
+   ParGridFunction src_fun(src_fe.get());
+
+   // To be used with standard fe
+   FunctionCoefficient coeff(example_fun);
+
+   // To be used with vector fe
+   VectorFunctionCoefficient vector_coeff(dim, &vector_fun);
+
+   if (use_vector_fe)
+   {
+      src_fun.ProjectCoefficient(vector_coeff);
+      src_fun.Update();
+   }
+   else
+   {
+      src_fun.ProjectCoefficient(coeff);
+      src_fun.Update();
+   }
+
+   ParGridFunction dest_fun(dest_fe.get());
+   dest_fun = 0.0;
+   dest_fun.Update();
+
+   ParMortarAssembler assembler(src_fe, dest_fe);
+   assembler.SetAssembleMassAndCouplingTogether(
+      assemble_mass_and_coupling_together);
+   assembler.SetVerbose(verbose);
+
+   if (use_vector_fe)
+   {
+      assembler.AddMortarIntegrator(make_shared<VectorL2MortarIntegrator>());
+   }
+   else
+   {
+      assembler.AddMortarIntegrator(make_shared<L2MortarIntegrator>());
+   }
+
+   if (assembler.Transfer(src_fun, dest_fun))
+   {
+
+      if (visualization)
+      {
+         double src_err = 0;
+         double dest_err = 0;
+
+         if (use_vector_fe)
+         {
+            src_err = src_fun.ComputeL2Error(vector_coeff);
+            dest_err = dest_fun.ComputeL2Error(vector_coeff);
+         }
+         else
+         {
+            src_err = src_fun.ComputeL2Error(coeff);
+            dest_err = dest_fun.ComputeL2Error(coeff);
+         }
+
+         if (rank == 0)
+         {
+            mfem::out << "l2 error: src: " << src_err << ", dest: " << dest_err
+                      << std::endl;
+         }
+
+         plot(*p_src_mesh, src_fun, "source");
+         plot(*p_dest_mesh, dest_fun, "destination");
+      }
+   }
+   else
+   {
+      mfem::out << "No intersection no transfer!" << std::endl;
+   }
+
+   // Finalize transfer library context
+   FinalizeTransfer();
+   return MPI_Finalize();
+}
@@ -0,0 +1,202 @@
+//                  MFEM + Moonolith Example 2 (parallel version)
+//
+// Compile with: make ex2p
+//
+// Moonolith sample runs:
+//   mpirun -np 4 ex2p
+//   mpirun -np 4 ex2p --source_refinements 1 --dest_refinements 2
+//   mpirun -np 4 ex2p -s ../../data/inline-hex.mesh -d ../../data/inline-tet.mesh
+//
+// Description:  This example code demonstrates the use of MFEM for transferring
+//               discrete fields from one finite element mesh to another. The
+//               meshes can be of arbitrary shape and completely unrelated with
+//               each other. This feature can be used for implementing immersed
+//               domain methods for fluid-structure interaction or general
+//               multi-physics applications.
+//
+//               This particular example concerns discontinuous Galerkin FEM with
+//               adaptive mesh refinement for parallel runtimes.
+
+#include "example_utils.hpp"
+#include "mfem.hpp"
+
+using namespace mfem;
+using namespace std;
+
+void destination_transform(const Vector &x, Vector &x_new)
+{
+   x_new = x;
+   // x_new *= .5;
+}
+
+int main(int argc, char *argv[])
+{
+   MPI_Init(&argc, &argv);
+
+   int num_procs, rank;
+
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+   // Init transfer library context, with MPI handled outside the library
+   InitTransfer(argc, argv, MPI_COMM_WORLD);
+
+   const char *source_mesh_file = "../../data/inline-tri.mesh";
+   const char *destination_mesh_file = "../../data/inline-quad.mesh";
+
+   int src_n_refinements = 0;
+   int dest_n_refinements = 0;
+
+   // Source fe order has to be greater or equal than destination order
+   int source_fe_order = 1;
+   int dest_fe_order = 0;
+   bool visualization = true;
+   bool verbose = false;
+   int max_iterations = 30000;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&source_mesh_file, "-s", "--source_mesh",
+                  "Mesh file to use for src.");
+   args.AddOption(&destination_mesh_file, "-d", "--destination_mesh",
+                  "Mesh file to use for dest.");
+   args.AddOption(&src_n_refinements, "-sr", "--source_refinements",
+                  "Number of src refinements");
+   args.AddOption(&dest_n_refinements, "-dr", "--dest_refinements",
+                  "Number of dest refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&source_fe_order, "-so", "--source_fe_order",
+                  "Order of the src finite elements");
+   args.AddOption(&dest_fe_order, "-do", "--dest_fe_order",
+                  "Order of the dest finite elements");
+   args.AddOption(&verbose, "-verb", "--verbose", "--no-verb", "--no-verbose",
+                  "Enable/Disable verbose output");
+   args.AddOption(&max_iterations, "-m", "--max_iterations",
+                  "Max number of solver iterations");
+   args.Parse();
+   check_options(args);
+
+   if (source_fe_order == 0 &&  dest_fe_order != 0)
+   {
+      mfem::out <<
+                "Source fe order should not be 0 unless destination fe order is also 0!\n";
+
+      FinalizeTransfer();
+      return MPI_Finalize();
+   }
+
+   ifstream imesh(source_mesh_file);
+   shared_ptr<Mesh> src_mesh, dest_mesh;
+   if (imesh)
+   {
+      src_mesh = make_shared<Mesh>(imesh, 1, 1);
+      imesh.close();
+   }
+   else
+   {
+      if (rank == 0)
+         mfem::err << "WARNING: Source mesh file not found: " << source_mesh_file
+                   << "\n"
+                   << "Using default 2D triangle mesh.";
+      src_mesh = make_shared<Mesh>(4, 4, Element::TRIANGLE);
+   }
+
+   imesh.open(destination_mesh_file);
+   if (imesh)
+   {
+      dest_mesh = make_shared<Mesh>(imesh, 1, 1);
+      imesh.close();
+   }
+   else
+   {
+      if (rank == 0)
+         mfem::err << "WARNING: Destination mesh file not found: "
+                   << destination_mesh_file << "\n"
+                   << "Using default 2D quad mesh.";
+      dest_mesh = make_shared<Mesh>(4, 4, Element::QUADRILATERAL);
+   }
+
+   dest_mesh->Transform(&destination_transform);
+
+   for (int i = 0; i < src_n_refinements; ++i)
+   {
+      src_mesh->UniformRefinement();
+   }
+
+   for (int i = 0; i < dest_n_refinements; ++i)
+   {
+      dest_mesh->UniformRefinement();
+   }
+
+   src_mesh->EnsureNCMesh();
+   dest_mesh->EnsureNCMesh();
+
+   {
+      for (int l = 0; l < 4; l++)
+      {
+         src_mesh->RandomRefinement(0.1); // 10% probability
+      }
+   }
+
+   {
+      for (int l = 0; l < 4; l++)
+      {
+         dest_mesh->RandomRefinement(0.1); // 10% probability
+      }
+   }
+
+   auto p_src_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *src_mesh);
+   auto p_dest_mesh = make_shared<ParMesh>(MPI_COMM_WORLD, *dest_mesh);
+
+   auto src_fe_coll =
+      make_shared<DG_FECollection>(source_fe_order, p_src_mesh->Dimension());
+   auto src_fe =
+      make_shared<ParFiniteElementSpace>(p_src_mesh.get(), src_fe_coll.get());
+
+   auto dest_fe_coll =
+      make_shared<DG_FECollection>(dest_fe_order, p_dest_mesh->Dimension());
+   auto dest_fe =
+      make_shared<ParFiniteElementSpace>(p_dest_mesh.get(), dest_fe_coll.get());
+
+   ParGridFunction src_fun(src_fe.get());
+   FunctionCoefficient coeff(example_fun);
+   make_fun(*src_fe, coeff, src_fun);
+
+   ParGridFunction dest_fun(dest_fe.get());
+   dest_fun = 0.0;
+   dest_fun.Update();
+
+   ParMortarAssembler assembler(src_fe, dest_fe);
+   assembler.SetVerbose(verbose);
+   assembler.SetMaxSolverIterations(max_iterations);
+
+   assembler.AddMortarIntegrator(make_shared<L2MortarIntegrator>());
+   if (assembler.Transfer(src_fun, dest_fun))
+   {
+      if (visualization)
+      {
+
+         const double src_err = src_fun.ComputeL2Error(coeff);
+         const double dest_err = dest_fun.ComputeL2Error(coeff);
+
+         if (rank == 0)
+         {
+            mfem::out << "l2 error: src: " << src_err << ", dest: " << dest_err
+                      << std::endl;
+         }
+
+         plot(*p_src_mesh, src_fun, "source");
+         plot(*p_dest_mesh, dest_fun, "destination");
+      }
+   }
+   else
+   {
+      mfem::out << "Transfer failed! Use --verbose option for diagnostic!" <<
+                std::endl;
+   }
+
+   // Finalize transfer library context
+   FinalizeTransfer();
+   return MPI_Finalize();
+}
@@ -0,0 +1,96 @@
+#include <algorithm>
+#include <assert.h>
+#include <cstdlib>
+#include <memory>
+
+#ifdef MFEM_USE_MPI
+#include <mpi.h>
+#endif // MFEM_USE_MPI
+
+#include "mfem.hpp"
+
+inline void check_options(mfem::OptionsParser &args)
+{
+   using namespace std;
+   using namespace mfem;
+
+   int rank = 0;
+#ifdef MFEM_USE_MPI
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+#endif // MFEM_USE_MPI
+
+   if (!args.Good())
+   {
+      if (rank == 0)
+      {
+         args.PrintUsage(cout);
+      }
+
+#ifdef MFEM_USE_MPI
+      MPI_Finalize();
+      MPI_Abort(MPI_COMM_WORLD, 1);
+#else
+      abort();
+#endif // MFEM_USE_MPI
+   }
+
+   if (rank == 0)
+   {
+      args.PrintOptions(cout);
+   }
+}
+
+inline void make_fun(mfem::FiniteElementSpace &fe, mfem::Coefficient &c,
+                     mfem::GridFunction &f)
+{
+   using namespace std;
+   using namespace mfem;
+
+   f.SetSpace(&fe);
+   f.ProjectCoefficient(c);
+   f.Update();
+}
+
+inline double example_fun(const mfem::Vector &x)
+{
+   using namespace std;
+   using namespace mfem;
+
+   const int n = x.Size();
+   double ret = 0;
+   for (int k = 0; k < n; ++k)
+   {
+      ret += x(k) * x(k);
+   }
+
+   return sqrt(ret);
+}
+
+void vector_fun(const mfem::Vector &x, mfem::Vector &f)
+{
+   const double n = x.Norml2();
+   f.SetSize(x.Size());
+   f = n;
+}
+
+inline void plot(mfem::Mesh &mesh, mfem::GridFunction &x, std::string title)
+{
+   using namespace std;
+   using namespace mfem;
+
+   int num_procs = 1, rank = 0;
+
+#ifdef MFEM_USE_MPI
+   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+   MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
+#endif // MFEM_USE_MPI
+
+   char vishost[] = "localhost";
+   int visport = 19916;
+   socketstream sol_sock(vishost, visport);
+   sol_sock << "parallel " << num_procs << " " << rank << "\n";
+   sol_sock.precision(8);
+   sol_sock << "solution\n" << mesh << x
+            << "window_title '"<< title << "'\n" << flush;
+   sol_sock << flush;
+}
@@ -10,17 +10,19 @@
 # CONTRIBUTING.md for details.

 # Use the MFEM build directory
-MFEM_DIR ?= ../../..
-MFEM_BUILD_DIR ?= ../../..
-SRC = $(if $(MFEM_DIR:../../..=),$(MFEM_DIR)/examples/dpg_tests/grad-div,)
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = $(if $(MFEM_DIR:../..=),$(MFEM_DIR)/examples/moonolith/,)
 CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+# Use the MFEM install directory
+# MFEM_INSTALL_DIR = ../../mfem
+# CONFIG_MK = $(MFEM_INSTALL_DIR)/share/mfem/config.mk

 MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

-SEQ_EXAMPLES = primal_dpg 
-PAR_EXAMPLES = 
-
+SEQ_EXAMPLES = ex1
+PAR_EXAMPLES = ex1p ex2p
 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
 else
@@ -29,7 +31,7 @@ endif

 .SUFFIXES:
 .SUFFIXES: .o .cpp .mk
-.PHONY: all clean clean-build clean-exec
+.PHONY: all clean clean-build

 # Remove built-in rule
 %: %.cpp
@@ -38,22 +40,39 @@ endif
 %: $(SRC)%.cpp $(MFEM_LIB_FILE) $(CONFIG_MK)
 	$(MFEM_CXX) $(MFEM_FLAGS) $< -o $@ $(MFEM_LIBS)

-all: $(EXAMPLES) 
+all: $(EXAMPLES)
+
+ifeq ($(MFEM_USE_MOONOLITH),NO)
+$(EXAMPLES):
+	$(error MFEM is not configured with MOONOLITH)
+endif

 MFEM_TESTS = EXAMPLES
 include $(MFEM_TEST_MK)

 # Testing: Parallel vs. serial runs
-RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
+RUN_MPI_NP = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP)
+RUN_MPI = $(RUN_MPI_NP) $(MFEM_MPI_NP)
+SERIAL_NAME := Serial MOONOLITH example
+PARALLEL_NAME := Parallel MOONOLITH example
 %-test-par: %
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example)
+	@$(call mfem-test,$<, $(RUN_MPI), $(PARALLEL_NAME))
 %-test-seq: %
-	@$(call mfem-test,$<,, Serial example)
+	@$(call mfem-test,$<,, $(SERIAL_NAME))

-clean: clean-build clean-exec 
+# Testing: Example-specific execution options:
+ex1-test-par: ex1
+	@$(call mfem-test,$<, $(RUN_MPI_NP) 1, $(PARALLEL_NAME))
+
+# Testing: "test" target and mfem-test* variables are defined in config/test.mk
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+clean: clean-build

 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints

-clean-exec:
@@ -58,17 +58,29 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  ceed/algebraic.cpp
-  ceed/full-assembly.cpp
-  ceed/solvers-atpmg.cpp
-  ceed/convection.cpp
-  ceed/diffusion.cpp
-  ceed/nlconvection.cpp
-  ceed/mass.cpp
-  ceed/operator.cpp
-  ceed/util.cpp
+  ceed/interface/basis.cpp
+  ceed/interface/restriction.cpp
+  ceed/interface/operator.cpp
+  ceed/interface/util.cpp
+  ceed/integrators/convection/convection.cpp
+  ceed/integrators/diffusion/diffusion.cpp
+  ceed/integrators/nlconvection/nlconvection.cpp
+  ceed/integrators/mass/mass.cpp
+  ceed/solvers/algebraic.cpp
+  ceed/solvers/full-assembly.cpp
+  ceed/solvers/solvers-atpmg.cpp
  linearform.cpp
+  linearform_ext.cpp
  lininteg.cpp
+  lininteg_domain.cpp
+  lininteg_domain_grad.cpp
+  lor/lor.cpp
+  lor/lor_ads.cpp
+  lor/lor_ams.cpp
+  lor/lor_batched.cpp
+  lor/lor_h1.cpp
+  lor/lor_nd.cpp
+  lor/lor_rt.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -118,7 +130,6 @@ set(SRCS
  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
-  lor.cpp
  )

 set(HDRS
@@ -149,19 +160,31 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  ceed/algebraic.hpp
-  ceed/full-assembly.hpp
-  ceed/solvers-atpmg.hpp
-  ceed/coefficient.hpp
-  ceed/convection.hpp
-  ceed/diffusion.hpp
-  ceed/integrator.hpp
-  ceed/mass.hpp
-  ceed/nlconvection.hpp
-  ceed/operator.hpp
-  ceed/util.hpp
+  ceed/interface/basis.hpp
+  ceed/interface/integrator.hpp
+  ceed/interface/interface.hpp
+  ceed/interface/operator.hpp
+  ceed/interface/restriction.hpp
+  ceed/interface/util.hpp
+  ceed/integrators/convection/convection.hpp
+  ceed/integrators/diffusion/diffusion.hpp
+  ceed/integrators/mass/mass.hpp
+  ceed/integrators/nlconvection/nlconvection.hpp
+  ceed/interface/coefficient.hpp
+  ceed/solvers/algebraic.hpp
+  ceed/solvers/full-assembly.hpp
+  ceed/solvers/solvers-atpmg.hpp
  linearform.hpp
+  linearform_ext.hpp
  lininteg.hpp
+  lor/lor.hpp
+  lor/lor_ads.hpp
+  lor/lor_ams.hpp
+  lor/lor_batched.hpp
+  lor/lor_h1.hpp
+  lor/lor_nd.hpp
+  lor/lor_rt.hpp
+  lor/lor_util.hpp
  multigrid.hpp
  nonlinearform.hpp
  nonlinearform_ext.hpp
@@ -188,7 +211,6 @@ set(HDRS
  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
-  lor.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -100,6 +100,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)

   // Copy the pointers to the integrators
   domain_integs = bf->domain_integs;
+   domain_integs_marker = bf->domain_integs_marker;

   boundary_integs = bf->boundary_integs;
   boundary_integs_marker = bf->boundary_integs_marker;
@@ -112,6 +113,138 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)
   AllocMat();
 }

+BilinearForm::BilinearForm(BilinearForm &&other)
+   : Matrix(other.fes->GetVSize()), mat(other.mat), mat_e(other.mat_e),
+     fes(other.fes), assembly(other.assembly), batch(other.batch),
+     ext(other.ext), sequence(other.sequence), extern_bfs(other.extern_bfs),
+     element_matrices(other.element_matrices), static_cond(other.static_cond),
+     hybridization(other.hybridization), diag_policy(other.diag_policy),
+     precompute_sparsity(other.precompute_sparsity)
+{
+   // We swap stored integrators and markers with the moved nonlinear form
+   mfem::Swap(domain_integs, other.domain_integs);
+   mfem::Swap(domain_integs_marker, other.domain_integs_marker);
+   mfem::Swap(boundary_integs, other.boundary_integs);
+   mfem::Swap(boundary_integs_marker, other.boundary_integs_marker);
+   mfem::Swap(interior_face_integs, other.interior_face_integs);
+   mfem::Swap(boundary_face_integs, other.boundary_face_integs);
+   mfem::Swap(boundary_face_integs_marker, other.boundary_face_integs_marker);
+
+   /// Leave the moved nonlinear form in a state as if it was just constructed
+   /// with fes
+   other.sequence = fes->GetSequence();
+   other.mat = nullptr;
+   other.mat_e = nullptr;
+   other.extern_bfs = 0;
+   other.element_matrices = nullptr;
+   other.static_cond = nullptr;
+   other.hybridization = nullptr;
+   other.precompute_sparsity = 0;
+   other.diag_policy = DIAG_KEEP;
+
+   other.assembly = AssemblyLevel::LEGACY;
+   other.batch = 1;
+   other.ext = nullptr;
+}
+
+BilinearForm& BilinearForm::operator=(BilinearForm &&other)
+{
+   if (this != &other)
+   {
+      /// Cleanup current bilinear form first
+      delete mat_e;
+      delete mat;
+      delete element_matrices;
+      delete static_cond;
+      delete hybridization;
+      for (int i = 0; i < domain_integs.Size(); i++) { delete domain_integs[i]; }
+      for (int i = 0; i < boundary_integs.Size(); i++) { delete boundary_integs[i]; }
+      for (int i = 0; i < interior_face_integs.Size(); i++) { delete interior_face_integs[i]; }
+      for (int i = 0; i < boundary_face_integs.Size(); i++) { delete boundary_face_integs[i]; }
+      delete ext;
+
+      /// Null out all our integs and set size of their arrays to zero
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         domain_integs[k] = nullptr;
+      }
+      domain_integs.SetSize(0);
+      for (int k = 0; k < boundary_integs.Size(); k++)
+      {
+         boundary_integs[k] = nullptr;
+      }
+      boundary_integs.SetSize(0);
+      for (int k = 0; k < interior_face_integs.Size(); k++)
+      {
+         interior_face_integs[k] = nullptr;
+      }
+      interior_face_integs.SetSize(0);
+      for (int k = 0; k < boundary_face_integs.Size(); ++k)
+      {
+         boundary_face_integs[k] = nullptr;
+      }
+      boundary_face_integs.SetSize(0);
+
+      /// Null out all our markers and set size of their arrays to zero
+      for (int k = 0; k < domain_integs_marker.Size(); ++k)
+      {
+         domain_integs_marker[k] = nullptr;
+      }
+      domain_integs_marker.SetSize(0);
+      for (int k = 0; k < boundary_integs_marker.Size(); ++k)
+      {
+         boundary_integs_marker[k] = nullptr;
+      }
+      boundary_integs_marker.SetSize(0);
+      for (int k = 0; k < boundary_face_integs_marker.Size(); ++k)
+      {
+         boundary_face_integs_marker[k] = nullptr;
+      }
+      boundary_face_integs_marker.SetSize(0);
+
+      /// Now steal data from other bilinear form leaving it in a state as if
+      /// it was just constructed with fes
+      Matrix::operator=(std::move(other));
+
+      mat = other.mat;
+      other.mat = nullptr;
+      mat_e = other.mat_e;
+      other.mat_e = nullptr;
+      fes = other.fes;
+      assembly = other.assembly;
+      other.assembly = AssemblyLevel::LEGACY;
+      batch = other.batch;
+      other.batch = 1;
+      ext = other.ext;
+      other.ext = nullptr;
+      sequence = other.sequence;
+      other.sequence = fes->GetSequence();
+      extern_bfs = other.extern_bfs;
+      other.extern_bfs = 0;
+
+      // Swap our empty integ and marker arrays with the moved bilinear form
+      mfem::Swap(domain_integs, other.domain_integs);
+      mfem::Swap(domain_integs_marker, other.domain_integs_marker);
+      mfem::Swap(boundary_integs, other.boundary_integs);
+      mfem::Swap(boundary_integs_marker, other.boundary_integs_marker);
+      mfem::Swap(interior_face_integs, other.interior_face_integs);
+      mfem::Swap(boundary_face_integs, other.boundary_face_integs);
+      mfem::Swap(boundary_face_integs_marker, other.boundary_face_integs_marker);
+
+      element_matrices = other.element_matrices;
+      other.element_matrices = nullptr;
+      static_cond = other.static_cond;
+      other.static_cond = nullptr;
+      hybridization = other.hybridization;
+      other.hybridization = nullptr;
+      diag_policy = other.diag_policy;
+      other.diag_policy = DIAG_KEEP;
+      precompute_sparsity = other.precompute_sparsity;
+      other.precompute_sparsity = 0;
+   }
+   return *this;
+}
+
 void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
 {
   if (ext)
@@ -1763,12 +1896,24 @@ void MixedBilinearForm::FormRectangularSystemMatrix(

   mat->Finalize();

-   if (test_P) // TODO: Must actually check for trial_P too
+   if (test_P && trial_P)
   {
      SparseMatrix *m = RAP(*test_P, *mat, *trial_P);
      delete mat;
      mat = m;
   }
+   else if (test_P)
+   {
+      SparseMatrix *m = TransposeMult(*test_P, *mat);
+      delete mat;
+      mat = m;
+   }
+   else if (trial_P)
+   {
+      SparseMatrix *m = mfem::Mult(*mat, *trial_P);
+      delete mat;
+      mat = m;
+   }

   Array<int> ess_trial_tdof_marker, ess_test_tdof_marker;
   FiniteElementSpace::ListToMarker(trial_tdof_list, trial_fes->GetTrueVSize(),
@@ -90,23 +90,23 @@ protected:
   int extern_bfs;

   /// Set of Domain Integrators to be applied.
-   Array<BilinearFormIntegrator*> domain_integs;
+   Array<BilinearFormIntegrator*> domain_integs; // owned
   /// Element attribute marker (should be of length mesh->attributes.Max() or
   /// 0 if mesh->attributes is empty)
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>             domain_integs_marker;
+   Array<Array<int>*>             domain_integs_marker; // not owned

   /// Set of Boundary Integrators to be applied.
-   Array<BilinearFormIntegrator*> boundary_integs;
+   Array<BilinearFormIntegrator*> boundary_integs; // owned
   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.

   /// Set of interior face Integrators to be applied.
-   Array<BilinearFormIntegrator*> interior_face_integs;
+   Array<BilinearFormIntegrator*> interior_face_integs; // owned

   /// Set of boundary face Integrators to be applied.
-   Array<BilinearFormIntegrator*> boundary_face_integs;
+   Array<BilinearFormIntegrator*> boundary_face_integs; // owned
   Array<Array<int>*> boundary_face_integs_marker; ///< Entries are not owned.

   DenseMatrix elemmat;
@@ -141,13 +141,6 @@ protected:
      ext = NULL;
   }

-private:
-   /// Copy construction is not supported; body is undefined.
-   BilinearForm(const BilinearForm &);
-
-   /// Copy assignment is not supported; body is undefined.
-   BilinearForm &operator=(const BilinearForm &);
-
 public:
   /// Creates bilinear form associated with FE space @a *f.
   /** The pointer @a f is not owned by the newly constructed object. */
@@ -165,6 +158,20 @@ public:
       #precompute_sparsity, see UsePrecomputedSparsity() for details. */
   BilinearForm(FiniteElementSpace *f, BilinearForm *bf, int ps = 0);

+   /// Explicitly prohibit copy construction/assignment of BilinearForms
+   BilinearForm(const BilinearForm &) = delete;
+   BilinearForm &operator=(const BilinearForm &) = delete;
+
+   /// Move constructor for BilinearForm.
+   /** This constructor "steals" the owned data members from the @a other
+       BilinearForm. */
+   BilinearForm(BilinearForm &&other);
+
+   /// Move assignment operator for BilinearForm
+   /** This assignment first frees all owned data, then "steals" the owned data
+       members from the @a other BilinearForm. */
+   BilinearForm& operator=(BilinearForm &&other);
+
   /// Get the size of the BilinearForm as a square matrix.
   int Size() const { return height; }

@@ -305,38 +312,66 @@ public:
   /// Finalizes the matrix initialization.
   virtual void Finalize(int skip_zeros = 1);

-   /// Returns a const reference to the sparse matrix.
+   /** @brief Returns a const reference to the sparse matrix:  \f$ M \f$
+
+       This will fail if HasSpMat() is false. */
   const SparseMatrix &SpMat() const
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }

-   /// Returns a reference to the sparse matrix:  \f$ M \f$
+   /** @brief Returns a reference to the sparse matrix:  \f$ M \f$
+
+       This will fail if HasSpMat() is false. */
   SparseMatrix &SpMat()
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }

+   /** @brief Returns true if the sparse matrix is not null, false otherwise.
+
+       @sa SpMat(). */
+   bool HasSpMat()
+   {
+      return mat != nullptr;
+   }
+
+
   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

-   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
+       \f$ M_e \f$
+
+       This will fail if HasSpMatElim() is false. */
   const SparseMatrix &SpMatElim() const
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
      return *mat_e;
   }

-   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   /** @brief Returns a reference to the sparse matrix of eliminated b.c.:
+       \f$ M_e \f$
+
+       This will fail if HasSpMatElim() is false. */
   SparseMatrix &SpMatElim()
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
      return *mat_e;
   }

+   /**  @brief Returns true if the sparse matrix of eliminated b.c.s is not null,
+        false otherwise.
+
+        @sa SpMatElim(). */
+   bool HasSpMatElim()
+   {
+      return mat_e != nullptr;
+   }
+
   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);
   /// Adds new Domain Integrator restricted to certain elements specified by
@@ -16,7 +16,7 @@
 #include "bilinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pgridfunc.hpp"
-#include "ceed/util.hpp"
+#include "ceed/interface/util.hpp"

 namespace mfem
 {
@@ -903,7 +903,8 @@ void FABilinearFormExtension::Assemble()
   }
   else // We create, compute the sparsity, and fill the sparse matrix
   {
-      mat = new SparseMatrix(height, width, 0);
+      mat = new SparseMatrix;
+      mat->OverrideSize(height, width);
      if (fes.IsDGSpace())
      {
         const L2ElementRestriction *restE =
@@ -144,14 +144,6 @@ void BilinearFormIntegrator::AssembleFaceMatrix (
               "   is not implemented for this class.");
 }

-void BilinearFormIntegrator::AssembleTraceFaceMatrix(int elem,
-                                                     const FiniteElement &trial_face_fe,  const FiniteElement &test_fe,
-                                                     FaceElementTransformations &Trans, DenseMatrix &elmat)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleTraceFaceMatrix(...)\n"
-               "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AssembleFaceMatrix(
   const FiniteElement &trial_face_fe, const FiniteElement &test_fe1,
   const FiniteElement &test_fe2, FaceElementTransformations &Trans,
@@ -1044,7 +1036,8 @@ void DiffusionIntegrator::AssembleElementVector(

 void DiffusionIntegrator::ComputeElementFlux
 ( const FiniteElement &el, ElementTransformation &Trans,
-  Vector &u, const FiniteElement &fluxelem, Vector &flux, bool with_coef )
+  Vector &u, const FiniteElement &fluxelem, Vector &flux, bool with_coef,
+  const IntegrationRule *ir)
 {
   int nd, spaceDim, fnd;

@@ -1065,8 +1058,6 @@ void DiffusionIntegrator::ComputeElementFlux
                  "Unexpected height for MatrixCoefficient");
   }

-   MFEM_VERIFY(!SMQ, "SymmetricMatrixCoefficient not supported here");
-
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix dshape(nd,dim), invdfdx(dim, spaceDim);
   DenseMatrix M(MQ ? spaceDim : 0);
@@ -1081,13 +1072,16 @@ void DiffusionIntegrator::ComputeElementFlux
   vecdxt.SetSize(spaceDim);
   pointflux.SetSize(MQ || VQ ? spaceDim : 0);

-   const IntegrationRule &ir = fluxelem.GetNodes();
-   fnd = ir.GetNPoints();
+   if (!ir)
+   {
+      ir = &fluxelem.GetNodes();
+   }
+   fnd = ir->GetNPoints();
   flux.SetSize( fnd * spaceDim );

   for (int i = 0; i < fnd; i++)
   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
+      const IntegrationPoint &ip = ir->IntPoint(i);
      el.CalcDShape(ip, dshape);
      dshape.MultTranspose(u, vec);

@@ -1154,8 +1148,6 @@ double DiffusionIntegrator::ComputeFluxEnergy
   D.SetSize(VQ ? VQ->GetVDim() : 0);
 #endif

-   MFEM_VERIFY(!SMQ, "SymmetricMatrixCoefficient not supported here");
-
   shape.SetSize(nd);
   pointflux.SetSize(spaceDim);
   if (d_energy) { vec.SetSize(spaceDim); }
@@ -1680,7 +1672,8 @@ void VectorFEDivergenceIntegrator::AssembleElementMatrix2(
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
      trial_fe.CalcDivShape(ip, divshape);
-      test_fe.CalcShape(ip, shape);
+      Trans.SetIntPoint(&ip);
+      test_fe.CalcPhysShape(Trans, shape);
      double w = ip.weight;
      if (Q)
      {
@@ -2013,12 +2006,14 @@ void CurlCurlIntegrator::AssembleElementMatrix
 void CurlCurlIntegrator
 ::ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans,
                     Vector &u, const FiniteElement &fluxelem, Vector &flux,
-                     bool with_coef)
+                     bool with_coef, const IntegrationRule *ir)
 {
 #ifdef MFEM_THREAD_SAFE
   DenseMatrix projcurl;
 #endif

+   MFEM_VERIFY(ir == NULL, "Integration rule (ir) must be NULL")
+
   fluxelem.ProjectCurl(el, Trans, projcurl);

   flux.SetSize(projcurl.Height());
@@ -2591,55 +2586,6 @@ void DivDivIntegrator::AssembleElementMatrix(
   }
 }

-void DivDivIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe,
-   const FiniteElement &test_fe,
-   ElementTransformation &Trans,
-   DenseMatrix &elmat)
-{
-   int tr_nd = trial_fe.GetDof();
-   int te_nd = test_fe.GetDof();
-   double c;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector divshape(tr_nd);
-   Vector te_divshape(te_nd);
-#else
-   divshape.SetSize(tr_nd);
-   te_divshape.SetSize(te_nd);
-#endif
-   elmat.SetSize(te_nd,tr_nd);
-
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order = 2 * max(test_fe.GetOrder(),
-                          trial_fe.GetOrder()) - 2; // <--- OK for RTk
-      ir = &IntRules.Get(test_fe.GetGeomType(), order);
-   }
-
-   elmat = 0.0;
-
-   for (int i = 0; i < ir -> GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      trial_fe.CalcDivShape(ip,divshape);
-      test_fe.CalcDivShape(ip,te_divshape);
-
-      Trans.SetIntPoint (&ip);
-      c = ip.weight / Trans.Weight();
-
-      if (Q)
-      {
-         c *= Q -> Eval (Trans, ip);
-      }
-
-      te_divshape *= c;
-      AddMultVWt(te_divshape, divshape, elmat);
-   }
-}

 void VectorDiffusionIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -2899,7 +2845,7 @@ void ElasticityIntegrator::AssembleElementMatrix(
 void ElasticityIntegrator::ComputeElementFlux(
   const mfem::FiniteElement &el, ElementTransformation &Trans,
   Vector &u, const mfem::FiniteElement &fluxelem, Vector &flux,
-   bool with_coef)
+   bool with_coef, const IntegrationRule *ir)
 {
   const int dof = el.GetDof();
   const int dim = el.GetDim();
@@ -2922,14 +2868,17 @@ void ElasticityIntegrator::ComputeElementFlux(
   DenseMatrix gh(gh_data, dim, dim);
   DenseMatrix grad(grad_data, dim, dim);

-   const IntegrationRule &ir = fluxelem.GetNodes();
-   const int fnd = ir.GetNPoints();
+   if (!ir)
+   {
+      ir = &fluxelem.GetNodes();
+   }
+   const int fnd = ir->GetNPoints();
   flux.SetSize(fnd * tdim);

   DenseMatrix loc_data_mat(u.GetData(), dof, dim);
   for (int i = 0; i < fnd; i++)
   {
-      const IntegrationPoint &ip = ir.IntPoint(i);
+      const IntegrationPoint &ip = ir->IntPoint(i);
      el.CalcDShape(ip, dshape);
      MultAtB(loc_data_mat, dshape, gh);

@@ -3831,7 +3780,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
      for (i = 0; i < ndof1; i++)
         for (j = 0; j < face_ndof; j++)
         {
-            elmat(i, j) += shape1_n(i) * face_shape(j);
+            elmat(i, j) -= shape1_n(i) * face_shape(j);
         }
      if (ndof2)
      {
@@ -3839,135 +3788,12 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
         for (i = 0; i < ndof2; i++)
            for (j = 0; j < face_ndof; j++)
            {
-               elmat(ndof1+i, j) -= shape2_n(i) * face_shape(j);
+               elmat(ndof1+i, j) += shape2_n(i) * face_shape(j);
            }
      }
   }
 }

-void TraceIntegrator::AssembleTraceFaceMatrix(int elem,
-                                              const FiniteElement &trial_face_fe,
-                                              const FiniteElement &test_fe,
-                                              FaceElementTransformations & Trans,
-                                              DenseMatrix &elmat)
-{
-   int i, j, face_ndof, ndof;
-   int order;
-
-   face_ndof = trial_face_fe.GetDof();
-   ndof = test_fe.GetDof();
-
-   face_shape.SetSize(face_ndof);
-   shape.SetSize(ndof);
-
-   elmat.SetSize(ndof, face_ndof);
-   elmat = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      order = test_fe.GetOrder();
-      order += trial_face_fe.GetOrder();
-      if (trial_face_fe.GetMapType() == FiniteElement::VALUE)
-      {
-         order += Trans.OrderW();
-      }
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
-   }
-
-   int iel = Trans.Elem1->ElementNo;
-   if (iel != elem)
-   {
-      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
-   }
-
-   double scale = 1.0;
-   if (iel != elem) { scale = -1.; }
-   for (int p = 0; p < ir->GetNPoints(); p++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(p);
-
-      // Set the integration point in the face and the neighboring elements
-      Trans.SetAllIntPoints(&ip);
-      // Trace finite element shape function
-      trial_face_fe.CalcPhysShape(Trans,face_shape);
-
-      // Finite element shape function
-      ElementTransformation * eltrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
-      test_fe.CalcPhysShape(*eltrans, shape);
-
-      face_shape *= Trans.Weight()*ip.weight;
-      for (i = 0; i < ndof; i++)
-      {
-         for (j = 0; j < face_ndof; j++)
-         {
-            elmat(i, j) += scale * shape(i) * face_shape(j);
-         }
-      }
-   }
-}
-
-void NormalTraceIntegrator::AssembleTraceFaceMatrix(int elem,
-                                                    const FiniteElement &trial_face_fe,
-                                                    const FiniteElement &test_fe,
-                                                    FaceElementTransformations &Trans,
-                                                    DenseMatrix &elmat)
-{
-   int i, j, face_ndof, ndof, dim;
-   int order;
-
-   MFEM_VERIFY(trial_face_fe.GetMapType() == FiniteElement::VALUE, "");
-
-   face_ndof = trial_face_fe.GetDof();
-   ndof = test_fe.GetDof();
-   dim = test_fe.GetDim();
-
-   face_shape.SetSize(face_ndof);
-   normal.SetSize(dim);
-   shape.SetSize(ndof,dim);
-   shape_n.SetSize(ndof);
-
-   elmat.SetSize(ndof, face_ndof);
-   elmat = 0.0;
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      order = test_fe.GetOrder();
-      order += trial_face_fe.GetOrder();
-      ir = &IntRules.Get(Trans.GetGeometryType(), order);
-   }
-
-   int iel = Trans.Elem1->ElementNo;
-   if (iel != elem)
-   {
-      MFEM_VERIFY(elem == Trans.Elem2->ElementNo, "Elem != Trans.Elem2->ElementNo");
-   }
-
-   double scale = 1.0;
-   if (iel != elem) { scale = -1.; }
-
-   for (int p = 0; p < ir->GetNPoints(); p++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(p);
-      Trans.SetAllIntPoints(&ip);
-      trial_face_fe.CalcPhysShape(Trans, face_shape);
-      CalcOrtho(Trans.Jacobian(),normal);
-      ElementTransformation * etrans = (iel == elem) ? Trans.Elem1 : Trans.Elem2;
-      test_fe.CalcVShape(*etrans, shape);
-      shape.Mult(normal, shape_n);
-      face_shape *= ip.weight;
-
-      for (i = 0; i < ndof; i++)
-      {
-         for (j = 0; j < face_ndof; j++)
-         {
-            elmat(i, j) += scale * shape_n(i) * face_shape(j);
-         }
-      }
-   }
-}
-

 void NormalInterpolator::AssembleElementMatrix2(
   const FiniteElement &dom_fe, const FiniteElement &ran_fe,
@@ -15,6 +15,7 @@
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
 #include "fespace.hpp"
+#include "ceed/interface/util.hpp"

 namespace mfem
 {
@@ -150,12 +151,6 @@ public:
                                   FaceElementTransformations &Trans,
                                   DenseMatrix &elmat);

-   virtual void AssembleTraceFaceMatrix(int elem,
-                                        const FiniteElement &trial_face_fe,
-                                        const FiniteElement &test_fe,
-                                        FaceElementTransformations &Trans,
-                                        DenseMatrix &elmat);
-
   /** Abstract method used for assembling TraceFaceIntegrators in a
       MixedBilinearForm. */
   virtual void AssembleFaceMatrix(const FiniteElement &trial_face_fe,
@@ -219,12 +214,18 @@ public:
                             of the method may choose not to scale the "flux"
                             function by any coefficients describing the
                             integrator.
+       @param[in] ir  If passed (the default value is NULL), the implementation
+                        of the method will ignore the integration rule provided
+                        by the @a fluxelem parameter and, instead, compute the
+                        discrete flux at the points specified by the integration
+                        rule @a ir.
    */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u,
                                   const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true) { }
+                                   Vector &flux, bool with_coef = true,
+                                   const IntegrationRule *ir = NULL) { }

   /** @brief Virtual method required for Zienkiewicz-Zhu type error estimators.

@@ -1905,6 +1906,7 @@ protected:
                           const FiniteElementSpace &test_fes);

   virtual void AddMultPA(const Vector&, Vector&) const;
+   virtual void AddMultTransposePA(const Vector&, Vector&) const;

 private:
   // PA extension
@@ -1963,6 +1965,7 @@ protected:
                           const FiniteElementSpace &test_fes);

   virtual void AddMultPA(const Vector&, Vector&) const;
+   virtual void AddMultTransposePA(const Vector&, Vector&) const;

 private:
   // PA extension
@@ -2074,7 +2077,6 @@ protected:
   Coefficient *Q;
   VectorCoefficient *VQ;
   MatrixCoefficient *MQ;
-   SymmetricMatrixCoefficient *SMQ;

 private:
   Vector vec, vecdxt, pointflux, shape;
@@ -2096,30 +2098,24 @@ public:
   /// Construct a diffusion integrator with coefficient Q = 1
   DiffusionIntegrator(const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+        Q(NULL), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a scalar coefficient q
   DiffusionIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(&q), VQ(NULL), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+        Q(&q), VQ(NULL), MQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a vector coefficient q
   DiffusionIntegrator(VectorCoefficient &q,
                       const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(&q), MQ(NULL), SMQ(NULL), maps(NULL), geom(NULL) { }
+        Q(NULL), VQ(&q), MQ(NULL), maps(NULL), geom(NULL) { }

   /// Construct a diffusion integrator with a matrix coefficient q
   DiffusionIntegrator(MatrixCoefficient &q,
                       const IntegrationRule *ir = nullptr)
      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(&q), SMQ(NULL), maps(NULL), geom(NULL) { }
-
-   /// Construct a diffusion integrator with a symmetric matrix coefficient q
-   DiffusionIntegrator(SymmetricMatrixCoefficient &q,
-                       const IntegrationRule *ir = nullptr)
-      : BilinearFormIntegrator(ir),
-        Q(NULL), VQ(NULL), MQ(NULL), SMQ(&q), maps(NULL), geom(NULL) { }
+        Q(NULL), VQ(NULL), MQ(&q), maps(NULL), geom(NULL) { }

   /** Given a particular Finite Element computes the element stiffness matrix
       elmat. */
@@ -2141,7 +2137,8 @@ public:
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true);
+                                   Vector &flux, bool with_coef = true,
+                                   const IntegrationRule *ir = NULL);

   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
@@ -2170,6 +2167,8 @@ public:
                                         const FiniteElement &test_fe);

   bool SupportsCeed() const { return DeviceCanUseCeed(); }
+
+   Coefficient *GetCoefficient() const { return Q; }
 };

 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
@@ -2229,6 +2228,8 @@ public:
                                         ElementTransformation &Trans);

   bool SupportsCeed() const { return DeviceCanUseCeed(); }
+
+   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Mass integrator (u, v) restricted to the boundary of a domain */
@@ -2397,8 +2398,9 @@ public:
    scalar function given by FiniteElement through standard transformation.
    Here, u is the trial function and p is the test function.

-    Note: the element matrix returned by AssembleElementMatrix2 does NOT depend
-    on the ElementTransformation Trans. */
+    Note: if the test space does not have map type INTEGRAL, then the element
+    matrix returned by AssembleElementMatrix2 will not depend on the
+    ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2529,7 +2531,6 @@ protected:
   Coefficient *Q;
   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;
-   SymmetricMatrixCoefficient *SMQ;

   // PA extension
   Vector pa_data;
@@ -2540,18 +2541,15 @@ protected:
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; SMQ = NULL; }
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
   /// Construct a bilinear form integrator for Nedelec elements
   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL), SMQ(NULL) { }
+      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL) { }
   CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
                      const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL), SMQ(NULL) { }
+      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL) { }
   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq), SMQ(NULL) { }
-   CurlCurlIntegrator(SymmetricMatrixCoefficient &smq,
-                      const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(NULL), SMQ(&smq) { }
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq) { }

   /* Given a particular Finite Element, compute the
      element curl-curl matrix elmat */
@@ -2562,7 +2560,8 @@ public:
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef);
+                                   Vector &flux, bool with_coef,
+                                   const IntegrationRule *ir = NULL);

   virtual double ComputeFluxEnergy(const FiniteElement &fluxelem,
                                    ElementTransformation &Trans,
@@ -2572,6 +2571,8 @@ public:
   virtual void AssemblePA(const FiniteElementSpace &fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
+
+   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Integrator for (curl u, curl v) for FE spaces defined by 'dim' copies of a
@@ -2608,9 +2609,8 @@ public:
 class VectorFEMassIntegrator: public BilinearFormIntegrator
 {
 private:
-   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq,
-             SymmetricMatrixCoefficient *smq)
-   { Q = q; DQ = dq; MQ = mq; SMQ = smq; }
+   void Init(Coefficient *q, DiagonalMatrixCoefficient *dq, MatrixCoefficient *mq)
+   { Q = q; DQ = dq; MQ = mq; }

 #ifndef MFEM_THREAD_SAFE
   Vector shape;
@@ -2625,7 +2625,6 @@ protected:
   Coefficient *Q;
   DiagonalMatrixCoefficient *DQ;
   MatrixCoefficient *MQ;
-   SymmetricMatrixCoefficient *SMQ;

   // PA extension
   Vector pa_data;
@@ -2638,15 +2637,13 @@ protected:
   bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient

 public:
-   VectorFEMassIntegrator() { Init(NULL, NULL, NULL, NULL); }
-   VectorFEMassIntegrator(Coefficient *q_) { Init(q_, NULL, NULL, NULL); }
-   VectorFEMassIntegrator(Coefficient &q) { Init(&q, NULL, NULL, NULL); }
-   VectorFEMassIntegrator(DiagonalMatrixCoefficient *dq_) { Init(NULL, dq_, NULL, NULL); }
-   VectorFEMassIntegrator(DiagonalMatrixCoefficient &dq) { Init(NULL, &dq, NULL, NULL); }
-   VectorFEMassIntegrator(MatrixCoefficient *mq_) { Init(NULL, NULL, mq_, NULL); }
-   VectorFEMassIntegrator(MatrixCoefficient &mq) { Init(NULL, NULL, &mq, NULL); }
-   VectorFEMassIntegrator(SymmetricMatrixCoefficient &smq) { Init(NULL, NULL, NULL, &smq); }
-   VectorFEMassIntegrator(SymmetricMatrixCoefficient *smq) { Init(NULL, NULL, NULL, smq); }
+   VectorFEMassIntegrator() { Init(NULL, NULL, NULL); }
+   VectorFEMassIntegrator(Coefficient *q_) { Init(q_, NULL, NULL); }
+   VectorFEMassIntegrator(Coefficient &q) { Init(&q, NULL, NULL); }
+   VectorFEMassIntegrator(DiagonalMatrixCoefficient *dq_) { Init(NULL, dq_, NULL); }
+   VectorFEMassIntegrator(DiagonalMatrixCoefficient &dq) { Init(NULL, &dq, NULL); }
+   VectorFEMassIntegrator(MatrixCoefficient *mq_) { Init(NULL, NULL, mq_); }
+   VectorFEMassIntegrator(MatrixCoefficient &mq) { Init(NULL, NULL, &mq); }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -2661,7 +2658,10 @@ public:
   virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                           const FiniteElementSpace &test_fes);
   virtual void AddMultPA(const Vector &x, Vector &y) const;
+   virtual void AddMultTransposePA(const Vector &x, Vector &y) const;
   virtual void AssembleDiagonalPA(Vector& diag);
+
+   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Integrator for (Q div u, p) where u=(v1,...,vn) and all vi are in the same
@@ -2725,7 +2725,7 @@ protected:

 private:
 #ifndef MFEM_THREAD_SAFE
-   Vector divshape, te_divshape;
+   Vector divshape;
 #endif

   // PA extension
@@ -2742,10 +2742,7 @@ public:
   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
+   const Coefficient *GetCoefficient() const { return Q; }
 };

 /** Integrator for
@@ -2885,12 +2882,14 @@ public:
       of the stress components is: s_xx, s_yy, s_xy. In 3D, it is: s_xx, s_yy,
       s_zz, s_xy, s_xz, s_yz. In other words, @a flux is the local vector for
       a FE space with dim*(dim+1)/2 vector components, based on the finite
-       element @a fluxelem. */
+       element @a fluxelem. The integration rule is taken from @a fluxelem.
+       @a ir exists to specific an alternative integration rule. */
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u,
                                   const FiniteElement &fluxelem,
-                                   Vector &flux, bool with_coef = true);
+                                   Vector &flux, bool with_coef = true,
+                                   const IntegrationRule *ir = NULL);

   /** Compute the element energy (integral of the strain energy density)
       corresponding to the stress represented by @a flux which is a vector of
@@ -3240,19 +3239,6 @@ public:
                                   DenseMatrix &elmat);
 };

-class TraceIntegrator : public BilinearFormIntegrator
-{
-private:
-   Vector face_shape, shape;
-public:
-   TraceIntegrator() { }
-   void AssembleTraceFaceMatrix(int elem,
-                                const FiniteElement &trial_face_fe,
-                                const FiniteElement &test_fe,
-                                FaceElementTransformations &Trans,
-                                DenseMatrix &elmat);
-};
-
 /** Integrator for the form: < v, [w.n] > over all faces (the interface) where
    the trial variable v is defined on the interface and the test variable w is
    in an H(div)-conforming space. */
@@ -3272,23 +3258,6 @@ public:
                                   DenseMatrix &elmat);
 };

-class NormalTraceIntegrator : public BilinearFormIntegrator
-{
-private:
-   Vector face_shape, normal, shape_n;
-   DenseMatrix shape;
-
-public:
-   NormalTraceIntegrator() { }
-   virtual void AssembleTraceFaceMatrix(int ielem,
-                                        const FiniteElement &trial_face_fe,
-                                        const FiniteElement &test_fe,
-                                        FaceElementTransformations &Trans,
-                                        DenseMatrix &elmat);
-};
-
-
-
 /** Abstract class to serve as a base for local interpolators to be used in the
    DiscreteLinearOperator class. */
 class DiscreteInterpolator : public BilinearFormIntegrator { };
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/convection.hpp"
+#include "ceed/integrators/convection/convection.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/convection.hpp"
+#include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"

 namespace mfem
@@ -1497,6 +1497,7 @@ static void PAConvectionApply(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPAConvectionApply3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
         case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
         case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
         case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -1548,6 +1549,7 @@ static void PAConvectionApplyT(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPAConvectionApplyT3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
         case 0x23: return SmemPAConvectionApplyT3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
         case 0x24: return SmemPAConvectionApplyT3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
         case 0x26: return SmemPAConvectionApplyT3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -43,9 +43,10 @@ static void PADGTraceSetup2D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, 2, 2, NF);

-   MFEM_FORALL(f, NF, // can be optimized with Q1D thread for NF blocks
+   MFEM_FORALL(tid, Q1D*NF,
   {
-      for (int q = 0; q < Q1D; ++q)
+      const int f = tid / Q1D;
+      const int q = tid % Q1D;
      {
         const double r = const_r ? R(0,0) : R(q,f);
         const double v0 = const_v ? V(0,0,0) : V(0,q,f);
@@ -85,11 +86,12 @@ static void PADGTraceSetup3D(const int Q1D,
   auto W = w.Read();
   auto qd = Reshape(op.Write(), Q1D, Q1D, 2, 2, NF);

-   MFEM_FORALL(f, NF, // can be optimized with Q1D*Q1D threads for NF blocks
+   MFEM_FORALL(tid, Q1D*Q1D*NF,
   {
-      for (int q1 = 0; q1 < Q1D; ++q1)
+      int f = tid / (Q1D * Q1D);
+      int q2 = (tid / Q1D) % Q1D;
+      int q1 = tid % Q1D;
      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
         {
            const double r = const_r ? R(0,0,0) : R(q1,q2,f);
            const double v0 = const_v ? V(0,0,0,0) : V(0,q1,q2,f);
@@ -134,6 +136,9 @@ static void PADGTraceSetup(const int dim,

 void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
 {
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+
   nf = fes.GetNFbyType(type);
   if (nf==0) { return; }
   // Assumes tensor-product elements
@@ -151,7 +156,7 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   geom = mesh->GetFaceGeometricFactors(
             *ir,
             FaceGeometricFactors::DETERMINANTS |
-             FaceGeometricFactors::NORMALS, type);
+             FaceGeometricFactors::NORMALS, type, mt);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
@@ -693,6 +698,7 @@ static void PADGTraceApply(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPADGTraceApply3D<2,2,1>(NF,B,Bt,op,x,y);
         case 0x23: return SmemPADGTraceApply3D<2,3,1>(NF,B,Bt,op,x,y);
         case 0x34: return SmemPADGTraceApply3D<3,4,2>(NF,B,Bt,op,x,y);
         case 0x45: return SmemPADGTraceApply3D<4,5,2>(NF,B,Bt,op,x,y);
@@ -1122,6 +1128,7 @@ static void PADGTraceApplyTranspose(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPADGTraceApplyTranspose3D<2,2>(NF,B,Bt,op,x,y);
         case 0x23: return SmemPADGTraceApplyTranspose3D<2,3>(NF,B,Bt,op,x,y);
         case 0x34: return SmemPADGTraceApplyTranspose3D<3,4>(NF,B,Bt,op,x,y);
         case 0x45: return SmemPADGTraceApplyTranspose3D<4,5>(NF,B,Bt,op,x,y);
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -362,7 +362,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      MFEM_VERIFY(!VQ && !MQ && !SMQ,
+      MFEM_VERIFY(!VQ && !MQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
@@ -381,41 +381,14 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   int coeffDim = 1;
   Vector coeff;
   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (MQ)
-   {
-      symmetric = false;
-      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-
-      coeffDim = MQfullDim;
-
-      coeff.SetSize(MQfullDim * nq * ne);
-
-      DenseMatrix GM;
-      GM.SetSize(dim);
-
-      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            MQ->Eval(GM, *tr, ir->IntPoint(p));
-            for (int i=0; i<dim; ++i)
-               for (int j=0; j<dim; ++j)
-               {
-                  C(j+(i*dim), p, e) = GM(i,j);
-               }
-         }
-      }
-   }
-   else if (SMQ)
+   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
   {
      MFEM_VERIFY(SMQ->GetSize() == dim, "");
      coeffDim = symmDims;
      coeff.SetSize(symmDims * nq * ne);

-      DenseSymmetricMatrix SM;
-      SM.SetSize(dim);
+      DenseSymmetricMatrix sym_mat;
+      sym_mat.SetSize(dim);

      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);

@@ -424,12 +397,39 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            SMQ->Eval(SM, *tr, ir->IntPoint(p));
+            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
            int cnt = 0;
            for (int i=0; i<dim; ++i)
               for (int j=i; j<dim; ++j, ++cnt)
               {
-                  C(cnt, p, e) = SM(i,j);
+                  C(cnt, p, e) = sym_mat(i,j);
+               }
+         }
+      }
+   }
+   else if (MQ)
+   {
+      symmetric = false;
+      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+
+      coeffDim = MQfullDim;
+
+      coeff.SetSize(MQfullDim * nq * ne);
+
+      DenseMatrix mat;
+      mat.SetSize(dim);
+
+      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            MQ->Eval(mat, *tr, ir->IntPoint(p));
+            for (int i=0; i<dim; ++i)
+               for (int j=0; j<dim; ++j)
+               {
+                  C(j+(i*dim), p, e) = mat(i,j);
               }
         }
      }
@@ -978,12 +978,14 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

+   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+
   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = MQ ? MQfullDim : MQsymmDim;
-   const int coeffDim = (MQ || SMQ) ? MQdim : (DQ ? DQ->GetVDim() : 1);
+   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);

-   symmetric = (MQ == NULL);
+   symmetric = (SMQ || MQ == NULL);

   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
@@ -992,7 +994,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   Vector coeff(coeffDim * ne * nq);
   coeff = 1.0;
   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ || SMQ)
+   if (Q || DQ || MQ)
   {
      Vector DM(DQ ? coeffDim : 0);
      DenseMatrix GM;
@@ -1002,35 +1004,24 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
      {
         MFEM_VERIFY(coeffDim == dimc, "");
      }
-      if (MQ)
-      {
-         GM.SetSize(dimc);
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
-      }
      if (SMQ)
      {
         SM.SetSize(dimc);
         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
      }
+      else if (MQ)
+      {
+         GM.SetSize(dimc);
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
+      }

      for (int e=0; e<ne; ++e)
      {
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            if (MQ)
-            {
-               MQ->Eval(GM, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dimc; ++i)
-                  for (int j=0; j<dimc; ++j)
-                  {
-                     coeffh(j+(i*dimc), p, e) = GM(i,j);
-                  }
-
-            }
-            else if (SMQ)
+            if (SMQ)
            {
               SMQ->Eval(SM, *tr, ir->IntPoint(p));

@@ -1041,6 +1032,17 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
                     coeffh(cnt, p, e) = SM(i,j);
                  }

+            }
+            else if (MQ)
+            {
+               MQ->Eval(GM, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dimc; ++i)
+                  for (int j=0; j<dimc; ++j)
+                  {
+                     coeffh(j+(i*dimc), p, e) = GM(i,j);
+                  }
+
            }
            else if (DQ)
            {
@@ -3633,8 +3635,8 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      }
      else
      {
-         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(), geom->J,
-                            coeff, pa_data);
+         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(),
+                            geom->J, coeff, pa_data);
      }
   }
   else if (testType == mfem::FiniteElement::DIV &&
@@ -4679,6 +4681,362 @@ static void PAHcurlHdivApply3D(const int D1D,
   }); // end of element loop
 }

+// Apply to x corresponding to DOF's in H(div) (test), integrated against the
+// curl of H(curl) trial functions corresponding to y.
+template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
+static void PAHcurlHdivApply3DTranspose(const int D1D,
+                                        const int D1Dtest,
+                                        const int Q1D,
+                                        const int NE,
+                                        const Array<double> &bo,
+                                        const Array<double> &bc,
+                                        const Array<double> &bot,
+                                        const Array<double> &bct,
+                                        const Array<double> &gct,
+                                        const Vector &pa_data,
+                                        const Vector &x,
+                                        Vector &y)
+{
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
+   // Using Piola transformations (\nabla\times u) F = 1/det(dF) dF \hat{\nabla}\times\hat{u}
+   // for u in H(curl) and w = (1 / det (dF)) dF \hat{w} for w in H(div), we get
+   // (\nabla\times u) \cdot w = 1/det(dF)^2 \hat{\nabla}\times\hat{u}^T dF^T dF \hat{w}
+   // If c = 0, \hat{\nabla}\times\hat{u} reduces to [0, (u_0)_{x_2}, -(u_0)_{x_1}]
+   // If c = 1, \hat{\nabla}\times\hat{u} reduces to [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
+   // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
+
+   constexpr static int VDIM = 3;
+
+   auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
+   auto Bc = Reshape(bc.Read(), Q1D, D1D);
+   auto Bot = Reshape(bot.Read(), D1Dtest-1, Q1D);
+   auto Bct = Reshape(bct.Read(), D1Dtest, Q1D);
+   auto Gct = Reshape(gct.Read(), D1D, Q1D);
+   auto op = Reshape(pa_data.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto X = Reshape(x.Read(), 3*(D1Dtest-1)*(D1Dtest-1)*D1D, NE);
+   auto Y = Reshape(y.ReadWrite(), 3*(D1D-1)*D1D*D1D, NE);
+
+   MFEM_FORALL(e, NE,
+   {
+      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
+
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               for (int c = 0; c < VDIM; ++c)
+               {
+                  mass[qz][qy][qx][c] = 0.0;
+               }
+            }
+         }
+      }
+
+      int osc = 0;
+
+      for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
+      {
+         const int D1Dz = (c == 2) ? D1D : D1D - 1;
+         const int D1Dy = (c == 1) ? D1D : D1D - 1;
+         const int D1Dx = (c == 0) ? D1D : D1D - 1;
+
+         for (int dz = 0; dz < D1Dz; ++dz)
+         {
+            double massXY[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massXY[qy][qx] = 0.0;
+               }
+            }
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               double massX[HDIV_MAX_Q1D];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  massX[qx] = 0.0;
+               }
+
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double t = X(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc, e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     massX[qx] += t * ((c == 0) ? Bc(qx,dx) : Bo(qx,dx));
+                  }
+               }
+
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const double wy = (c == 1) ? Bc(qy,dy) : Bo(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const double wx = massX[qx];
+                     massXY[qy][qx] += wx * wy;
+                  }
+               }
+            }
+
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const double wz = (c == 2) ? Bc(qz,dz) : Bo(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     mass[qz][qy][qx][c] += massXY[qy][qx] * wz;
+                  }
+               }
+            }
+         }
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }  // loop (c) over components
+
+      // Apply D operator.
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double O11 = op(qx,qy,qz,0,e);
+               const double O12 = op(qx,qy,qz,1,e);
+               const double O13 = op(qx,qy,qz,2,e);
+               const double O22 = op(qx,qy,qz,3,e);
+               const double O23 = op(qx,qy,qz,4,e);
+               const double O33 = op(qx,qy,qz,5,e);
+               const double massX = mass[qz][qy][qx][0];
+               const double massY = mass[qz][qy][qx][1];
+               const double massZ = mass[qz][qy][qx][2];
+               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
+               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+            }
+         }
+      }
+
+      // x component
+      osc = 0;
+      {
+         const int D1Dz = D1D;
+         const int D1Dy = D1D;
+         const int D1Dx = D1D - 1;
+
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            double gradXY12[MAX_D1D][MAX_D1D];
+            double gradXY21[MAX_D1D][MAX_D1D];
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  gradXY12[dy][dx] = 0.0;
+                  gradXY21[dy][dx] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massX[MAX_D1D][2];
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  for (int n = 0; n < 2; ++n)
+                  {
+                     massX[dx][n] = 0.0;
+                  }
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     const double wx = Bot(dx,qx);
+
+                     massX[dx][0] += wx * mass[qz][qy][qx][1];
+                     massX[dx][1] += wx * mass[qz][qy][qx][2];
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = Bct(dy,qy);
+                  const double wDy = Gct(dy,qy);
+
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     gradXY21[dy][dx] += massX[dx][0] * wy;
+                     gradXY12[dy][dx] += massX[dx][1] * wDy;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = Bct(dz,qz);
+               const double wDz = Gct(dz,qz);
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [0, (u_0)_{x_2}, -(u_0)_{x_1}]
+                     // (u_0)_{x_2} * (op * curl)_1 - (u_0)_{x_1} * (op * curl)_2
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
+                       e) += (gradXY21[dy][dx] * wDz) - (gradXY12[dy][dx] * wz);
+                  }
+               }
+            }
+         }  // loop qz
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }
+
+      // y component
+      {
+         const int D1Dz = D1D;
+         const int D1Dy = D1D - 1;
+         const int D1Dx = D1D;
+
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            double gradXY02[MAX_D1D][MAX_D1D];
+            double gradXY20[MAX_D1D][MAX_D1D];
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  gradXY02[dy][dx] = 0.0;
+                  gradXY20[dy][dx] = 0.0;
+               }
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               double massY[MAX_D1D][2];
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  massY[dy][0] = 0.0;
+                  massY[dy][1] = 0.0;
+               }
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int dy = 0; dy < D1Dy; ++dy)
+                  {
+                     const double wy = Bot(dy,qy);
+
+                     massY[dy][0] += wy * mass[qz][qy][qx][2];
+                     massY[dy][1] += wy * mass[qz][qy][qx][0];
+                  }
+               }
+               for (int dx = 0; dx < D1Dx; ++dx)
+               {
+                  const double wx = Bct(dx,qx);
+                  const double wDx = Gct(dx,qx);
+
+                  for (int dy = 0; dy < D1Dy; ++dy)
+                  {
+                     gradXY02[dy][dx] += massY[dy][0] * wDx;
+                     gradXY20[dy][dx] += massY[dy][1] * wx;
+                  }
+               }
+            }
+
+            for (int dz = 0; dz < D1Dz; ++dz)
+            {
+               const double wz = Bct(dz,qz);
+               const double wDz = Gct(dz,qz);
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dx = 0; dx < D1Dx; ++dx)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [-(u_1)_{x_2}, 0, (u_1)_{x_0}]
+                     // -(u_1)_{x_2} * (op * curl)_0 + (u_1)_{x_0} * (op * curl)_2
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
+                       e) += (-gradXY20[dy][dx] * wDz) + (gradXY02[dy][dx] * wz);
+                  }
+               }
+            }
+         }  // loop qz
+
+         osc += D1Dx * D1Dy * D1Dz;
+      }
+
+      // z component
+      {
+         const int D1Dz = D1D - 1;
+         const int D1Dy = D1D;
+         const int D1Dx = D1D;
+
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            double gradYZ01[MAX_D1D][MAX_D1D];
+            double gradYZ10[MAX_D1D][MAX_D1D];
+
+            for (int dy = 0; dy < D1Dy; ++dy)
+            {
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  gradYZ01[dz][dy] = 0.0;
+                  gradYZ10[dz][dy] = 0.0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               double massZ[MAX_D1D][2];
+               for (int dz = 0; dz < D1Dz; ++dz)
+               {
+                  for (int n = 0; n < 2; ++n)
+                  {
+                     massZ[dz][n] = 0.0;
+                  }
+               }
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  for (int dz = 0; dz < D1Dz; ++dz)
+                  {
+                     const double wz = Bot(dz,qz);
+
+                     massZ[dz][0] += wz * mass[qz][qy][qx][0];
+                     massZ[dz][1] += wz * mass[qz][qy][qx][1];
+                  }
+               }
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  const double wy = Bct(dy,qy);
+                  const double wDy = Gct(dy,qy);
+
+                  for (int dz = 0; dz < D1Dz; ++dz)
+                  {
+                     gradYZ01[dz][dy] += wy * massZ[dz][1];
+                     gradYZ10[dz][dy] += wDy * massZ[dz][0];
+                  }
+               }
+            }
+
+            for (int dx = 0; dx < D1Dx; ++dx)
+            {
+               const double wx = Bct(dx,qx);
+               const double wDx = Gct(dx,qx);
+
+               for (int dy = 0; dy < D1Dy; ++dy)
+               {
+                  for (int dz = 0; dz < D1Dz; ++dz)
+                  {
+                     // \hat{\nabla}\times\hat{u} is [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
+                     // (u_2)_{x_1} * (op * curl)_0 - (u_2)_{x_0} * (op * curl)_1
+                     Y(dx + ((dy + (dz * D1Dy)) * D1Dx) + osc,
+                       e) += (gradYZ10[dz][dy] * wx) - (gradYZ01[dz][dy] * wDx);
+                  }
+               }
+            }
+         }  // loop qx
+      }
+   }); // end of element loop
+}
+
 void MixedVectorCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
   if (testType == mfem::FiniteElement::CURL &&
@@ -4723,6 +5081,20 @@ void MixedVectorCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

+void MixedVectorCurlIntegrator::AddMultTransposePA(const Vector &x,
+                                                   Vector &y) const
+{
+   if (testType == mfem::FiniteElement::DIV &&
+       trialType == mfem::FiniteElement::CURL && dim == 3)
+      PAHcurlHdivApply3DTranspose(dofs1D, dofs1Dtest, quad1D, ne, mapsO->B,
+                                  mapsC->B, mapsOtest->Bt, mapsCtest->Bt,
+                                  mapsC->Gt, pa_data, x, y);
+   else
+   {
+      MFEM_ABORT("Unsupported dimension or space!");
+   }
+}
+
 void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
                                               &trial_fes,
                                               const FiniteElementSpace &test_fes)
@@ -4761,8 +5133,16 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

+   testType = test_el->GetDerivType();
+   trialType = trial_el->GetDerivType();
+
+   const bool curlSpaces = (testType == mfem::FiniteElement::CURL &&
+                            trialType == mfem::FiniteElement::CURL);
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+
   coeffDim = DQ ? 3 : 1;
-   const int ndata = DQ ? 9 : 1;
+   const int ndata = curlSpaces ? (DQ ? 9 : 1) : symmDims;

   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

@@ -4799,9 +5179,6 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
      }
   }

-   testType = test_el->GetDerivType();
-   trialType = trial_el->GetDerivType();
-
   if (trialType == mfem::FiniteElement::CURL && dim == 3)
   {
      if (coeffDim == 1)
@@ -4810,10 +5187,16 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
      }
      else
      {
-         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(), geom->J,
-                            coeff, pa_data);
+         PAHcurlHdivSetup3D(quad1D, coeffDim, ne, false, ir->GetWeights(),
+                            geom->J, coeff, pa_data);
      }
   }
+   else if (trialType == mfem::FiniteElement::DIV && dim == 3 &&
+            test_el->GetOrder() == trial_el->GetOrder())
+   {
+      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
+                        pa_data);
+   }
   else
   {
      MFEM_ABORT("Unknown kernel.");
@@ -5463,6 +5846,29 @@ void MixedVectorWeakCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
         PAHcurlL2Apply3DTranspose(dofs1D, quad1D, ndata, ne, mapsO->B,
                                   mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->Gt, pa_data, x, y);
   }
+   else if (testType == mfem::FiniteElement::CURL &&
+            trialType == mfem::FiniteElement::DIV && dim == 3)
+   {
+      PAHcurlHdivApply3DTranspose(dofs1D, dofs1D, quad1D, ne, mapsO->B,
+                                  mapsC->B, mapsO->Bt, mapsC->Bt,
+                                  mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension or space!");
+   }
+}
+
+void MixedVectorWeakCurlIntegrator::AddMultTransposePA(const Vector &x,
+                                                       Vector &y) const
+{
+   if (testType == mfem::FiniteElement::CURL &&
+       trialType == mfem::FiniteElement::DIV && dim == 3)
+   {
+      PAHcurlHdivApply3D(dofs1D, dofs1D, quad1D, ne, mapsO->B,
+                         mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G,
+                         pa_data, x, y);
+   }
   else
   {
      MFEM_ABORT("Unsupported dimension or space!");
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -372,8 +372,8 @@ void PAHcurlHdivSetup2D(const int Q1D,
               const double R21 = D2*J21;
               const double R22 = D2*J22;
               y(i11,qx,qy,e) = w_detJ * ( J22*R11 - J12*R21); // 1,1
-               y(i12,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2
-               y(i21,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1
+               y(i21,qx,qy,e) = w_detJ * ( J22*R12 - J12*R22); // 1,2 (transpose)
+               y(i12,qx,qy,e) = w_detJ * (-J21*R11 + J11*R21); // 2,1 (transpose)
               y(i22,qx,qy,e) = w_detJ * (-J21*R12 + J11*R22); // 2,2
            }
         }
@@ -389,6 +389,7 @@ void PAHcurlHdivMassApply3D(const int D1D,
                            const int NE,
                            const bool scalarCoeff,
                            const bool trialHcurl,
+                            const bool transpose,
                            const Array<double> &Bo_,
                            const Array<double> &Bc_,
                            const Array<double> &Bot_,
@@ -413,6 +414,13 @@ void PAHcurlHdivMassApply3D(const int D1D,
   auto y = Reshape(y_.ReadWrite(), 3*(D1Dtest-1)*D1Dtest*
                    (trialHcurl ? D1Dtest-1 : D1Dtest), NE);

+   const int i12 = transpose ? 3 : 1;
+   const int i13 = transpose ? 6 : 2;
+   const int i21 = transpose ? 1 : 3;
+   const int i23 = transpose ? 7 : 5;
+   const int i31 = transpose ? 2 : 6;
+   const int i32 = transpose ? 5 : 7;
+
   MFEM_FORALL(e, NE,
   {
      double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
@@ -507,13 +515,13 @@ void PAHcurlHdivMassApply3D(const int D1D,
            for (int qx = 0; qx < Q1D; ++qx)
            {
               const double O11 = op(0,qx,qy,qz,e);
-               const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,qz,e);
-               const double O13 = scalarCoeff ? 0.0 : op(2,qx,qy,qz,e);
-               const double O21 = scalarCoeff ? 0.0 : op(3,qx,qy,qz,e);
+               const double O12 = scalarCoeff ? 0.0 : op(i12,qx,qy,qz,e);
+               const double O13 = scalarCoeff ? 0.0 : op(i13,qx,qy,qz,e);
+               const double O21 = scalarCoeff ? 0.0 : op(i21,qx,qy,qz,e);
               const double O22 = scalarCoeff ? O11 : op(4,qx,qy,qz,e);
-               const double O23 = scalarCoeff ? 0.0 : op(5,qx,qy,qz,e);
-               const double O31 = scalarCoeff ? 0.0 : op(6,qx,qy,qz,e);
-               const double O32 = scalarCoeff ? 0.0 : op(7,qx,qy,qz,e);
+               const double O23 = scalarCoeff ? 0.0 : op(i23,qx,qy,qz,e);
+               const double O31 = scalarCoeff ? 0.0 : op(i31,qx,qy,qz,e);
+               const double O32 = scalarCoeff ? 0.0 : op(i32,qx,qy,qz,e);
               const double O33 = scalarCoeff ? O11 : op(8,qx,qy,qz,e);
               const double massX = mass[qz][qy][qx][0];
               const double massY = mass[qz][qy][qx][1];
@@ -601,6 +609,7 @@ void PAHcurlHdivMassApply2D(const int D1D,
                            const int NE,
                            const bool scalarCoeff,
                            const bool trialHcurl,
+                            const bool transpose,
                            const Array<double> &Bo_,
                            const Array<double> &Bc_,
                            const Array<double> &Bot_,
@@ -624,6 +633,9 @@ void PAHcurlHdivMassApply2D(const int D1D,
   auto x = Reshape(x_.Read(), 2*(D1D-1)*D1D, NE);
   auto y = Reshape(y_.ReadWrite(), 2*(D1Dtest-1)*D1Dtest, NE);

+   const int i12 = transpose ? 2 : 1;
+   const int i21 = transpose ? 1 : 2;
+
   MFEM_FORALL(e, NE,
   {
      double mass[MAX_Q1D][MAX_Q1D][VDIM];
@@ -685,8 +697,8 @@ void PAHcurlHdivMassApply2D(const int D1D,
         for (int qx = 0; qx < Q1D; ++qx)
         {
            const double O11 = op(0,qx,qy,e);
-            const double O12 = scalarCoeff ? 0.0 : op(1,qx,qy,e);
-            const double O21 = scalarCoeff ? 0.0 : op(2,qx,qy,e);
+            const double O12 = scalarCoeff ? 0.0 : op(i12,qx,qy,e);
+            const double O21 = scalarCoeff ? 0.0 : op(i21,qx,qy,e);
            const double O22 = scalarCoeff ? O11 : op(3,qx,qy,e);
            const double massX = mass[qy][qx][0];
            const double massY = mass[qy][qx][1];
@@ -785,12 +797,14 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   trial_fetype = trial_el->GetDerivType();
   test_fetype = test_el->GetDerivType();

+   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+
   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = MQ ? MQfullDim : MQsymmDim;
-   const int coeffDim = (MQ || SMQ) ? MQdim : (DQ ? DQ->GetVDim() : 1);
+   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);

-   symmetric = (MQ == NULL);
+   symmetric = (SMQ || MQ == NULL);

   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
@@ -807,7 +821,7 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   Vector coeff(coeffDim * ne * nq);
   coeff = 1.0;
   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ || SMQ)
+   if (Q || DQ || MQ)
   {
      Vector DM(DQ ? coeffDim : 0);
      DenseMatrix M;
@@ -817,34 +831,25 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      {
         MFEM_VERIFY(coeffDim == dim, "");
      }
-      if (MQ)
-      {
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-         M.SetSize(dim);
-      }
      if (SMQ)
      {
         MFEM_VERIFY(SMQ->GetSize() == dim, "");
         SM.SetSize(dim);
      }
+      else if (MQ)
+      {
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+         M.SetSize(dim);
+      }
+

      for (int e=0; e<ne; ++e)
      {
         ElementTransformation *tr = mesh->GetElementTransformation(e);
         for (int p=0; p<nq; ++p)
         {
-            if (MQ)
-            {
-               MQ->Eval(M, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dim; ++i)
-                  for (int j=0; j<dim; ++j)
-                  {
-                     coeffh(j+(i*dim), p, e) = M(i,j);
-                  }
-            }
-            else if (SMQ)
+            if (SMQ)
            {
               SMQ->Eval(SM, *tr, ir->IntPoint(p));
               int cnt = 0;
@@ -854,6 +859,16 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                     coeffh(cnt, p, e) = SM(i,j);
                  }
            }
+            else if (MQ)
+            {
+               MQ->Eval(M, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dim; ++i)
+                  for (int j=0; j<dim; ++j)
+                  {
+                     coeffh(j+(i*dim), p, e) = M(i,j);
+                  }
+            }
            else if (DQ)
            {
               DQ->Eval(DM, *tr, ir->IntPoint(p));
@@ -1024,16 +1039,16 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_curl && test_div)
      {
-         const bool scalarCoeff = !(DQ || MQ || SMQ);
+         const bool scalarCoeff = !(DQ || MQ);
         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                true, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                true, false, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
      }
      else if (trial_div && test_curl)
      {
-         const bool scalarCoeff = !(DQ || MQ || SMQ);
+         const bool scalarCoeff = !(DQ || MQ);
         PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                false, mapsO->B, mapsC->B, mapsOtest->Bt,
+                                false, false, mapsO->B, mapsC->B, mapsOtest->Bt,
                                mapsCtest->Bt, pa_data, x, y);
      }
      else
@@ -1055,10 +1070,10 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if ((trial_curl && test_div) || (trial_div && test_curl))
      {
-         const bool scalarCoeff = !(DQ || MQ || SMQ);
+         const bool scalarCoeff = !(DQ || MQ);
         PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
-                                trial_curl, mapsO->B, mapsC->B, mapsOtest->Bt,
-                                mapsCtest->Bt, pa_data, x, y);
+                                trial_curl, false, mapsO->B, mapsC->B,
+                                mapsOtest->Bt, mapsCtest->Bt, pa_data, x, y);
      }
      else
      {
@@ -1067,6 +1082,39 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
   }
 }

+void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,
+                                                Vector &y) const
+{
+   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
+   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
+   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
+   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
+
+   bool symmetricSpaces = true;
+
+   if (dim == 3 && ((trial_div && test_curl) || (trial_curl && test_div)))
+   {
+      const bool scalarCoeff = !(DQ || MQ);
+      PAHcurlHdivMassApply3D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                             trial_div, true, mapsO->B, mapsC->B, mapsOtest->Bt,
+                             mapsCtest->Bt, pa_data, x, y);
+      symmetricSpaces = false;
+   }
+   else if (dim == 2 && ((trial_curl && test_div) || (trial_div && test_curl)))
+   {
+      const bool scalarCoeff = !(DQ || MQ);
+      PAHcurlHdivMassApply2D(dofs1D, dofs1Dtest, quad1D, ne, scalarCoeff,
+                             !trial_curl, true, mapsO->B, mapsC->B, mapsOtest->Bt,
+                             mapsCtest->Bt, pa_data, x, y);
+      symmetricSpaces = false;
+   }
+
+   if (symmetricSpaces)
+   {
+      this->AddMultPA(x, y);
+   }
+}
+
 void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
                                               &trial_fes,
                                               const FiniteElementSpace &test_fes)
@@ -1,512 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-BlockBilinearForm::BlockBilinearForm(Array<FiniteElementSpace *> & fespaces_) :
-   Matrix(0), fespaces(fespaces_)
-{
-   height = 0;
-   nblocks = fespaces.Size();
-   dof_offsets.SetSize(nblocks+1);
-   tdof_offsets.SetSize(nblocks+1);
-   dof_offsets[0] = 0;
-   tdof_offsets[0] = 0;
-   for (int i =0; i<nblocks; i++)
-   {
-      dof_offsets[i+1] = fespaces[i]->GetVSize();
-      tdof_offsets[i+1] = fespaces[i]->GetTrueVSize();
-   }
-   dof_offsets.PartialSum();
-   tdof_offsets.PartialSum();
-   height = dof_offsets[nblocks];
-   width = height;
-   mat = mat_e = NULL;
-   extern_bfs = 0;
-   element_matrices = NULL;
-   diag_policy = DIAG_KEEP;
-}
-
-
-// Allocate appropriate SparseMatrix and assign it to mat
-void BlockBilinearForm::AllocMat()
-{
-   mat = new SparseMatrix(height);
-}
-
-void BlockBilinearForm::BuildProlongation()
-{
-   P = new BlockMatrix(dof_offsets, tdof_offsets);
-   R = new BlockMatrix(tdof_offsets, dof_offsets);
-   for (int i = 0; i<nblocks; i++)
-   {
-      const SparseMatrix *P_ = fespaces[i]->GetConformingProlongation();
-      const SparseMatrix *R_ = fespaces[i]->GetRestrictionMatrix();
-      P->SetBlock(i,i,const_cast<SparseMatrix*>(P_));
-      R->SetBlock(i,i,const_cast<SparseMatrix*>(R_));
-   }
-}
-
-void BlockBilinearForm::ConformingAssemble()
-{
-   Finalize(0);
-   MFEM_ASSERT(mat, "the BilinearForm is not assembled");
-
-   if (!P) { BuildProlongation(); }
-
-   SparseMatrix * Pm = P->CreateMonolithic();
-
-   SparseMatrix *Pt = Transpose(*Pm);
-
-   SparseMatrix *PtA = mfem::Mult(*Pt, *mat);
-   delete mat;
-   if (mat_e)
-   {
-      SparseMatrix *PtAe = mfem::Mult(*Pt, *mat_e);
-      delete mat_e;
-      mat_e = PtAe;
-   }
-   delete Pt;
-   mat = mfem::Mult(*PtA, *Pm);
-   delete PtA;
-   if (mat_e)
-   {
-      SparseMatrix *PtAeP = mfem::Mult(*mat_e, *Pm);
-      delete mat_e;
-      mat_e = PtAeP;
-   }
-   delete Pm;
-   height = mat->Height();
-   width = mat->Width();
-}
-
-void BlockBilinearForm::Mult(const Vector &x, Vector &y) const
-{
-   // TODO
-}
-
-
-double& BlockBilinearForm::Elem (int i, int j)
-{
-   return mat -> Elem(i,j);
-}
-
-const double& BlockBilinearForm::Elem (int i, int j) const
-{
-   return mat -> Elem(i,j);
-}
-
-MatrixInverse * BlockBilinearForm::Inverse() const
-{
-   return mat -> Inverse();
-}
-
-void BlockBilinearForm::Finalize(int skip_zeros)
-{
-   mat->Finalize(skip_zeros);
-   if (mat_e) { mat_e->Finalize(skip_zeros); }
-}
-
-/// Adds new Block Domain Integrator. Assumes ownership of @a bfi.
-void BlockBilinearForm::AddDomainIntegrator(BlockBilinearFormIntegrator *bfi)
-{
-   domain_integs.Append(bfi);
-}
-
-/// Assembles the form i.e. sums over all domain integrators.
-void BlockBilinearForm::Assemble(int skip_zeros)
-{
-   ElementTransformation *eltrans;
-   DofTransformation * doftrans_j, *doftrans_k;
-   Mesh *mesh = fespaces[0] -> GetMesh();
-   DenseMatrix elmat, *elmat_p;
-   int nblocks = fespaces.Size();
-   Array<const FiniteElement *> fe(nblocks);
-   Array<int> vdofs_j, vdofs_k;
-   Array<int> offsetvdofs_j;
-   Array<int> elementblockoffsets(nblocks+1);
-   elementblockoffsets[0] = 0;
-   Array<int> blockoffsets(nblocks+1);
-   blockoffsets[0] = 0;
-   for (int i =0; i<nblocks; i++)
-   {
-      blockoffsets[i+1] = fespaces[i]->GetVSize();
-   }
-   blockoffsets.PartialSum();
-   // mfem::out << "blockoffsets = " ; blockoffsets.Print();
-
-   if (mat == NULL)
-   {
-      AllocMat();
-   }
-
-   if (domain_integs.Size())
-   {
-      // loop through elements
-      for (int i = 0; i < mesh -> GetNE(); i++)
-      {
-         if (element_matrices)
-         {
-            elmat_p = &(*element_matrices)(i);
-         }
-         else
-         {
-            elmat.SetSize(0);
-            for (int k = 0; k < domain_integs.Size(); k++)
-            {
-               for (int j = 0; j<nblocks; j++)
-               {
-                  fe[j] = fespaces[j]->GetFE(i);
-                  elementblockoffsets[j+1] = fe[j]->GetDof();
-               }
-               elementblockoffsets.PartialSum();
-               eltrans = mesh->GetElementTransformation(i);
-               domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
-               if (elmat.Size() == 0)
-               {
-                  elmat = elemmat;
-               }
-               else
-               {
-                  elmat += elemmat;
-               }
-            }
-         }
-         if (elmat.Size() == 0)
-         {
-            continue;
-         }
-         else
-         {
-            elmat_p = &elmat;
-         }
-         vdofs.SetSize(0);
-         for (int j = 0; j<nblocks; j++)
-         {
-            doftrans_j = fespaces[j]->GetElementVDofs(i, vdofs_j);
-            int jbeg = elementblockoffsets[j];
-            int jend = elementblockoffsets[j+1]-1;
-            int offset_j = blockoffsets[j];
-            offsetvdofs_j.SetSize(vdofs_j.Size());
-
-            for (int l = 0; l<vdofs_j.Size(); l++)
-            {
-               offsetvdofs_j[l] = vdofs_j[l]<0 ? -offset_j + vdofs_j[l]
-                                  :  offset_j + vdofs_j[l];
-            }
-            vdofs.Append(offsetvdofs_j);
-            for (int k = 0; k<nblocks; k++)
-            {
-               doftrans_k = fespaces[k]->GetElementVDofs(i, vdofs_k);
-               if (doftrans_k || doftrans_j)
-               {
-                  int kbeg = elementblockoffsets[k];
-                  int kend = elementblockoffsets[k+1]-1;
-                  DenseMatrix A;
-                  elmat_p->GetSubMatrix(jbeg,jend,kbeg, kend, A);
-                  TransformDual(doftrans_j, doftrans_k, A);
-                  elmat_p->SetSubMatrix(jbeg,kbeg,A);
-               }
-            }
-         }
-         mat->AddSubMatrix(vdofs,vdofs,*elmat_p, skip_zeros);
-      }
-   }
-
-}
-
-
-
-void BlockBilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                         Vector &x,
-                                         Vector &b, OperatorHandle &A, Vector &X,
-                                         Vector &B, int copy_interior)
-{
-   FormSystemMatrix(ess_tdof_list, A);
-
-   if (!P)
-   {
-      EliminateVDofsInRHS(ess_tdof_list, x, b);
-      X.MakeRef(x, 0, x.Size());
-      B.MakeRef(b, 0, b.Size());
-      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
-   }
-   else // non conforming space
-   {
-      B.SetSize(P->Width());
-      P->MultTranspose(b, B);
-      X.SetSize(R->Height());
-
-      mfem::out << "R height, width  = " << R->Height() <<" x "<< R->Width() <<
-                std::endl;
-
-      R->Mult(x, X);
-      EliminateVDofsInRHS(ess_tdof_list, X, B);
-      if (!copy_interior) { X.SetSubVectorComplement(ess_tdof_list, 0.0); }
-   }
-
-}
-
-void BlockBilinearForm::FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                         OperatorHandle &A)
-{
-   if (!mat_e)
-   {
-      const SparseMatrix *P_ = fespaces[0]->GetConformingProlongation();
-      if (P_) { ConformingAssemble(); }
-      EliminateVDofs(ess_tdof_list, diag_policy);
-      const int remove_zeros = 0;
-      Finalize(remove_zeros);
-   }
-   A.Reset(mat, false);
-}
-
-void BlockBilinearForm::RecoverFEMSolution(const Vector &X, const Vector &b,
-                                           Vector &x)
-{
-   if (!P)
-   {
-      x.SyncMemory(X);
-   }
-   else
-   {
-      // Apply conforming prolongation
-      x.SetSize(P->Height());
-      P->Mult(X, x);
-   }
-}
-
-
-
-void BlockBilinearForm::ComputeElementMatrices()
-{
-   MFEM_ABORT("BlockBilinearForm::ComputeElementMatrices:not implemented yet")
-}
-
-void BlockBilinearForm::ComputeElementMatrix(int i, DenseMatrix &elmat)
-{
-   if (element_matrices)
-   {
-      elmat.SetSize(element_matrices->SizeI(), element_matrices->SizeJ());
-      elmat = element_matrices->GetData(i);
-      return;
-   }
-
-   int nblocks = fespaces.Size();
-   Array<const FiniteElement *> fe(nblocks);
-   ElementTransformation *eltrans;
-
-   elmat.SetSize(0);
-   if (domain_integs.Size())
-   {
-      for (int j = 0; j<nblocks; j++)
-      {
-         fe[j] = fespaces[j]->GetFE(i);
-      }
-      eltrans = fespaces[0]->GetElementTransformation(i);
-      domain_integs[0]->AssembleElementMatrix(fe, *eltrans, elmat);
-      for (int k = 1; k < domain_integs.Size(); k++)
-      {
-         domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
-         elmat += elemmat;
-      }
-   }
-   else
-   {
-      int matsize = 0;
-      for (int j = 0; j<nblocks; j++)
-      {
-         matsize += fespaces[j]->GetFE(i)->GetDof();
-      }
-      elmat.SetSize(matsize);
-      elmat = 0.0;
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                             const Vector &sol, Vector &rhs,
-                                             DiagonalPolicy dpolicy)
-{
-   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
-   // Array<int> ess_dofs, conf_ess_dofs;
-   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
-
-   // if (fes->GetVSize() == height)
-   // {
-   //    EliminateEssentialBCFromDofs(ess_dofs, sol, rhs, dpolicy);
-   // }
-   // else
-   // {
-   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
-   //    EliminateEssentialBCFromDofs(conf_ess_dofs, sol, rhs, dpolicy);
-   // }
-}
-
-void BlockBilinearForm::EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                                             DiagonalPolicy dpolicy)
-{
-   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBC: not implemented yet");
-   // Array<int> ess_dofs, conf_ess_dofs;
-   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
-
-   // if (fes->GetVSize() == height)
-   // {
-   //    EliminateEssentialBCFromDofs(ess_dofs, dpolicy);
-   // }
-   // else
-   // {
-   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
-   //    EliminateEssentialBCFromDofs(conf_ess_dofs, dpolicy);
-   // }
-}
-
-void BlockBilinearForm::EliminateEssentialBCDiag (const Array<int>
-                                                  &bdr_attr_is_ess,
-                                                  double value)
-{
-   MFEM_ABORT("BlockBilinearForm::EliminateEssentialBCDiag: not implemented yet");
-   // Array<int> ess_dofs, conf_ess_dofs;
-   // fes->GetEssentialVDofs(bdr_attr_is_ess, ess_dofs);
-
-   // if (fes->GetVSize() == height)
-   // {
-   //    EliminateEssentialBCFromDofsDiag(ess_dofs, value);
-   // }
-   // else
-   // {
-   //    fes->GetRestrictionMatrix()->BooleanMult(ess_dofs, conf_ess_dofs);
-   //    EliminateEssentialBCFromDofsDiag(conf_ess_dofs, value);
-   // }
-}
-
-void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
-                                       const Vector &sol, Vector &rhs,
-                                       DiagonalPolicy dpolicy)
-{
-   vdofs.HostRead();
-   for (int i = 0; i < vdofs.Size(); i++)
-   {
-      int vdof = vdofs[i];
-      if ( vdof >= 0 )
-      {
-         mat -> EliminateRowCol (vdof, sol(vdof), rhs, dpolicy);
-      }
-      else
-      {
-         mat -> EliminateRowCol (-1-vdof, sol(-1-vdof), rhs, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateVDofs(const Array<int> &vdofs,
-                                       DiagonalPolicy dpolicy)
-{
-   if (mat_e == NULL)
-   {
-      mat_e = new SparseMatrix(height);
-   }
-
-   // mat -> EliminateCols(vdofs, *mat_e,)
-
-   for (int i = 0; i < vdofs.Size(); i++)
-   {
-      int vdof = vdofs[i];
-      if ( vdof >= 0 )
-      {
-         mat -> EliminateRowCol (vdof, *mat_e, dpolicy);
-      }
-      else
-      {
-         mat -> EliminateRowCol (-1-vdof, *mat_e, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBCFromDofs(
-   const Array<int> &ess_dofs, const Vector &sol, Vector &rhs,
-   DiagonalPolicy dpolicy)
-{
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
-   MFEM_ASSERT(sol.Size() == height, "incorrect sol Vector size");
-   MFEM_ASSERT(rhs.Size() == height, "incorrect rhs Vector size");
-
-   for (int i = 0; i < ess_dofs.Size(); i++)
-   {
-      if (ess_dofs[i] < 0)
-      {
-         mat -> EliminateRowCol (i, sol(i), rhs, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBCFromDofs (const Array<int>
-                                                      &ess_dofs,
-                                                      DiagonalPolicy dpolicy)
-{
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
-
-   for (int i = 0; i < ess_dofs.Size(); i++)
-   {
-      if (ess_dofs[i] < 0)
-      {
-         mat -> EliminateRowCol (i, dpolicy);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateEssentialBCFromDofsDiag (
-   const Array<int> &ess_dofs,
-   double value)
-{
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
-
-   for (int i = 0; i < ess_dofs.Size(); i++)
-   {
-      if (ess_dofs[i] < 0)
-      {
-         mat -> EliminateRowColDiag (i, value);
-      }
-   }
-}
-
-void BlockBilinearForm::EliminateVDofsInRHS(
-   const Array<int> &vdofs, const Vector &x, Vector &b)
-{
-   mat_e->AddMult(x, b, -1.);
-   mat->PartMult(vdofs, x, b);
-}
-
-
-
-BlockBilinearForm::~BlockBilinearForm()
-{
-   delete mat_e;
-   delete mat;
-   delete element_matrices;
-
-   for (int k=0; k < domain_integs.Size(); k++)
-   {
-      delete domain_integs[k];
-   }
-   for (int k=0; k < trace_integs.Size(); k++)
-   {
-      delete trace_integs[k];
-   }
-   delete P;
-   delete R;
-}
-
-
-
-} // namespace mfem
@@ -1,288 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCKBILINEARFORM
-#define MFEM_BLOCKBILINEARFORM
-
-#include "../config/config.hpp"
-#include "../linalg/linalg.hpp"
-
-namespace mfem
-{
-
-/** @brief A "square matrix" operator for the associated FE space and
-    BLFIntegrators The sum of all the BLFIntegrators can be used form the matrix
-    M.  */
-class BlockBilinearForm : public Matrix
-{
-
-protected:
-
-   int nblocks;
-   Array<int> dof_offsets;
-   Array<int> tdof_offsets;
-
-   /// Sparse matrix \f$ M \f$ to be associated with the form. Owned.
-   SparseMatrix *mat;
-
-   /** @brief Sparse Matrix \f$ M_e \f$ used to store the eliminations
-        from the b.c.  Owned.
-       \f$ M + M_e = M_{original} \f$ */
-   SparseMatrix *mat_e;
-
-   /// FE spaces on which the block form lives. Not owned.
-   Array<FiniteElementSpace * > fespaces;
-
-   /** @brief Indicates the Mesh::sequence corresponding to the current state of
-      the BilinearForm. */
-   long sequence;
-
-   /** @brief Indicates the BlockBilinearFormIntegrator%s stored in #domain_integs,
-     are owned by another BlockBilinearForm. */
-   int extern_bfs;
-
-   /// Set of Domain Integrators to be applied.
-   Array<BlockBilinearFormIntegrator * > domain_integs;
-
-   /// Trace integrators.
-   Array<BlockBilinearFormIntegrator * > trace_integs;
-
-   DenseMatrix elemmat;
-   Array<int>  vdofs;
-
-   DenseTensor *element_matrices; ///< Owned.
-
-   BlockMatrix * P = nullptr; // Block Prolongation
-   BlockMatrix * R = nullptr; // Block Restriction
-
-   /** This data member allows one to specify what should be done to the
-    diagonal matrix entries and corresponding RHS values upon elimination of
-    the constrained DoFs. */
-   DiagonalPolicy diag_policy;
-
-   // Allocate appropriate SparseMatrix and assign it to mat
-   void AllocMat();
-
-   void ConformingAssemble();
-
-   void BuildProlongation();
-
-
-private:
-
-public:
-
-   /// Creates bilinear form associated with FE spaces @a *fespaces.
-   BlockBilinearForm(Array<FiniteElementSpace * > & fespaces_);
-
-   /// Get the size of the BilinearForm as a square matrix.
-   int Size() const { return height; }
-
-
-   /// Pre-allocate the internal SparseMatrix before assembly.
-   void AllocateMatrix() { if (mat == NULL) { AllocMat(); } }
-
-   /// Returns a reference to: \f$ M_{ij} \f$
-   const double &operator()(int i, int j) { return (*mat)(i,j); }
-
-
-   /// Matrix vector multiplication:  \f$ y = M x \f$
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   /** @brief Matrix vector multiplication with the original uneliminated
-       matrix.  The original matrix is \f$ M + M_e \f$ so we have:
-       \f$ y = M x + M_e x \f$ */
-   void FullMult(const Vector &x, Vector &y) const
-   { mat->Mult(x, y); mat_e->AddMult(x, y); }
-
-   virtual double &Elem(int i, int j);
-   virtual const double &Elem(int i, int j) const;
-   virtual MatrixInverse *Inverse() const;
-
-   /// Finalizes the matrix initialization.
-   virtual void Finalize(int skip_zeros = 1);
-
-   /// Returns a const reference to the sparse matrix.
-   const SparseMatrix &SpMat() const
-   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
-   }
-
-   /// Returns a reference to the sparse matrix:  \f$ M \f$
-   SparseMatrix &SpMat()
-   {
-      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
-      return *mat;
-   }
-
-   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
-   const SparseMatrix &SpMatElim() const
-   {
-      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
-      return *mat_e;
-   }
-
-   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
-   SparseMatrix &SpMatElim()
-   {
-      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
-      return *mat_e;
-   }
-
-   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
-   void AddDomainIntegrator(BlockBilinearFormIntegrator *bfi);
-
-   /// Adds new Trace Integrator. Assumes ownership of @a bfi.
-   void AddTraceIntegrator(BlockBilinearFormIntegrator *bfi);
-
-   /// Sets all sparse values of \f$ M \f$ and \f$ M_e \f$ to 'a'.
-   void operator=(const double a)
-   {
-      if (mat != NULL) { *mat = a; }
-      if (mat_e != NULL) { *mat_e = a; }
-   }
-
-   /// Assembles the form i.e. sums over all domain integrators.
-   void Assemble(int skip_zeros = 1);
-
-
-   virtual void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x,
-                                 Vector &b, OperatorHandle &A, Vector &X,
-                                 Vector &B, int copy_interior = 0);
-
-   /** @brief Form the linear system A X = B, corresponding to this bilinear
-       form and the linear form @a b(.). */
-   /** Version of the method FormLinearSystem() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
-   template <typename OpType>
-   void FormLinearSystem(const Array<int> &ess_tdof_list, Vector &x, Vector &b,
-                         OpType &A, Vector &X, Vector &B,
-                         int copy_interior = 0)
-   {
-      OperatorHandle Ah;
-      FormLinearSystem(ess_tdof_list, x, b, Ah, X, B, copy_interior);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   virtual void FormSystemMatrix(const Array<int> &ess_tdof_list,
-                                 OperatorHandle &A);
-
-   /// Form the linear system matrix A, see FormLinearSystem() for details.
-   /** Version of the method FormSystemMatrix() where the system matrix is
-       returned in the variable @a A, of type OpType, holding a *reference* to
-       the system matrix (created with the method OpType::MakeRef()). The
-       reference will be invalidated when SetOperatorType(), Update(), or the
-       destructor is called. */
-   template <typename OpType>
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OpType &A)
-   {
-      OperatorHandle Ah;
-      FormSystemMatrix(ess_tdof_list, Ah);
-      OpType *A_ptr = Ah.Is<OpType>();
-      MFEM_VERIFY(A_ptr, "invalid OpType used");
-      A.MakeRef(*A_ptr);
-   }
-
-   virtual void RecoverFEMSolution(const Vector &X, const Vector &b, Vector &x);
-
-
-   void ComputeElementMatrices();
-
-   /// Free the memory used by the element matrices.
-   void FreeElementMatrices()
-   { delete element_matrices; element_matrices = NULL; }
-
-   /// Compute the element matrix of the given element
-   /** The element matrix is computed by calling the domain integrators
-       or the one stored internally by a prior call of ComputeElementMatrices()
-       is returned when available.
-   */
-   void ComputeElementMatrix(int i, DenseMatrix &elmat);
-
-   /// Eliminate essential boundary DOFs from the system.
-   /** The array @a bdr_attr_is_ess marks boundary attributes that constitute
-       the essential part of the boundary. By default, the diagonal at the
-       essential DOFs is set to 1.0. This behavior is controlled by the argument
-       @a dpolicy. */
-   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                             const Vector &sol, Vector &rhs,
-                             DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /// Eliminate essential boundary DOFs from the system matrix.
-   void EliminateEssentialBC(const Array<int> &bdr_attr_is_ess,
-                             DiagonalPolicy dpolicy = DIAG_ONE);
-   /// Perform elimination and set the diagonal entry to the given value
-   void EliminateEssentialBCDiag(const Array<int> &bdr_attr_is_ess,
-                                 double value);
-
-   /// Eliminate the given @a vdofs.
-   /** NOTE: here, @a vdofs is a list of DOFs from all the fespaces
-       In this case the eliminations are applied to the internal \f$ M \f$
-       and @a rhs without storing the elimination matrix \f$ M_e \f$. */
-   void EliminateVDofs(const Array<int> &vdofs, const Vector &sol, Vector &rhs,
-                       DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /// Eliminate the given @a vdofs (all the fespaces), storing the eliminated part internally in \f$ M_e \f$.
-   /** This method works in conjunction with EliminateVDofsInRHS() and allows
-       elimination of boundary conditions in multiple right-hand sides. In this
-       method, @a vdofs is a list of DOFs. */
-   void EliminateVDofs(const Array<int> &vdofs,
-                       DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /** @brief Similar to
-       EliminateVDofs(const Array<int> &, const Vector &, Vector &, DiagonalPolicy)
-       but here @a ess_dofs is a marker (boolean) array on all vector-dofs
-       (@a ess_dofs[i] < 0 is true). */
-   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs, const Vector &sol,
-                                     Vector &rhs, DiagonalPolicy dpolicy = DIAG_ONE);
-
-   /** @brief Similar to EliminateVDofs(const Array<int> &, DiagonalPolicy) but
-       here @a ess_dofs is a marker (boolean) array on all vector-dofs
-       (@a ess_dofs[i] < 0 is true). */
-   void EliminateEssentialBCFromDofs(const Array<int> &ess_dofs,
-                                     DiagonalPolicy dpolicy = DIAG_ONE);
-   /// Perform elimination and set the diagonal entry to the given value
-   void EliminateEssentialBCFromDofsDiag(const Array<int> &ess_dofs,
-                                         double value);
-
-   /** @brief Use the stored eliminated part of the matrix (see
-       EliminateVDofs(const Array<int> &, DiagonalPolicy)) to modify the r.h.s.
-       @a b; @a vdofs is a list of DOFs (non-directional, i.e. >= 0). */
-   void EliminateVDofsInRHS(const Array<int> &vdofs, const Vector &x,
-                            Vector &b);
-
-
-   /// Sets diagonal policy used upon construction of the linear system.
-   /** Policies include:
-
-       - DIAG_ZERO (Set the diagonal values to zero)
-       - DIAG_ONE  (Set the diagonal values to one)
-       - DIAG_KEEP (Keep the diagonal values)
-   */
-   void SetDiagonalPolicy(DiagonalPolicy policy)
-   {
-      diag_policy = policy;
-   }
-
-   /// Destroys bilinear form.
-   virtual ~BlockBilinearForm();
-
-};
-
-} // namespace mfem
-
-#endif
@@ -1,136 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-void BlockBilinearFormIntegrator::AssembleElementMatrix(
-   const Array<const FiniteElement *> &el,
-   ElementTransformation &Trans,
-   DenseMatrix &elmat)
-{
-   mfem_error ("BlockBilinearFormIntegrator::AssembleElementMatrix\n"
-               "   is not implemented for this class.");
-}
-
-void BlockLinearFormIntegrator::AssembleRHSElementVect(
-   const Array<const FiniteElement *> &el,
-   ElementTransformation &Trans,
-   Vector &elvect)
-{
-   mfem_error ("BlockLinearFormIntegrator::AssembleElementVector\n"
-               "   is not implemented for this class.");
-}
-
-/** Given a particular Finite Element computes the element vector */
-void TestBlockBilinearFormIntegrator::AssembleElementMatrix
-(const Array<const FiniteElement *> &el,
- ElementTransformation &Trans,
- DenseMatrix &elmat)
-{
-   int nd = 0;
-   int nblocks = el.Size();
-   Array<int> offsets(nblocks+1);
-   offsets[0] = 0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      nd += el[i]->GetDof();
-      offsets[i+1] = el[i]->GetDof();
-   }
-   offsets.PartialSum();
-   elmat.SetSize(nd);
-   elmat = 0.0;
-   DenseMatrix dmat;
-
-   if (blfis.NumRows())
-   {
-      // Get the matrices directly from the existing BilinearFormIntegrators
-      for (int i = 0; i<nblocks; i++)
-      {
-         // mfem::out << "i = " << i << std::endl;
-         int offset_i = offsets[i];
-         const FiniteElement * fe_i = el[i];
-         for (int j = 0; j<nblocks; j++)
-         {
-            // mfem::out << "j = " << j << std::endl;
-            BilinearFormIntegrator * blfi = blfis(i,j);
-            if (!blfi) { continue; }
-            if (j == i)
-            {
-               blfi->AssembleElementMatrix(*fe_i,Trans,dmat);
-               // mfem::out << "j 1 = " << j << std::endl;
-               elmat.SetSubMatrix(offset_i,dmat);
-            }
-            else
-            {
-               const FiniteElement * fe_j = el[j];
-               blfi->AssembleElementMatrix2(*fe_j,*fe_i,Trans,dmat);
-               // mfem::out << "j 2 = " << j << std::endl;
-               int offset_j = offsets[j];
-               elmat.SetSubMatrix(offset_i,offset_j,dmat);
-            }
-         }
-      }
-      return;
-   }
-
-   // else compute the matrices
-   elmat = 25.0;
-   // TODO
-
-}
-
-/** Given a particular Finite Element computes the element vector */
-void TestBlockLinearFormIntegrator::AssembleRHSElementVect
-(const Array<const FiniteElement *> &el,
- ElementTransformation &Trans,
- Vector &elvector)
-{
-   int nd = 0;
-   int nblocks = el.Size();
-   Array<int> offsets(nblocks+1);
-   offsets[0] = 0;
-   for (int i = 0; i<nblocks; i++)
-   {
-      nd += el[i]->GetDof();
-      offsets[i+1] = el[i]->GetDof();
-   }
-   offsets.PartialSum();
-   elvector.SetSize(nd);
-   elvector = 0.0;
-   Vector subvector;
-
-   if (lfis.Size())
-   {
-      // Get the matrices directly from the existing BilinearFormIntegrators
-      for (int i = 0; i<nblocks; i++)
-      {
-         int offset = offsets[i];
-         const FiniteElement * fe_i = el[i];
-         LinearFormIntegrator * lfi = lfis[i];
-         if (!lfi)
-         {
-            continue;
-         }
-         lfi->AssembleRHSElementVect(*fe_i,Trans,subvector);
-         elvector.SetVector(subvector,offset);
-      }
-      return;
-   }
-
-   // else, compute the block linear form integrator
-   // elvector = 1.0;
-   // TODO
-}
-
-} // namespace mfem
@@ -1,147 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCKINTEG
-#define MFEM_BLOCKINTEG
-
-#include "../config/config.hpp"
-#include "fe.hpp"
-#include "coefficient.hpp"
-#include "fespace.hpp"
-
-namespace mfem
-{
-/** The abstract base class BlockBilinearFormIntegrator is
- a generalization of the BilinearFormIntegrator class suitable
- for block formulations. */
-class BlockBilinearFormIntegrator
-{
-protected:
-   const IntegrationRule *IntRule;
-   BlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : IntRule(ir) { }
-public:
-
-
-   /// Given a particular Finite Element computes the element matrix elmat.
-   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
-                                      ElementTransformation &Trans,
-                                      DenseMatrix &elmat);
-
-   virtual ~BlockBilinearFormIntegrator() { }
-};
-
-/** The abstract base class BlockBilinearFormIntegrator is
- a generalization of the BilinearFormIntegrator class suitable
- for block formulations. */
-class BlockLinearFormIntegrator
-{
-protected:
-   const IntegrationRule *IntRule;
-   BlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : IntRule(ir) { }
-
-public:
-   /// Given a particular Finite Element computes the element matrix elmat.
-   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
-                                       ElementTransformation &Trans,
-                                       Vector &elvect);
-
-   virtual ~BlockLinearFormIntegrator() { }
-};
-
-
-
-
-class TestBlockBilinearFormIntegrator: public BlockBilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-   Array<const FiniteElementSpace * > fespaces;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
-
-   Array2D<BilinearFormIntegrator *> blfis;
-
-
-public:
-
-   TestBlockBilinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : BlockBilinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
-
-   /// Construct a mass integrator with coefficient q
-   TestBlockBilinearFormIntegrator(Coefficient &q,
-                                   const IntegrationRule *ir = NULL)
-      : BlockBilinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL)   { }
-
-
-   TestBlockBilinearFormIntegrator(Array2D<BilinearFormIntegrator *> blfis_)
-      : BlockBilinearFormIntegrator(NULL), blfis(blfis_) { }
-
-   void SetIntegrators(Array2D<BilinearFormIntegrator *> blfis_)
-   {
-      blfis = blfis_;
-   }
-
-
-   /** Given a particular Finite Element computes the element matrix
-       elmat. */
-   virtual void AssembleElementMatrix(const Array<const FiniteElement *> &el,
-                                      ElementTransformation &Trans,
-                                      DenseMatrix &elmat);
-
-   virtual ~TestBlockBilinearFormIntegrator() { }
-
-};
-
-/** Class for local vector assembly */
-class TestBlockLinearFormIntegrator: public BlockLinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-   Array<const FiniteElementSpace * > fespaces;
-   const DofToQuad *maps;         ///< Not owned
-   const GeometricFactors *geom;  ///< Not owned
-   int dim, ne, nq, dofs1D, quad1D;
-   Array<LinearFormIntegrator *> lfis;
-
-public:
-
-   TestBlockLinearFormIntegrator(const IntegrationRule *ir = NULL)
-      : BlockLinearFormIntegrator(ir), Q(NULL), maps(NULL), geom(NULL) { }
-
-   /// Construct a test linear integrator with coefficient q
-   TestBlockLinearFormIntegrator(Coefficient &q, const IntegrationRule *ir = NULL)
-      : BlockLinearFormIntegrator(ir), Q(&q), maps(NULL), geom(NULL) { }
-
-
-   TestBlockLinearFormIntegrator(Array<LinearFormIntegrator *> lfis_)
-      : BlockLinearFormIntegrator(NULL), lfis(lfis_) { }
-
-   void SetIntegrators(Array<LinearFormIntegrator *> lfis_)
-   {
-      lfis = lfis_;
-   }
-
-   /** Given a particular Finite Element computes the element vector */
-   virtual void AssembleRHSElementVect(const Array<const FiniteElement *> &el,
-                                       ElementTransformation &Trans,
-                                       Vector &elvector);
-
-
-};
-
-
-} // namespace mfem
-
-
-#endif
@@ -1,123 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#include "fem.hpp"
-
-namespace mfem
-{
-
-BlockLinearForm::BlockLinearForm(Array<FiniteElementSpace * > & fespaces_) :
-   Vector(0), fespaces(fespaces_)
-{
-   int s = 0;
-   int nblocks = fespaces.Size();
-   for (int i =0; i<nblocks; i++)
-   {
-      s += fespaces[i]->GetVSize();
-   }
-   // mfem::out << "size = " << size << std::endl;
-
-   SetSize(s);
-
-}
-
-
-void BlockLinearForm::AddDomainIntegrator(BlockLinearFormIntegrator *lfi)
-{
-   domain_integs.Append(lfi);
-}
-
-void BlockLinearForm::Assemble()
-{
-   ElementTransformation *eltrans;
-   DofTransformation *doftrans;
-   Mesh *mesh = fespaces[0] -> GetMesh();
-   Vector subvect,elvect, *elvect_p;
-
-   int nblocks = fespaces.Size();
-   Array<const FiniteElement *> fe(nblocks);
-   Array<int> offsetvdofs;
-   Array<int> elementblockoffsets(nblocks+1);
-   elementblockoffsets[0] = 0;
-   Array<int> blockoffsets(nblocks+1);
-   blockoffsets[0] = 0;
-   for (int i =0; i<nblocks; i++)
-   {
-      blockoffsets[i+1] = fespaces[i]->GetVSize();
-   }
-   blockoffsets.PartialSum();
-
-   Vector::operator=(0.0);
-
-   if (domain_integs.Size())
-   {
-      // loop through elements
-      for (int i = 0; i < mesh -> GetNE(); i++)
-      {
-         elvect.SetSize(0);
-         for (int k = 0; k < domain_integs.Size(); k++)
-         {
-            for (int j = 0; j<nblocks; j++)
-            {
-               fe[j] = fespaces[j]->GetFE(i);
-               elementblockoffsets[j+1] = fe[j]->GetDof();
-            }
-            elementblockoffsets.PartialSum();
-            eltrans = mesh->GetElementTransformation(i);
-
-            domain_integs[k]->AssembleRHSElementVect(fe, *eltrans, elemvect);
-            if (elvect.Size() == 0)
-            {
-               elvect = elemvect;
-            }
-            else
-            {
-               elvect += elemvect;
-            }
-         }
-         if (elvect.Size() == 0)
-         {
-            continue;
-         }
-         else
-         {
-            elvect_p = &elvect;
-         }
-
-         double *data = elvect_p->GetData();
-
-         for (int j = 0; j<nblocks; j++)
-         {
-            doftrans = fespaces[j]->GetElementVDofs(i, vdofs);
-            int offset = blockoffsets[j];
-            offsetvdofs.SetSize(vdofs.Size());
-            for (int l = 0; l<vdofs.Size(); l++)
-            {
-               offsetvdofs[l] = vdofs[l]<0 ? -offset + vdofs[l]
-                                :  offset + vdofs[l];
-            }
-            int jbeg = elementblockoffsets[j];
-            int jend = elementblockoffsets[j+1]-1;
-            subvect.SetSize(jend-jbeg+1);
-            subvect.SetData(&data[jbeg]);
-
-            if (doftrans)
-            {
-               doftrans->TransformDual(subvect);
-            }
-            AddElementVector(offsetvdofs,subvect);
-         }
-      }
-   }
-}
-
-
-} // name space mfem
@@ -1,49 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BLOCKLINEARFORM
-#define MFEM_BLOCKLINEARFORM
-
-#include "../config/config.hpp"
-#include "../linalg/linalg.hpp"
-
-namespace mfem
-{
-
-
-class BlockLinearForm : public Vector
-{
-protected:
-   /// FE spaces on which the LinearForm lives. Not owned.
-   Array<FiniteElementSpace * > fespaces;
-
-   /// Set of Domain Integrators to be applied.
-   Array<BlockLinearFormIntegrator*> domain_integs;
-
-   Vector elemvect;
-   Array<int> vdofs;
-
-public:
-   BlockLinearForm(Array<FiniteElementSpace * > & fespaces_);
-
-   /// Adds new Domain Integrator. Assumes ownership of @a lfi.
-   void AddDomainIntegrator(BlockLinearFormIntegrator *lfi);
-
-   /// Assembles the block linear form i.e. sums over all domain integrators.
-   void Assemble();
-
-
-};
-
-} // namespace mfem
-
-
-#endif
--- a/Show More
+++ b/Show More