Try the 'lgtm' CodeQL queries

Try the 'security-and-quality' CodeQL queries
Try the 'security-extended' CodeQL queries
2022-08-25 15:25:59 -07:00 · 2022-08-25 12:30:23 -07:00 · 2022-08-25 10:13:22 -07:00 · 2022-08-24 17:11:03 -07:00 · 2022-08-24 17:07:14 -07:00 · 2022-08-23 13:18:50 -07:00
277 changed files with 15240 additions and 7772 deletions
@@ -1,10 +1,13 @@
 version: '{build}'

 # https://www.appveyor.com/docs/build-environment/#build-worker-images
-image: Visual Studio 2017
+image: Visual Studio 2019

 install:

+# Start from outside clone directory
+- cd ..
+
 # Install MS-MPI
 - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
 - MSMpiSetup.exe -unattend
@@ -15,6 +18,11 @@ install:
 - msmpisdk.msi /passive
 - set PATH=C:\Program Files\Microsoft MPI\Bin;%PATH%

+# Set MSMPI environment variables needed for CMake detection
+- set MSMPI_LIB32=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86
+- set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
+- set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
+
 # Install METIS, use a mirror because the original source server is not always
 # up. Original url:
 #    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
@@ -25,20 +33,24 @@ install:
 - cmake -H. -Bbuild
 # -DCMAKE_BUILD_TYPE=Release
 - cmake --build build
+- set METIS_PATH=%cd%
 - cd ..

 # Install hypre
 - ps: Start-FileDownload 'https://github.com/hypre-space/hypre/archive/v2.19.0.tar.gz'
 - 7z x v2.19.0.tar.gz -so | 7z x -si -ttar > nul
 - cd hypre-2.19.0/src
- cmake -H. -Bbuild -DMPI_C_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DMPI_C_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include"
+- cmake -H. -Bbuild
 - cmake --build build
 - cmake --build build --target install
 - cd ../..

+# Return to clone directory
+- cd %APPVEYOR_BUILD_FOLDER%
+
 # MFEM
 before_build:
- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DMPI_CXX_LIBRARIES="C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x86\msmpi.lib" -DMPI_CXX_INCLUDE_PATH="C:\Program Files (x86)\Microsoft SDKs\MPI\Include" -DHYPRE_DIR=%cd%\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%cd%\metis-5.1.0\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%cd%\metis-5.1.0\include
+- cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_parallel -DMFEM_USE_MPI=TRUE -DMFEM_USE_METIS_5=TRUE -DHYPRE_DIR=%cd%\..\hypre-2.19.0\src\hypre -DMETIS_LIBRARIES=%METIS_PATH%\build\libmetis\Debug\metis.lib -DMETIS_INCLUDE_DIRS=%METIS_PATH%\include
 - cmake -H. -DCMAKE_INSTALL_PREFIX=install -Bbuild_serial -DMFEM_USE_MPI=FALSE

 build_script:
@@ -0,0 +1,82 @@
+name: "Docker"
+
+on:
+
+  # Always have a base image ready to go - this is a nightly build
+  schedule:
+    - cron: 0 3 * * *
+
+  # Allow manual trigger of a build
+  workflow_dispatch:
+
+  # On push to main we build and deploy images
+  push:
+    branches:
+      - master
+
+  # Publish packages on release
+  release:
+    types: [published]
+
+jobs:
+  build:
+    if: github.repository == 'mfem/mfem' # Don't run in forks
+    permissions:
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+
+        # Dockerfiles to build, a matrix supports future expanded builds
+        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
+                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
+
+    runs-on: ubuntu-latest
+    name: Build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Make Space For Build
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+
+       # It's easier to reference named variables than indexes of the matrix
+      - name: Set Environment
+        env:
+          dockerfile: ${{ matrix.container[0] }}
+          uri: ${{ matrix.container[1] }}
+        run: |
+          echo "dockerfile=$dockerfile" >> $GITHUB_ENV
+          echo "uri=$uri" >> $GITHUB_ENV
+
+      - name: Pull previous layers for cache
+        run: docker pull ${uri}:latest || echo "No container to pull"
+
+      - name: Build Container
+        run: |
+           container=$uri:latest
+           docker build -f ${dockerfile} -t ${container} .
+           echo "container=$container" >> $GITHUB_ENV
+
+      - name: GHCR Login
+        if: (github.event_name != 'pull_request')
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Deploy
+        if: (github.event_name != 'pull_request')
+        run: |
+            docker push ${container}
+
+      - name: Tag and Push Release
+        if: (github.event_name == 'release')
+        run: |
+            tag=${GITHUB_REF#refs/tags/}
+            echo "Tagging and releasing ${uri}:${tag}"
+            docker tag ${uri}:latest ${uri}:${tag}
+            docker push ${uri}:${tag}
@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.

 # In this CI section, we build different variants of mfem and run test on them.
-name: builds-and-tests
+name: "Tests"

 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -27,6 +27,7 @@ on:
      - master
      - next
  pull_request:
+  workflow_dispatch:

 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -46,11 +47,18 @@ jobs:
  builds-and-tests:
    strategy:
      matrix:
-        os: [ubuntu-18.04, macos-10.15]
+        os: [ubuntu-latest, macos-latest, windows-latest]
        target: [dbg, opt]
        mpi: [seq, par]
-        build-system: [make]
+        build-system: [make, cmake]
        hypre-target: [int32]
+        exclude:
+          - os: ubuntu-latest
+            build-system: cmake
+          - os: macos-latest
+            build-system: cmake
+          - os: windows-latest
+            build-system: make
        # 'include' allows us to:
        # - Add a variable to all jobs without creating a new matrix dimension.
        #   Codecov is defined that way.
@@ -64,13 +72,15 @@ jobs:
            codecov: NO
          - target: opt
            codecov: YES
-          - os: ubuntu-18.04
+          - os: windows-latest
+            codecov: NO
+          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
-          - os: ubuntu-18.04
+          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
@@ -102,33 +112,37 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-18.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-18.04'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install lcov

+    - name: get MPI (Windows)
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      uses: mpi4py/setup-mpi@v1.0.3
+
    # Get Hypre through cache, or build it.
    # Install will only run on cache miss.
    - name: cache hypre
@@ -137,36 +151,67 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
+      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
        target: ${{ matrix.hypre-target }}
+        build-system: make
+
+    - name: get hypre (Windows)
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
+      uses: mfem/github-actions/build-hypre@v2.2
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}
+        build-system: cmake

    # Get Metis through cache, or build it.
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: install metis
-      if: matrix.mpi == 'par' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

+    - name: cache vcpkg (Windows)
+      id: vcpkg-cache
+      uses: actions/cache@v3
+      with:
+        path: vcpkg_cache
+        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
+
+    - name: prepare binary cache location
+      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p vcpkg_cache
+
+    - name: install metis (Windows)
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      env:
+        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+      run: |
+        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
+
    # MFEM build and test
    - name: build
-      uses: mfem/github-actions/build-mfem@v2.1
+      uses: mfem/github-actions/build-mfem@v2.2
+      env:
+        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
      with:
        os: ${{ matrix.os }}
        target: ${{ matrix.target }}
@@ -176,6 +221,8 @@ jobs:
        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
        metis-dir: ${{ env.METIS_TOP_DIR }}
        mfem-dir: ${{ env.MFEM_TOP_DIR }}
+        config-options: ${{ env.MFEM_EXTRA_CONFIG }}
+        library-only: ${{ matrix.target == 'dbg' }}

    # Run checks (and only checks) on debug targets
    - name: checks
@@ -193,10 +240,27 @@ jobs:
      run: |
        cd ${{ env.MFEM_TOP_DIR }} && make test

-    - name: cmake unit tests
-      if: matrix.build-system == 'cmake'
+    - name: cmake checks
+      if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
      run: |
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure
+        CTEST_CONFIG="Debug"
+        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
+      shell: bash
+
+    - name: cmake unit tests (Ubuntu 20.04)
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
+      run: |
+        CTEST_CONFIG="Release"
+        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
+        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
+      shell: bash
+
+    - name: cmake tests
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
+      run: |
+        CTEST_CONFIG="Release"
+        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
+      shell: bash

    # Code coverage (process and upload reports)
    - name: codecov
@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "Static Analysis"
+
+on:
+  push:
+    branches: [ "master", "next"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+        queries: lgtm
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: build-analysis
+name: "Build Analysis"

 permissions:
  actions: write
@@ -20,6 +20,7 @@ on:
      - master
      - next
  pull_request:
+  workflow_dispatch:

 env:
  HYPRE_ARCHIVE: v2.19.0.tar.gz
@@ -30,7 +31,7 @@ env:

 jobs:
  gitignore:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest

    steps:
    - name: Cancel Previous Runs
@@ -53,11 +54,11 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2

    - name: Get Hypre
      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.0
+      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
        dir: ${{ env.HYPRE_TOP_DIR }}
@@ -68,18 +69,18 @@ jobs:
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.0
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: Install Metis
      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.0
+      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
        dir: ${{ env.METIS_TOP_DIR }}

    # MFEM build and test
    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.0
+      uses: mfem/github-actions/build-mfem@v2.2
      with:
        os: ${{ runner.os }}
        target: opt
@@ -9,23 +9,35 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: repo-check
+name: "Checks"

 permissions:
  actions: write

 on:
  push:
+  pull_request:
+  workflow_dispatch:
+
+# This workflow is run on pushes to any branch in the MFEM repo (with or without
+# PRs), as well as on updates to PRs from forks. In particular, we do not
+# duplicate work by running on both pushes and updates to local PRs. We do that
+# by checking if the workflow trigger is 'push' ("github.event_name == 'push'")
+# and if we are in a fork ("github.event.pull_request.head.repo.full_name !=
+# github.repository").

 jobs:
  file-headers-check:
-    runs-on: ubuntu-18.04
-
+    runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.9.0
      with:
        access_token: ${{ github.token }}
+
    - name: checkout mfem
      uses: actions/checkout@v2

@@ -49,7 +61,10 @@ jobs:
      continue-on-error: true

    - name: wrap-up
-      if: steps.copyright.outcome != 'success' || steps.license.outcome != 'success' || steps.release.outcome != 'success'
+      if: |
+        steps.copyright.outcome != 'success' ||
+        steps.license.outcome != 'success' ||
+        steps.release.outcome != 'success'
      run: |
        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
          echo "copyright check failed, unroll log for details"
@@ -63,23 +78,27 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-18.04
-
+    runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2

    - name: get astyle
      run: |
-        sudo apt-get install astyle=3.1-1ubuntu2
+        sudo apt-get install astyle

    - name: style check
      run: |
        ./config/githooks/pre-push --style

  documentation:
-    runs-on: ubuntu-18.04
-
+    runs-on: ubuntu-latest
+    if: |
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -87,6 +106,8 @@ jobs:
    - name: get doxygen and graphviz
      run: |
        sudo apt-get install doxygen graphviz
+        cd doc
+        doxygen -u CodeDocumentation.conf.in 2>/dev/null

    - name: build documentation
      run: |
@@ -94,9 +115,12 @@ jobs:
        ./runtest documentation

  branch-history:
-    if: github.ref != 'refs/heads/next' && github.ref != 'refs/heads/master'
-    runs-on: ubuntu-18.04
-
+    if: |
+      github.ref != 'refs/heads/next' &&
+      github.ref != 'refs/heads/master' &&
+      (github.event_name == 'push' ||
+       github.event.pull_request.head.repo.full_name != github.repository)
+    runs-on: ubuntu-latest
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -261,12 +261,14 @@ miniapps/performance/sol.*

 miniapps/shifted/distance
 miniapps/shifted/ParaViewDistance
+miniapps/shifted/ParaViewLSF
 miniapps/shifted/extrapolate
 miniapps/shifted/ParaViewExtrapolate
 miniapps/shifted/diffusion
 miniapps/shifted/diffusion.mesh
 miniapps/shifted/diffusion.gf
 miniapps/shifted/ParaViewDiffusion
+miniapps/shifted/lsf_integral

 miniapps/tools/display-basis
 miniapps/tools/load-dc
@@ -305,6 +307,8 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh

+miniapps/hooke/hooke
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -10,17 +10,23 @@

 Version 4.4.1 (development)
 ===========================
- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
-  spatial Gaussian white noise.

- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
-  See fem/estimators.hpp.
+Meshing improvements
+--------------------
+- Added support for mixed meshes and pyramids in GSLIB-FindPoints.

- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
-  which provides parallel non-conforming, non-matching, variational, volumetric
-  mesh information transfer. With ParMortarAssember, fields can be exchanged
-  between arbitrarily distributed and unrelated finite element meshes in a
-  variationally consistent way.
+Discretization improvements
+---------------------------
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration.
+
+- Added support for partial assembly and fully matrix-free operators on mixed
+  meshes (different element types and p-adaptivity) through libCEED, including
+  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
+  currently limited by MFEM capabilities, i.e. 2D serial meshes. All mixed
+  element topologies are supported in serial and parallel: segment, triangle,
+  square, tetrahedron, cube, prism, and pyramid.

 - Added full assembly and device support for several LinearForm integrators:
  * DomainLF: (f, v)
@@ -28,12 +34,59 @@ Version 4.4.1 (development)
  * DomainLFGrad: (f, grad(v))
  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))

+- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
+  spatial Gaussian white noise.
+
+- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
+  See fem/estimators.hpp.
+
+Linear and nonlinear solvers
+----------------------------
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new elasticity miniapp, Hooke, that showcases a low-level approach of
+  using MFEM to solve a nonlinear elasticity problem based on the fundamental
+  finite element operator decomposition. The miniapp also integrates with
+  automatic differentiation tools like a native dual number implementation or a
+  third party library such as Enzyme. See miniapps/elasticity for more details.
+
 - Add a new example code, Example 33/33p, to demonstrate the solution of
  spectral fractional PDEs with MFEM.

- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration.
+Integrations, testing and documentation
+---------------------------------------
+- Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
+
+- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
+  which provides parallel non-conforming, non-matching, variational, volumetric
+  mesh information transfer. With ParMortarAssember, fields can be exchanged
+  between arbitrarily distributed and unrelated finite element meshes in a
+  variationally consistent way.
+
+- Added support for the LLVM-based automatic differentiation tool Enzyme, see
+  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
+  header are provided. The functionality and interaction are demonstrated in a
+  new miniapp in miniapps/elasticity.
+
+- Added example for body-fitted volumetric and shape integration using the
+  Algoim library.
+
+- Added Windows 2022 CI testing with GitHub actions.
+
+Miscellaneous
+-------------
+- Various other simplifications, extensions, and bugfixes in the code.
+
+
+- Added boundary elimination with device support for `SparseMatrix` and
+  `HypreParMatrix`.
+  
+- When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
+  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
+  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).
+
+- Added TMOP metrics for mesh untangling and worst-case quality improvement.

 Version 4.4, released on March 21, 2022
 =======================================
@@ -66,6 +119,11 @@ Meshing improvements
 - Added a simpler interface to access mesh face information, see FaceInformation
  and GetFaceInformation in the Mesh class.

+- Added the method ParMesh::GetSerialMesh() that reconstructs a partitioned
+  parallel mesh on a given single rank. Also, added the method
+  ParMesh::PrintAsSerial() that saves the reconstructed serial mesh to a C++
+  stream on rank 0.
+
 - Gmsh meshes where all elements have zero physical tag (the default Gmsh output
  format if no physical groups are defined) are now successfully loaded, and
  elements are reassigned attribute number 1.
@@ -155,6 +213,9 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
  formatting. See the "make style" target.

+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New
@@ -165,6 +226,13 @@ Miscellaneous

 - Fixed several MinGW build issues on Windows.

+- In various places in the library, replace the use of 'long' with 'long long'
+  to better support Win64 builds where 'long' is 32-bit and 'long long' is
+  64-bit. On Linux and MacOS, both types are typically 64-bit.
+
+- Update various "MemoryUsage" methods to return 'std::size_t' instead of 'long'
+  since the latter is 32-bit in Win64 builds.
+
 - Added 'double' atomicAdd implementation for previous versions of CUDA.

 - HypreParVector and Vector now support C++ move semantics, and the copy
@@ -136,6 +136,8 @@ if (MFEM_USE_CUDA)
       "CUDA flags set for MFEM" FORCE)
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
+   set(CUBLAS_FOUND TRUE)
+   set(CUSBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -268,7 +270,7 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
  if(APPLE)
    # On macOS, the compiler needs additional help to find the <omp.h> header.
    # See issue #2642 for more information.
-    include_directories(${OpenMP_CXX_INCLUDE_DIRS})
+    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
  endif(APPLE)
 endif()

@@ -431,6 +433,11 @@ if (MFEM_USE_CALIPER)
  find_package(Caliper REQUIRED)
 endif()

+# Algoim
+if (MFEM_USE_ALGOIM)
+  find_package(Algoim REQUIRED)
+endif()
+
 # ADIOS2 for parallel I/O
 if (MFEM_USE_ADIOS2)
   find_package(ADIOS2 REQUIRED)
@@ -447,6 +454,11 @@ if (MFEM_USE_PARELAG)
   find_package(PARELAG REQUIRED)
 endif()

+# Enzyme
+if (MFEM_USE_ENZYME)
+   find_package(ENZYME REQUIRED)
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -473,8 +485,8 @@ endif()
 set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH)
+    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
+    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -489,7 +501,6 @@ endforeach(TPL)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
-include_directories(${TPL_INCLUDE_DIRS})

 if (OPENMP_FOUND)
  message(STATUS "MFEM: using package OpenMP")
@@ -546,6 +557,15 @@ target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES})
 if (MINGW)
  target_link_libraries(mfem PRIVATE ws2_32)
 endif()
+if (MSVC)
+  target_compile_options(mfem PUBLIC "/wd4819")
+endif()
+message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
+target_include_directories(mfem
+  PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+    ${TPL_INCLUDE_DIRS})
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")

@@ -674,6 +694,10 @@ set(INSTALL_LIB_DIR lib
 set(INSTALL_CMAKE_DIR lib/cmake/mfem
  CACHE PATH "Relative path for installing cmake config files.")

+target_include_directories(mfem
+  PUBLIC
+    $<INSTALL_INTERFACE:${INSTALL_INCLUDE_DIR}>)
+
 # The 'install' target will not depend on 'all'.
 # set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)

@@ -707,7 +731,7 @@ endif()
 if (MFEM_USE_CEED)
  install(DIRECTORY ${MFEM_SOURCE_DIRS}
    DESTINATION ${INSTALL_INCLUDE_DIR}/mfem
-    FILES_MATCHING PATTERN "fem/ceed/*.h")
+    FILES_MATCHING PATTERN "fem/ceed/integrators/*/*.h")
 endif()

 # Install ${HEADERS}
@@ -738,14 +762,8 @@ export(TARGETS ${PROJECT_NAME}
 # TODO: How do we register the install-tree? Replacing the build-tree?
 export(PACKAGE ${PROJECT_NAME})

-# Extract the include directories required to use MFEM
-get_target_property(MFEM_TPL_INCLUDE_DIRS mfem INCLUDE_DIRECTORIES)
-if (NOT MFEM_TPL_INCLUDE_DIRS)
-  set(MFEM_TPL_INCLUDE_DIRS "")
-endif()
-
 # This is the build-tree version
-set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${MFEM_TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${PROJECT_BINARY_DIR} ${TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${PROJECT_BINARY_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}/MFEMConfig.cmake
@@ -753,7 +771,7 @@ configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR)

 # This is the version that will be installed
-set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${MFEM_TPL_INCLUDE_DIRS})
+set(INCLUDE_INSTALL_DIRS ${INSTALL_INCLUDE_DIR}  ${TPL_INCLUDE_DIRS})
 set(LIB_INSTALL_DIR ${INSTALL_LIB_DIR})
 configure_package_config_file(config/cmake/MFEMConfig.cmake.in
  ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/MFEMConfig.cmake
@@ -131,6 +131,7 @@ The MFEM source code has the following structure:
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
+  │   ├── hooke
  │   ├── meshing
  │   ├── mtop
  │   ├── navier
@@ -471,6 +471,14 @@ MFEM_USE_CODIPACK = YES/NO
  Enable automatic differentiation using the CoDiPack library.
  www.scicomp.uni-kl.de/codi/

+MFEM_USE_ALGOIM = YES/NO
+  Enable the usage of Algoim - a collection of high-order accurate numerical 
+  methods and C++ algorithms for working with implicitly-defined geometry and 
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
+  provides interface to Algoim v1. Thus, to check out the specific state use: 
+  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
+  https://algoim.github.io
+
 MFEM_USE_ADFORWARD = YES/NO
  Enable forward mode for AD packages. This option is valid
  only if the AD package supports two modes (backward/forward).
@@ -550,6 +558,14 @@ MFEM_USE_PARELAG = YES/NO
   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
   currently only concerns the miniapps.

+MFEM_USE_ENZYME = YES/NO
+   Enables automatic differentiation support through the LLVM plugin Enzyme.
+   This requires the compiler to be set to clang (>=14.0.0). We also advise to
+   use the link time optimization (LTO) plugin, to enable functions that you
+   define over multiple files (compilation units) and want to be differentiated
+   automatically, to work. This requires to also use LLVM/LLD for linking.
+   Recommended options are in config/defaults.mk.
+
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -738,6 +754,20 @@ The specific libraries and their options are:
  Options: GSLIB_OPT, GSLIB_LIB.
  Versions: GSLIB >= 1.0.7.

+- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
+  headers so it just needs to be downloaded at the same level as MFEM. Download
+  the specific version we use as:
+  "git clone https://github.com/algoim/algoim.git;
+   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
+  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
+  Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
+  "ln -s blitz-1.0.2 blitz".
+  Build Blitz using CMake as:
+  "cmake . -DCMAKE_INSTALL_PREFIX=.; make lib; make install"
+  URL: https://github.com/blitzpp/blitz/archive/refs/tags/1.0.2.tar.gz
+  Options: BLITZ_OPT, BLITZ_LIB
+  Versions: BLITZ = 1.0.2
+
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
@@ -814,6 +844,12 @@ The specific libraries and their options are:
  URL: https://github.com/LLNL/parelag
  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.

+- Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
+  URL: https://github.com/EnzymeAD/Enzyme
+  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
+  Versions: Enzyme >= v0.0.33.
+
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -952,6 +988,7 @@ MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
 MFEM_USE_PARELAG
+MFEM_USE_ENZYME

 The following options are CMake specific:

@@ -1011,6 +1048,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - FMS
 - BENCHMARK
 - ParELAG
+ - Enzyme

 The following built-in CMake packages are also used:

@@ -7,21 +7,24 @@

                               https://mfem.org

-MFEM is a modular parallel C++ library for finite element methods. Its goal is
-to enable high-performance scalable finite element discretization research and
-application development on a wide variety of platforms, ranging from laptops to
-supercomputers.
+[MFEM](https://mfem.org) is a modular parallel C++ library for finite element
+methods. Its goal is to enable high-performance scalable finite element
+discretization research and application development on a wide variety of
+platforms, ranging from laptops to supercomputers.

 We welcome contributions and feedback from the community. Please see the file
-CONTRIBUTING.md for additional details about our development process.
+[CONTRIBUTING.md](CONTRIBUTING.md) for additional details about our development
+process.

-* For building instructions, see the file INSTALL, or type "make help".
+* For building instructions, see the file [INSTALL](INSTALL), or type "make help".

-* Copyright and licensing information can be found in files LICENSE and NOTICE.
+* Copyright and licensing information can be found in files [LICENSE](LICENSE) and [NOTICE](NOTICE).

 * The best starting point for new users interested in MFEM's features is to
  review the examples and miniapps at https://mfem.org/examples.

+* Instructions for learning with Docker are in [config/docker](config/docker).
+
 Conceptually, MFEM can be viewed as a finite element toolbox that provides the
 building blocks for developing finite element algorithms in a manner similar to
 that of MATLAB for linear algebra methods. In particular, MFEM provides support
@@ -58,12 +61,16 @@ solvers from the hypre library. Comprehensive support for other external
 packages, e.g. PETSc, SUNDIALS and libCEED is also included, giving access to
 additional linear and nonlinear solvers, preconditioners, time integrators, etc.

-For examples of using MFEM, see the examples/ and miniapps/ directories, as well
-as the OpenGL visualization tool GLVis which is available at https://glvis.org.
+For examples of using MFEM, see the [examples/](examples) and [miniapps/](miniapps)
+directories, as well as the OpenGL visualization tool GLVis which is available
+at https://glvis.org.
+
+## License

 MFEM is distributed under the terms of the BSD-3 license. All new contributions
-must be made under this license. See LICENSE and NOTICE for details.
+must be made under this license. See [LICENSE](LICENSE) and [NOTICE](NOTICE) for
+details.

-SPDX-License-Identifier: BSD-3-Clause
-LLNL Release Number: LLNL-CODE-806117
+SPDX-License-Identifier: BSD-3-Clause <br>
+LLNL Release Number: LLNL-CODE-806117 <br>
 DOI: 10.11578/dc.20171025.1248
@@ -58,8 +58,10 @@ set(MFEM_USE_MOONOLITH @MFEM_USE_MOONOLITH@)
 set(MFEM_USE_CODIPACK @MFEM_USE_CODIPACK@)
 set(MFEM_USE_ADFORWARD @MFEM_USE_ADFORWARD@)
 set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
+set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
+set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -160,6 +160,9 @@
 // Enable MFEM functionality based on the Caliper library
 #cmakedefine MFEM_USE_CALIPER

+// Enable MFEM functionality based on the Algoim library
+#cmakedefine MFEM_USE_ALGOIM
+
 // Which library functions to use in class StopWatch for measuring time.
 // For a list of the available options, see INSTALL.
 // If not defined, an option is selected automatically.
@@ -187,4 +190,7 @@
 // Enable MFEM functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK

+// Enable Enzyme for AD
+#cmakedefine MFEM_USE_ENZYME
+
 #endif // MFEM_CONFIG_HEADER
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - ALGOIM_FOUND
+#   - ALGOIM_LIBRARIES
+#   - ALGOIM_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(Algoim ALGOIM ALGOIM_DIR
+       	"include" "algoim_quad.hpp"
+       	"" ""
+	"Paths to headers required by Algoim."
+	"Libraries required by Algoim.")
@@ -0,0 +1,22 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - BLITZ_FOUND
+#   - BLITZ_LIBRARIES
+#   - BLITZ_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(Blitz BLITZ BLITZ_DIR
+       	"include" "blitz/blitz.h"
+       	"lib" "blitz"
+  	"Paths to headers required by Blitz."
+       	"Libraries required by Blitz.")
@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+message(STATUS "Looking for ENZYME ...")
+message(STATUS "   in ENZYME_DIR = ${ENZYME_DIR}")
+
+# Make sure the directory and version combination works. Do nothing otherwise.
+if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
+  message(STATUS "Found ENZYME: ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
+
+  # Set ENZYME_FOUND
+  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
+
+  # Set CXX flags to accomodate the Enzyme Clang plugin
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
+  set(MFEM_USE_ENZYME YES)
+else()
+
+endif()
@@ -15,5 +15,5 @@
 #   - GSLIB_INCLUDE_DIRS

 include(MfemCmakeUtilities)
-mfem_find_package(gslib GSLIB GSLIB_DIR "include" gslib.h "lib" gs
+mfem_find_package(GSLIB GSLIB GSLIB_DIR "include" gslib.h "lib" gs
  "Paths to headers required by GSLIB." "Libraries required by GSLIB.")
@@ -894,7 +894,7 @@ function(mfem_export_mk_files)
      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH)
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -167,6 +167,9 @@
 // Enable functionality based on the Caliper library.
 // #define MFEM_USE_CALIPER

+// Enable functionality based on the Algoim library.
+// #define MFEM_USE_ALGOIM
+
 // Enable functionality based on the Umpire library.
 // #define MFEM_USE_UMPIRE

@@ -192,4 +195,7 @@
 // Enable functionality based on the Google Benchmark library.
 // #define MFEM_USE_BENCHMARK

+// Enable the Enzyme LLVM plugin
+// #define MFEM_USE_ENZYME
+
 #endif // MFEM_CONFIG_HEADER
@@ -63,6 +63,7 @@ MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
+MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -58,11 +58,13 @@ option(MFEM_USE_UMPIRE "Enable Umpire" OFF)
 option(MFEM_USE_SIMD "Enable use of SIMD intrinsics" OFF)
 option(MFEM_USE_ADIOS2 "Enable ADIOS2" OFF)
 option(MFEM_USE_CALIPER "Enable Caliper support" OFF)
+option(MFEM_USE_ALGOIM "Enable Algoim support" OFF)
 option(MFEM_USE_MKL_CPARDISO "Enable MKL CPardiso" OFF)
 option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
+option(MFEM_USE_ENZYME "Enable Enzyme" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -236,6 +238,11 @@ set(RAJA_DIR "${MFEM_DIR}/../raja" CACHE PATH "Path to RAJA")
 set(CEED_DIR "${MFEM_DIR}/../libCEED" CACHE PATH "Path to libCEED")
 set(UMPIRE_DIR "${MFEM_DIR}/../umpire" CACHE PATH "Path to Umpire")
 set(CALIPER_DIR "${MFEM_DIR}/../caliper" CACHE PATH "Path to Caliper")
+set(BLITZ_DIR "${MFEM_DIR}/../blitz" CACHE PATH "Path to Blitz")
+set(ALGOIM_DIR "${MFEM_DIR}/../algoim" CACHE PATH "Path to Algoim")
+set(ALGOIM_REQUIRED_PACKAGES "BLITZ" CACHE STRING
+    "Packages that ALGOIM depends on.")
+
 set(BENCHMARK_DIR "${MFEM_DIR}/../google-benchmark" CACHE PATH
    "Path to Google Benchmark")

@@ -42,6 +42,9 @@ STATIC = YES
 SHARED = NO

 # CUDA configuration options
+#
+# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with
+# clang as its host compiler.
 CUDA_CXX = nvcc
 CUDA_ARCH = sm_60
 CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
@@ -153,6 +156,7 @@ MFEM_USE_RAJA          = NO
 MFEM_USE_OCCA          = NO
 MFEM_USE_CEED          = NO
 MFEM_USE_CALIPER       = NO
+MFEM_USE_ALGOIM        = NO
 MFEM_USE_UMPIRE        = NO
 MFEM_USE_SIMD          = NO
 MFEM_USE_ADIOS2        = NO
@@ -162,6 +166,7 @@ MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
+MFEM_USE_ENZYME        = NO

 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -202,7 +207,7 @@ HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 ifeq (YES,$(MFEM_USE_CUDA))
   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
+   HYPRE_LIB += -lcusparse -lcurand -lcublas
 endif
 ifeq (YES,$(MFEM_USE_HIP))
   # This is only necessary when hypre is built with hip:
@@ -469,6 +474,16 @@ CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
 CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper

+# BLITZ library configuration
+BLITZ_DIR = @MFEM_DIR@/../blitz
+BLITZ_OPT = -I$(BLITZ_DIR)/include
+BLITZ_LIB = $(XLINKER)-rpath,$(BLITZ_DIR)/lib -L$(BLITZ_DIR)/lib -lblitz
+
+# ALGOIM library configuration
+ALGOIM_DIR = @MFEM_DIR@/../algoim
+ALGOIM_OPT = -I$(ALGOIM_DIR)/src $(BLITZ_OPT)
+ALGOIM_LIB = $(BLITZ_LIB)
+
 # BENCHMARK library configuration
 BENCHMARK_DIR = @MFEM_DIR@/../google-benchmark
 BENCHMARK_OPT = -I$(BENCHMARK_DIR)/include
@@ -509,6 +524,22 @@ PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
 PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG

+# Enzyme configuration
+
+# If you want to enable automatic differentiation at compile time, use the
+# options below, adapted to your configuration. To be more flexible, we
+# recommend using the Enzyme plugin during link time optimization. One option is
+# to add your options to the global compiler/linker flags like
+#
+# BASE_FLAGS += -flto
+# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\
+#                -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl,
+#
+ENZYME_DIR ?= @MFEM_DIR@/../enzyme
+ENZYME_VERSION ?= 14
+ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so
+ENZYME_LIB = ""
+
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -0,0 +1,27 @@
+FROM ghcr.io/mfem/mfem-ubuntu-base:latest as builder
+
+# docker build -t ghcr.io/mfem/mfem-ubuntu .
+
+COPY ./config/docker/spack.yaml /opt/mfem-env/spack.yaml
+RUN apt-get install -y python3 && \
+    cd /opt/mfem-env && \
+    . /opt/spack/share/spack/setup-env.sh && \
+    spack env activate . && \
+    spack env view regenerate
+
+FROM ubuntu:22.04
+
+COPY --from=builder /opt/view /opt/view
+COPY --from=builder /opt/mfem-view /opt/mfem-view
+
+RUN apt-get update && \
+    apt-get install -y unzip gfortran && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev
+
+ENV PATH=$PATH:/opt/mfem-view/bin
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/mfem-view/lib:/opt/mfem-view/lib64
+ENV DEBIAN_FRONTEND=noninteractive
+
+# The user will see the view on shell into the container
+WORKDIR /opt/mfem-view
+ENTRYPOINT ["/bin/bash"]
@@ -0,0 +1,47 @@
+FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
+
+# docker build -f Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
+
+RUN apt-get update && \
+    apt-get install -y unzip gfortran && \
+    spack compiler find && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev
+
+# /code is the working directory for code
+WORKDIR /code
+COPY . /code
+
+# This is for a spack environment/view to install from there
+RUN mkdir -p /opt/mfem-env \
+&&  (echo "spack:" \
+&&   echo "  view:" \
+&&   echo "    mfem:" \
+&&   echo "      root: /opt/mfem-view" \
+&&   echo "      link_type: copy" \
+&&   echo "  packages:" \
+&&   echo "    all:" \
+&&   echo "      target:" \
+&&   echo "      - x86_64_v3" \
+&&   echo "  config:" \
+&&   echo "    concretizer: clingo" \
+&&   echo "    compiler:" \
+&&   echo "      target:" \
+&&   echo "      - x86_64_v3" \
+&&   echo "    install_missing_compilers: true" \
+&&   echo "  concretization: together") > /opt/mfem-env/spack.yaml
+
+RUN cd /opt/mfem-env && \
+    . /opt/spack/share/spack/setup-env.sh && \
+    spack env activate . && \
+    spack develop --path /code mfem@master+examples+miniapps && \
+    spack add mfem@master+examples+miniapps # && \
+#    spack install
+
+# ensure mfem always on various paths
+#RUN cd /opt/mfem-env && \
+#    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+# Present the software install when we shell in
+# The view is at /opt/mfem-env/.spack-env/view
+#WORKDIR /opt/software
+#ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
@@ -0,0 +1,147 @@
+# mfem Docker
+
+We provide a [Dockerfile.base](Dockerfile.base) to build an ubuntu base image,
+and a [Dockerfile](Dockerfile) to build a smaller one with a multi-stage build.
+You can use this image for a demo of using mfem! 🎉️
+
+Updated containers are built and deployed on merges to the main branch and releases.
+If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
+
+### Usage
+
+Here is how to build the container. Note that we build so it belongs to the same
+namespace as the repository here. "ghcr.io" means "GitHub Container Registry" and
+is the [GitHub packages](https://github.com/features/packages) registry that supports
+ Docker images and other OCI artifacts. From the root of the repository:
+
+```bash
+$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
+$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
+```
+
+### Shell Ubuntu
+
+To shell into the container:
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu
+```
+
+This smaller image has a view where everything is installed.
+
+```bash
+$ ls
+bin  etc  include  lib  libexec  sbin  share  var
+```
+
+ - Examples are in share/mfem/examples
+ - Examples are in share/mfem/miniapps
+
+You can read more about interaction with these examples and miniapps below.
+
+### Shell Ubuntu Base
+
+To shell into the container:
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
+```
+
+Off the bat, you can see mfem libraries are in your path so you can jump into development:
+
+```bash
+env | grep mfem
+```
+```bash
+PKG_CONFIG_PATH=/opt/mfem-env/.spack-env/view/lib/pkgconfig:/opt/mfem-env/.spack-env/view/share/pkgconfig:/opt/mfem-env/.spack-env/view/lib64/pkgconfig
+PWD=/opt/mfem-env
+MANPATH=/opt/mfem-env/.spack-env/view/share/man:/opt/mfem-env/.spack-env/view/man:
+CMAKE_PREFIX_PATH=/opt/mfem-env/.spack-env/view
+SPACK_ENV=/opt/mfem-env
+ACLOCAL_PATH=/opt/mfem-env/.spack-env/view/share/aclocal
+LD_LIBRARY_PATH=/opt/mfem-env/.spack-env/view/lib:/opt/mfem-env/.spack-env/view/lib64
+PATH=/opt/mfem-env/.spack-env/view/bin:/opt/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+```
+
+#### Examples and MiniApps
+
+If you want to develop a tool that _uses_ mfem, you can find the built libraries in:
+
+```
+$ ls /opt/mfem-env/.spack-env/view/
+bin  etc  include  lib  libexec  sbin  share  var
+```
+
+And yes, this is the working directory when you shell into the container!
+You can find the examples here:
+
+
+```bash
+cd share/mfem/examples
+```
+```bash
+$ ./ex0
+Options used:
+   --mesh ../data/star.mesh
+   --order 1
+Number of unknowns: 101
+   Iteration :   0  (B r, r) = 0.184259
+   Iteration :   1  (B r, r) = 0.102754
+   Iteration :   2  (B r, r) = 0.00558141
+   Iteration :   3  (B r, r) = 1.5247e-05
+   Iteration :   4  (B r, r) = 1.13807e-07
+   Iteration :   5  (B r, r) = 6.27231e-09
+   Iteration :   6  (B r, r) = 3.76268e-11
+   Iteration :   7  (B r, r) = 6.07423e-13
+   Iteration :   8  (B r, r) = 4.10615e-15
+Average reduction factor = 0.140201
+```
+
+Try running a few, and look at the associated .cpp file for the source code!
+You can also explore the "mini apps," also in share/mfem, but under miniapps.
+
+```bash
+# This is run from the examples directory
+$ cd ../miniapps
+```
+```bash
+$ ls
+CMakeLists.txt  common            meshing  nurbs        shifted  toys
+adjoint         electromagnetics  mtop     parelag      solvers
+autodiff        gslib             navier   performance  tools
+```
+
+And an example in "toys"
+
+```bash
+cd toys
+```
+```bash
+$ ./automata -no-vis
+Options used:
+   --num-steps 16
+   --rule 90
+   --no-visualization
+
+Rule:
+ 111 110 101 100 011 010 001 000
+  0   1   0   1   1   0   1   0
+
+Applying rule...done.
+```
+
+Have fun!
+
+
+#### Your own App
+If you want to develop with your own code base
+(and mfem as is in the container) you can bind to somewhere else in the container (e.g., src)
+
+```bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
+```
+
+In the above, we can pretend your project is in the present working directory (PWD) and we are
+binding to source. You can then use the mfem in the container for development, and if you
+want to distribute your library or app in a container, you can use the mfem container as the base.
+
@@ -0,0 +1,11 @@
+spack:
+  specs: [mfem@master+examples+miniapps]
+  view: 
+    mfem:
+      root: /opt/mfem-view 
+      link_type: copy
+  concretization: together
+  develop:
+    mfem:
+      path: /code
+      spec: mfem@master+examples+miniapps
@@ -0,0 +1,8 @@
+--- a/CMakeLists.txt	Wed Dec 21 18:24:22 2016
+++ b/CMakeLists.txt	Wed Dec 21 18:24:26 2016
+@@ -20,4 +20,4 @@
+ # Recursively look for CMakeLists.txt in subdirs.
+ add_subdirectory("include")
+ add_subdirectory("libmetis")
+-add_subdirectory("programs")
+# add_subdirectory("programs")
@@ -0,0 +1,15 @@
+--- a/CMakeLists.txt	Sat Mar 30 17:24:45 2013
+++ b/CMakeLists.txt	Wed Dec 21 18:23:43 2016
+@@ -4,11 +4,7 @@
+ set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+ 
+-if(MSVC)
+-  set(METIS_INSTALL FALSE)
+-else()
+-  set(METIS_INSTALL TRUE)
+-endif()
+set(METIS_INSTALL TRUE)
+ 
+ # Configure libmetis library.
+ if(SHARED)
@@ -0,0 +1,34 @@
+diff --git a/include/metis.h b/include/metis.h
+index dc5406a..7732437 100644
+--- a/include/metis.h
+++ b/include/metis.h
+@@ -72,10 +72,14 @@ typedef __int64 int64_t;
+ #define PRId64       "I64d"
+ #define SCNd32       "ld"
+ #define SCNd64       "I64d"
+#ifdef _WIN32
+#include <stdint.h>
+#else
+ #define INT32_MIN    ((int32_t)_I32_MIN)
+ #define INT32_MAX    _I32_MAX
+ #define INT64_MIN    ((int64_t)_I64_MIN)
+ #define INT64_MAX    _I64_MAX
+#endif
+ #else
+ #include <inttypes.h>
+ #endif
+diff --git a/GKlib/gk_arch.h b/GKlib/gk_arch.h
+index 78b1431..7258763 100644
+--- a/GKlib/gk_arch.h
+++ b/GKlib/gk_arch.h
+@@ -32,8 +32,8 @@
+ 
+ 
+ #ifdef __MSC__ 
+-  #include "ms_stdint.h"
+-  #include "ms_inttypes.h"
+  #include <stdint.h>
+  #include <inttypes.h>
+   #include "ms_stat.h"
+ #else
+ #ifndef SUNOS
@@ -0,0 +1,11 @@
+--- a/GKlib/gk_arch.h	Wed Dec 21 18:34:18 2016
+++ b/GKlib/gk_arch.h	Wed Dec 21 18:30:49 2016
+@@ -58,7 +58,7 @@
+ #define PTRDIFF_MAX  INT64_MAX
+ #endif
+ 
+-#ifdef __MSC__
+#if defined(__MSC__) && (_MSC_VER < 1900)
+ /* MSC does not have rint() function */
+ #define rint(x) ((int)((x)+0.5))  
+ 
@@ -0,0 +1,14 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index e94f050..b9613a7 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -1,7 +1,8 @@
+ cmake_minimum_required(VERSION 2.8)
+ project(METIS)
+ 
+-set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
+set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
+
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+ 
+ set(METIS_INSTALL TRUE)
@@ -0,0 +1,11 @@
+--- a/libmetis/metislib.h	Sat Mar 30 17:24:45 2013
+++ b/libmetis/metislib.h	Wed Dec 21 18:30:59 2016
+@@ -31,7 +31,7 @@
+ #include <proto.h>
+ 
+ 
+-#if defined(COMPILER_MSC)
+#if defined(COMPILER_MSC) && (_MSC_VER < 1900)
+ #if defined(rint)
+   #undef rint
+ #endif
@@ -0,0 +1,10 @@
+--- a/libmetis/CMakeLists.txt	Sat Mar 30 17:24:45 2013
+++ b/libmetis/CMakeLists.txt	Wed Dec 21 17:41:37 2016
+@@ -11,6 +11,6 @@
+ if(METIS_INSTALL)
+   install(TARGETS metis
+     LIBRARY DESTINATION lib
+-    RUNTIME DESTINATION lib
+    RUNTIME DESTINATION bin
+     ARCHIVE DESTINATION lib)
+ endif()
@@ -0,0 +1,44 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index b9613a7..e43ffee 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -22,3 +22,23 @@ include_directories(include)
+ add_subdirectory("include")
+ add_subdirectory("libmetis")
+ # add_subdirectory("programs")
+
+if(METIS_INSTALL)
+  set(PRJ_NAME metis)
+  set(PRJ_VER 5.1.0)
+  install(EXPORT metisTargets 
+    FILE ${PRJ_NAME}Targets.cmake
+    DESTINATION lib/cmake/${PRJ_NAME})
+  include(CMakePackageConfigHelpers)
+  write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
+    VERSION ${PRJ_VER}
+    COMPATIBILITY SameMajorVersion)
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
+    "include(\${CMAKE_CURRENT_LIST_DIR}/${PRJ_NAME}Targets.cmake)")
+  install(FILES 
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
+    DESTINATION lib/cmake/${PRJ_NAME})
+endif()
+
+diff --git a/libmetis/CMakeLists.txt b/libmetis/CMakeLists.txt
+index 7a5fc74..5a68cf0 100644
+--- a/libmetis/CMakeLists.txt
+++ b/libmetis/CMakeLists.txt
+@@ -9,8 +9,9 @@ if(UNIX)
+ endif()
+ 
+ if(METIS_INSTALL)
+-  install(TARGETS metis
+  install(TARGETS metis EXPORT metisTargets
+     LIBRARY DESTINATION lib
+     RUNTIME DESTINATION bin
+-    ARCHIVE DESTINATION lib)
+    ARCHIVE DESTINATION lib
+    INCLUDES DESTINATION include)
+ endif()
@@ -0,0 +1,41 @@
+vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
+set(OPTIONS -DSHARED=OFF)
+
+set(METIS_VERSION 5.1.0)
+
+vcpkg_download_distfile(ARCHIVE
+    URLS "https://github.com/mfem/tpls/raw/gh-pages/metis-${METIS_VERSION}.tar.gz"
+    FILENAME "metis-${METIS_VERSION}.tar.gz"
+    SHA512 deea47749d13bd06fbeaf98a53c6c0b61603ddc17a43dae81d72c8015576f6495fd83c11b0ef68d024879ed5415c14ebdbd87ce49c181bdac680573bea8bdb25
+)
+
+vcpkg_extract_source_archive_ex(
+    OUT_SOURCE_PATH SOURCE_PATH
+    ARCHIVE ${ARCHIVE}
+    REF ${METIS_VERSION}
+    PATCHES
+        enable-install.patch
+        disable-programs.patch
+        fix-runtime-install-destination.patch
+        fix-metis-vs14-math.patch
+        fix-gklib-vs14-math.patch
+        fix-linux-build-error.patch
+        install-metisConfig.patch
+        fix-INT_MIN_define.patch
+)
+
+vcpkg_configure_cmake(
+    SOURCE_PATH ${SOURCE_PATH}
+    PREFER_NINJA
+    OPTIONS ${OPTIONS}
+)
+
+vcpkg_install_cmake()
+vcpkg_copy_pdbs()
+vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/metis)
+
+file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
+
+# Handle copyright
+file(COPY ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/metis)
+file(INSTALL ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
@@ -0,0 +1,7 @@
+{
+  "name": "metis-mfem",
+  "version-string": "5.1.0",
+  "port-version": 0,
+  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
+  "homepage": "https://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
+}
@@ -1,324 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-2
-
-elements
-15
-1 3 3 6 17 9
-1 3 17 7 4 8
-1 3 9 17 8 5
-1 3 0 10 18 11
-1 3 11 18 6 3
-1 3 12 1 13 19
-1 3 19 13 4 7
-1 3 2 14 20 15
-1 3 14 5 8 20
-1 3 20 8 4 13
-1 3 15 20 13 1
-1 3 0 11 21 16
-1 3 11 3 9 21
-1 3 21 9 5 14
-1 3 16 21 14 2
-
-boundary
-12
-1 1 6 17
-2 1 17 7
-3 1 0 10
-4 1 10 18
-5 1 18 6
-6 1 19 12
-7 1 12 1
-8 1 7 19
-9 1 15 2
-10 1 1 15
-11 1 16 0
-12 1 2 16
-
-vertices
-22
-
-nodes
-FiniteElementSpace
-FiniteElementCollection: H1_2D_P4
-VDim: 2
-Ordering: 1
-
-0.707106781186545 -0.707106781186545
-0.707106781186545 0.707106781186545
-0.707106781186545 0.707106781186545
-0.3535533905932725 -0.3535533905932725
-0.3535533905932725 0.3535533905932725
-0.3535533905932725 0.3535533905932725
-0 -0.3535533905932725
-0.3535533905932725 0
-0 0.3535533905932725
-0.3535533905932725 0
-0 -1
-0.5303300858899087 -0.5303300858899087
-1 0
-0.5303300858899087 0.5303300858899087
-0.5303300858899087 0.5303300858899087
-0 1
-1 0
-0 0
-0 -0.6721895948517682
-0.6721895948517682 0
-2.45326946669338e-17 0.6721895948517682
-0.6721895948517682 -2.45326946669338e-17
-0.2925042077682053 -0.3535533905932725
-0.1767766952966363 -0.3535533905932725
-0.06104918282506723 -0.3535533905932725
-0 -0.2925042077682053
-0 -0.1767766952966363
-0 -0.06104918282506723
-0.2925042077682053 -1.937718899096505e-18
-0.1767766952966363 -2.168768018011117e-18
-0.06104918282506723 -6.942024631559806e-19
-0.3535533905932725 -0.2925042077682053
-0.3535533905932725 -0.1767766952966363
-0.3535533905932725 -0.06104918282506723
-0.2925042077682053 -2.25773586854044e-19
-0.1767766952966363 2.904835825740127e-19
-0.06104918282506723 4.044254824131427e-19
-0.3535533905932725 0.2925042077682053
-0.3535533905932725 0.1767766952966363
-0.3535533905932725 0.06104918282506723
-0.2925042077682053 0.3535533905932725
-0.1767766952966363 0.3535533905932725
-0.06104918282506723 0.3535533905932725
-0 0.2925042077682053
-0 0.1767766952966363
-0 0.06104918282506723
-0.2925042077682053 0.3535533905932725
-0.1767766952966363 0.3535533905932725
-0.06104918282506723 0.3535533905932725
-0.3535533905932725 0.2925042077682053
-0.3535533905932725 0.1767766952966363
-0.3535533905932725 0.06104918282506723
-0.6145094428537778 -0.789413761659266
-0.3959323428473652 -0.9179997269052453
-0.1410304266231549 -0.989913000881131
-0 -0.9426395767133444
-0 -0.8348063668130653
-0 -0.7280766470976995
-0.4493360908158245 -0.5727497127416228
-0.2803631562583237 -0.6346920242481003
-0.09836326743220743 -0.6676463098413217
-0.6765821897740132 -0.6765821897740132
-0.6187184335382269 -0.6187184335382269
-0.5608546773024407 -0.5608546773024407
-0 -0.4080066739179119
-0 -0.51185560644339
-0 -0.6165744197248449
-0.3840779820058043 -0.3840779820058043
-0.4419417382415907 -0.4419417382415907
-0.4998054944773769 -0.4998054944773769
-0.789413761659266 0.6145094428537778
-0.9179997269052453 0.3959323428473652
-0.989913000881131 0.1410304266231549
-0.6765821897740132 0.6765821897740132
-0.6187184335382269 0.6187184335382269
-0.5608546773024407 0.5608546773024407
-0.5727497127416228 0.4493360908158245
-0.6346920242481003 0.2803631562583237
-0.6676463098413217 0.09836326743220743
-0.9426395767133444 0
-0.8348063668130653 0
-0.7280766470976995 0
-0.3840779820058043 0.3840779820058043
-0.4419417382415907 0.4419417382415907
-0.4998054944773769 0.4998054944773769
-0.4080066739179119 0
-0.51185560644339 0
-0.6165744197248449 0
-0.6765821897740132 0.6765821897740132
-0.6187184335382269 0.6187184335382269
-0.5608546773024407 0.5608546773024407
-0.4493360908158245 0.5727497127416228
-0.2803631562583237 0.6346920242481003
-0.09836326743220743 0.6676463098413217
-1.38261702840016e-17 0.9426395767133444
-2.635181002135987e-18 0.8348063668130653
-1.884789714682629e-17 0.7280766470976995
-0.6145094428537778 0.789413761659266
-0.3959323428473652 0.9179997269052453
-0.1410304266231549 0.989913000881131
-0.3840779820058043 0.3840779820058043
-0.4419417382415907 0.4419417382415907
-0.4998054944773769 0.4998054944773769
-5.694430050849992e-18 0.4080066739179119
-9.046264100643195e-18 0.51185560644339
-2.406637988827979e-17 0.6165744197248449
-0.4493360908158245 0.5727497127416228
-0.2803631562583237 0.6346920242481003
-0.09836326743220743 0.6676463098413217
-0.6145094428537778 0.789413761659266
-0.3959323428473652 0.9179997269052453
-0.1410304266231549 0.989913000881131
-0.5727497127416228 -0.4493360908158245
-0.6346920242481003 -0.2803631562583237
-0.6676463098413217 -0.09836326743220743
-0.9426395767133444 1.38261702840016e-17
-0.8348063668130653 2.635181002135987e-18
-0.7280766470976995 -1.884789714682629e-17
-0.789413761659266 -0.6145094428537778
-0.9179997269052453 -0.3959323428473652
-0.989913000881131 -0.1410304266231549
-0.4080066739179119 5.694430050849992e-18
-0.51185560644339 -9.046264100643195e-18
-0.6165744197248449 -2.406637988827979e-17
-0.5727497127416228 0.4493360908158245
-0.6346920242481003 0.2803631562583237
-0.6676463098413217 0.09836326743220743
-0.789413761659266 0.6145094428537778
-0.9179997269052453 0.3959323428473652
-0.989913000881131 0.1410304266231549
-0.2925042077682053 -0.2925042077682053
-0.1767766952966363 -0.2925042077682053
-0.06104918282506723 -0.2925042077682053
-0.2925042077682053 -0.1767766952966363
-0.1767766952966363 -0.1767766952966363
-0.06104918282506723 -0.1767766952966363
-0.2925042077682053 -0.06104918282506723
-0.1767766952966363 -0.06104918282506723
-0.06104918282506723 -0.06104918282506723
-0.06104918282506723 0.06104918282506723
-0.1767766952966363 0.06104918282506723
-0.2925042077682053 0.06104918282506723
-0.06104918282506723 0.1767766952966363
-0.1767766952966363 0.1767766952966363
-0.2925042077682053 0.1767766952966363
-0.06104918282506723 0.2925042077682053
-0.1767766952966363 0.2925042077682053
-0.2925042077682053 0.2925042077682053
-0.2925042077682053 0.06104918282506723
-0.1767766952966363 0.06104918282506723
-0.06104918282506723 0.06104918282506723
-0.2925042077682053 0.1767766952966363
-0.1767766952966363 0.1767766952966363
-0.06104918282506723 0.1767766952966363
-0.2925042077682053 0.2925042077682053
-0.1767766952966363 0.2925042077682053
-0.06104918282506723 0.2925042077682053
-0.5853433152017873 -0.7522063629666647
-0.3750499116779216 -0.8688976983986201
-0.1332489509046759 -0.9335917585418947
-0.530810235275664 -0.681431690503692
-0.3365495705701402 -0.7760363182652825
-0.118982888320896 -0.8276309647697756
-0.4772305101537735 -0.6103572471372083
-0.2994187471166691 -0.6834400186968074
-0.1053286339663329 -0.7226539726955523
-0.4216899203778937 -0.5350670333567459
-0.2616641810583542 -0.5860082222568944
-0.09155721275052374 -0.6128808628755988
-0.3699443285846694 -0.4634369827877818
-0.2271681353001397 -0.4938868826316494
-0.07907999957616745 -0.5096954174503119
-0.3190349108655897 -0.3915624660435183
-0.1938732741916287 -0.4019674242664178
-0.06713939478447406 -0.4072844098734093
-0.9335917585418947 0.1332489509046759
-0.8688976983986201 0.3750499116779216
-0.7522063629666647 0.5853433152017873
-0.8276309647697756 0.118982888320896
-0.7760363182652825 0.3365495705701402
-0.681431690503692 0.530810235275664
-0.7226539726955523 0.1053286339663329
-0.6834400186968074 0.2994187471166691
-0.6103572471372083 0.4772305101537735
-0.6128808628755988 0.09155721275052374
-0.5860082222568944 0.2616641810583542
-0.5350670333567459 0.4216899203778937
-0.5096954174503119 0.07907999957616745
-0.4938868826316494 0.2271681353001397
-0.4634369827877818 0.3699443285846694
-0.4072844098734093 0.06713939478447406
-0.4019674242664178 0.1938732741916287
-0.3915624660435183 0.3190349108655861
-0.5853433152017873 0.7522063629666647
-0.530810235275664 0.681431690503692
-0.4772305101537735 0.6103572471372083
-0.3750499116779216 0.8688976983986201
-0.3365495705701402 0.7760363182652825
-0.2994187471166691 0.6834400186968074
-0.1332489509046759 0.9335917585418947
-0.118982888320896 0.8276309647697756
-0.1053286339663329 0.7226539726955523
-0.4216899203778937 0.5350670333567459
-0.3699443285846694 0.4634369827877818
-0.3190349108655861 0.3915624660435183
-0.2616641810583542 0.5860082222568944
-0.2271681353001397 0.4938868826316494
-0.1938732741916287 0.4019674242664178
-0.09155721275052374 0.6128808628755988
-0.07907999957616745 0.5096954174503119
-0.06713939478447406 0.4072844098734093
-0.09155721275052374 0.6128808628755988
-0.07907999957616745 0.5096954174503119
-0.06713939478447406 0.4072844098734093
-0.2616641810583542 0.5860082222568944
-0.2271681353001397 0.4938868826316494
-0.1938732741916287 0.4019674242664178
-0.4216899203778937 0.5350670333567459
-0.3699443285846694 0.4634369827877818
-0.3190349108655897 0.3915624660435183
-0.1332489509046759 0.9335917585418947
-0.118982888320896 0.8276309647697756
-0.1053286339663329 0.7226539726955523
-0.3750499116779216 0.8688976983986201
-0.3365495705701402 0.7760363182652825
-0.2994187471166691 0.6834400186968074
-0.5853433152017873 0.7522063629666647
-0.530810235275664 0.681431690503692
-0.4772305101537735 0.6103572471372083
-0.7522063629666647 -0.5853433152017873
-0.681431690503692 -0.530810235275664
-0.6103572471372083 -0.4772305101537735
-0.8688976983986201 -0.3750499116779216
-0.7760363182652825 -0.3365495705701402
-0.6834400186968074 -0.2994187471166691
-0.9335917585418947 -0.1332489509046759
-0.8276309647697756 -0.118982888320896
-0.7226539726955523 -0.1053286339663329
-0.5350670333567459 -0.4216899203778937
-0.4634369827877818 -0.3699443285846694
-0.3915624660435183 -0.3190349108655861
-0.5860082222568944 -0.2616641810583542
-0.4938868826316494 -0.2271681353001397
-0.4019674242664178 -0.1938732741916287
-0.6128808628755988 -0.09155721275052374
-0.5096954174503119 -0.07907999957616745
-0.4072844098734093 -0.06713939478447406
-0.6128808628755988 0.09155721275052374
-0.5096954174503119 0.07907999957616745
-0.4072844098734093 0.06713939478447406
-0.5860082222568944 0.2616641810583542
-0.4938868826316494 0.2271681353001397
-0.4019674242664178 0.1938732741916287
-0.5350670333567459 0.4216899203778937
-0.4634369827877818 0.3699443285846694
-0.3915624660435183 0.3190349108655897
-0.9335917585418947 0.1332489509046759
-0.8276309647697756 0.118982888320896
-0.7226539726955523 0.1053286339663329
-0.8688976983986201 0.3750499116779216
-0.7760363182652825 0.3365495705701402
-0.6834400186968074 0.2994187471166691
-0.7522063629666647 0.5853433152017873
-0.681431690503692 0.530810235275664
-0.6103572471372083 0.4772305101537735
@@ -0,0 +1,56 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geomety Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+# PYRAMID     = 7
+
+dimension
+3
+
+elements
+4
+1 5 0 1 4 3 6 7 10 9
+1 6 4 1 5 10 7 11
+1 7 11 7 1 5 8
+1 4 2 5 1 8
+
+boundary
+14
+1 3 0 3 4 1
+1 3 6 7 10 9
+1 3 0 6 9 3
+1 3 0 1 7 6
+1 3 3 9 10 4
+1 2 1 5 2
+1 2 1 4 5
+1 2 1 8 7
+1 2 1 2 8
+1 2 2 5 8
+1 2 5 11 8
+1 2 7 8 11
+1 2 7 11 10
+1 3 4 10 11 5
+
+vertices
+12
+3
+0 0 0
+1 0 0
+2 0 0
+0 1 0
+1 1 0
+2 1 0
+0 0 1
+1 0 1
+2 0 1
+0 1 1
+1 1 1
+2 1 1
@@ -2349,7 +2349,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.

-DOT_GRAPH_MAX_NODES    = 50
+DOT_GRAPH_MAX_NODES    = 100

 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the
@@ -1,367 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <algorithm>
-
-using namespace std;
-using namespace mfem;
-
-
-void Prefine(FiniteElementSpace & fes_old,
-             GridFunction &u, Coefficient &gf_ex, GridFunction &orders_gf,
-             double min_thresh, double max_thresh);
-
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x, double);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   BilinearForm &M, &K;
-   const Vector &b;
-   Solver *M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-public:
-   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
-   void Update();
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution();
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/periodic-hexagon.mesh";
-   int ref_levels = 2;
-   int order = 1;
-   double t_final = 10.0;
-   double dt = 0.0005;
-   bool visualization = true;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh mesh0 = Mesh::MakeCartesian2D(64, 1, mfem::Element::QUADRILATERAL,false, 2,
-                                      1);
-
-
-   std::vector<Vector> translations = {Vector({2.0,0.0}), };
-
-
-   Mesh mesh = Mesh::MakePeriodic(mesh0,
-                                  mesh0.CreatePeriodicVertexMapping(translations));
-
-
-   mesh.EnsureNCMesh();
-
-   int dim = mesh.Dimension();
-
-   ODESolver *ode_solver = new RK4Solver;
-
-
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
-   FiniteElementSpace fes(&mesh, &fec);
-   FiniteElementSpace fes_old(&mesh, &fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
-   constexpr double alpha = -1.0;
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
-   k.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-   k.AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-
-   LinearForm b(&fes);
-   b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha,-0.5));
-
-   m.Assemble();
-   int skip_zeros = 0;
-   k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
-   k.Finalize(skip_zeros);
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u(&fes);
-   u0.SetTime(0.);
-   u.ProjectCoefficient(u0);
-
-   L2_FECollection orders_fec(0,dim);
-   FiniteElementSpace orders_fes(&mesh,&orders_fec);
-   GridFunction orders_gf(&orders_fes);
-   for (int i = 0; i<mesh.GetNE(); i++) { orders_gf(i) = order; }
-
-   socketstream sout;
-   socketstream meshout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      meshout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
-         sout << flush;
-         meshout.precision(precision);
-         meshout << "solution\n" << mesh << orders_gf;
-         meshout << flush;
-      }
-   }
-
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m, k, b);
-
-   double t = 0.0;
-   adv.SetTime(t);
-
-   GridFunction gf_ex(&fes);
-   FunctionCoefficient u_ex(u0_function);
-
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-         u_ex.SetTime(t);
-         Prefine(fes_old,u,u_ex, orders_gf, 5e-5, 5e-4);
-         m.Update();
-         m.Assemble();
-         m.Finalize();
-         k.Update();
-         k.Assemble(skip_zeros);
-         k.Finalize(skip_zeros);
-         b.Update();
-         b.Assemble();
-         adv.Update();
-         ode_solver->Init(adv);
-         if (visualization)
-         {
-            GridFunction * pr_u = ProlongToMaxOrder(&u);
-            sout << "solution\n" << mesh << *pr_u << flush;
-            meshout << "solution\n" << mesh << orders_gf << flush;
-         }
-      }
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
-{
-   Array<int> ess_tdof_list;
-   M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   M_solver.SetOperator(M);
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Update()
-{
-   height = M.Height();
-   width = M.Width();
-   z.SetSize(M.Height());
-
-   Array<int> ess_tdof_list;
-   delete M_prec;
-   M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   M_solver.SetOperator(M);
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-}
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   v.SetSize(2);
-   v(0) = 1.;
-   v(1) = 0.;
-}
-
-// Initial condition
-double u0_function(const Vector &x, double t)
-{
-   // give x0, y0;
-   double x0 = 0.5;
-   // double y0 = 0.5;
-   double w = 100.;
-   double c = 1.;
-   double ds = c*t;
-
-   double xx = x(0) - ds;
-   double yy = x(1) - ds;
-
-   double tol = 1e-6;
-   if (xx>= 2.0+tol || xx<= 0.0-tol)
-   {
-      xx -= (int)xx;
-   }
-   if (yy>= 1.0+tol || yy<= 0.0-tol)
-   {
-      yy -= (int)yy;
-   }
-
-   double dr2 = (xx-x0)*(xx-x0);
-   return 1. + exp(-w*dr2);
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   return 1.0;
-}
-
-
-void Prefine(FiniteElementSpace & fes_old,
-             GridFunction &u, Coefficient &ex, GridFunction &orders_gf,
-             double min_thresh, double max_thresh)
-{
-   // get element errors
-   FiniteElementSpace * fes = u.FESpace();
-   int ne = fes->GetMesh()->GetNE();
-   Vector errors(ne);
-   u.ComputeElementL2Errors(ex,errors);
-   for (int i = 0; i<ne; i++)
-   {
-      double error = errors(i);
-      int order = fes->GetElementOrder(i);
-      if (error < min_thresh && order > 1)
-      {
-         fes->SetElementOrder(i,order-1);
-      }
-      else if (error > max_thresh && order < 2)
-      {
-         fes->SetElementOrder(i, order+1);
-      }
-      else
-      {
-         // do nothing
-      }
-   }
-
-   fes->Update(false);
-
-   PRefinementTransferOperator * T = new PRefinementTransferOperator(fes_old,*fes);
-
-   GridFunction u_fine(fes);
-   T->Mult(u,u_fine);
-
-   // copy the orders to the old space
-   for (int i = 0; i<ne; i++)
-   {
-      int order = fes->GetElementOrder(i);
-      fes_old.SetElementOrder(i,order);
-      orders_gf(i) = order;
-   }
-   fes_old.Update(false);
-
-   delete T;
-
-   // update old gridfuntion;
-   u = u_fine;
-
-}
@@ -1,706 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <algorithm>
-
-using namespace std;
-using namespace mfem;
-
-double sx, sy;
-
-Vector vel;
-
-void Prefine(FiniteElementSpace & fes_old,
-             GridFunction &u, Coefficient &gf_ex, GridFunction &orders_gf,
-             double min_thresh, double max_thresh);
-
-void Hrefine(GridFunction &u, Coefficient &gf_ex, double min_thresh,
-             double max_thresh);
-
-void Hrefine2(GridFunction &u, Coefficient &gf_ex, double min_thresh,
-              double max_thresh);
-
-Table * Refine(Array<int> ref_actions, GridFunction &u, int depth_limit = 100);
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x, double);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-// Mesh bounding box
-Vector bb_min, bb_max;
-
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   BilinearForm &M, &K;
-   const Vector &b;
-   Solver *M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-public:
-   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
-   void Update();
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution();
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/periodic-hexagon.mesh";
-   int ref_levels = 2;
-   int order = 1;
-   sx = 1.0;
-   sy = 1.0;
-   double t_final = 1.0;
-   double dt = 0.002;
-   bool visualization = true;
-   int vis_steps = 5;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&sx, "-sx", "--sx",
-                  "mesh length in x direction");
-   args.AddOption(&sy, "-sy", "--sy",
-                  "mesh length in y direction");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   Mesh mesh0 = Mesh::MakeCartesian2D(16, 16, mfem::Element::QUADRILATERAL,false,
-                                      sx,
-                                      sy);
-
-
-   std::vector<Vector> translations = {Vector({sx,0.0}), Vector({0.0,sy})};
-
-
-   Mesh mesh = Mesh::MakePeriodic(mesh0,
-                                  mesh0.CreatePeriodicVertexMapping(translations));
-
-
-   mesh.EnsureNCMesh();
-
-   int dim = mesh.Dimension();
-
-   ODESolver *ode_solver = new RK4Solver;
-
-
-   mesh.GetBoundingBox(bb_min, bb_max, max(order, 1));
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
-   FiniteElementSpace fes(&mesh, &fec);
-   FiniteElementSpace fes_old(&mesh, &fec);
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
-   constexpr double alpha = -1.0;
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
-   k.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-   k.AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-
-   LinearForm b(&fes);
-   b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha,-0.5));
-
-   m.Assemble();
-   int skip_zeros = 0;
-   k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
-   k.Finalize(skip_zeros);
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u(&fes);
-   u0.SetTime(0.);
-   u.ProjectCoefficient(u0);
-
-
-   L2_FECollection orders_fec(0,dim);
-   FiniteElementSpace orders_fes(&mesh,&orders_fec);
-   GridFunction orders_gf(&orders_fes);
-   for (int i = 0; i<mesh.GetNE(); i++) { orders_gf(i) = order; }
-
-   socketstream sout;
-   // socketstream meshout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      // meshout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
-         sout << flush;
-         cin.get();
-         // meshout.precision(precision);
-         // meshout << "solution\n" << mesh << orders_gf;
-         // meshout << "mesh\n" << mesh ;
-         // meshout << flush;
-         // cin.get();
-      }
-   }
-
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m, k, b);
-
-   double t = 0.0;
-   adv.SetTime(t);
-
-   FunctionCoefficient u_ex(u0_function);
-
-   ode_solver->Init(adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-
-      ode_solver->Step(u, t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-         u_ex.SetTime(t);
-         // Prefine(fes_old,u,u_ex, orders_gf, 5e-5, 5e-4);
-
-         mfem::out << "Global L2 Error = " << u.ComputeL2Error(u_ex) << std::endl;
-         Hrefine2(u,u_ex, 5e-5, 5e-4);
-         // mfem::out << "number of elements = " << mesh.GetNE() << endl;
-
-
-         m.Update();
-         m.Assemble();
-         m.Finalize();
-         k.Update();
-         k.Assemble(skip_zeros);
-         k.Finalize(skip_zeros);
-         b.Update();
-         b.Assemble();
-         adv.Update();
-         ode_solver->Init(adv);
-         if (visualization)
-         {
-            // GridFunction gf_ex(&fes);
-            // gf_ex.ProjectCoefficient(u_ex);
-            GridFunction * pr_u = ProlongToMaxOrder(&u);
-            sout << "solution\n" << mesh << *pr_u << flush;
-
-            // meshout << "solution\n" << mesh << orders_gf << flush;
-            // meshout << "mesh\n" << mesh << flush;
-         }
-      }
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
-{
-   Array<int> ess_tdof_list;
-   M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.SetOperator(M);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Update()
-{
-   height = M.Height();
-   width = M.Width();
-   z.SetSize(M.Height());
-
-   Array<int> ess_tdof_list;
-   delete M_prec;
-   M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.SetOperator(M);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-}
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   v.SetSize(2);
-   v(0) = 1.;
-   v(1) = 1.;
-}
-
-// Initial condition
-double u0_function(const Vector &x, double t)
-{
-   // give x0, y0;
-
-   // Rotation matrix
-   double theta = M_PI/4;
-   //
-
-   double x0 = 0.5;
-   double y0 = 0.5;
-   double w = 100.;
-   double c = 1.;
-   double ds = c*t;
-   Vector a(2);
-   a(0) = cos(theta);
-   a(1) = sin(theta);
-   // double xx = x(0) - a(0)*ds;
-   // double yy = x(1) - a(1)*ds;
-   double xx = x(0) - ds;
-   double yy = x(1) - ds;
-
-   double tol = 1e-6;
-   if (xx>= sx+tol || xx<= 0.0-tol)
-   {
-      xx -= floor(xx/sx) * sx;
-   }
-   if (yy>= sy+tol || yy<= 0.0-tol)
-   {
-      yy -= floor(yy/sy) * sy;
-   }
-
-
-
-   // double d = (xx-x0)*a(0) + (yy-y0)*a(1);
-   // double d1 = (xx-x0-0.5)*a(0) + (yy-y0-0.5)*a(1);
-   // double d2 = (xx-x0+0.5)*a(0) + (yy-y0+0.5)*a(1);
-   // return 1. + exp(-w*(d*d)) + exp(-w*(d1*d1)) + exp(-w*(d2*d2));
-   double dr_x = (xx-x0)*(xx-x0);
-   double dr_y = (yy-y0)*(yy-y0);
-   return 1. + exp(-w*(dr_x+dr_y));
-   // return 1. + exp(-w*(dr_x));
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   return 1.0;
-}
-
-
-void Prefine(FiniteElementSpace & fes_old,
-             GridFunction &u, Coefficient &ex, GridFunction &orders_gf,
-             double min_thresh, double max_thresh)
-{
-   // get element errors
-   FiniteElementSpace * fes = u.FESpace();
-   int ne = fes->GetMesh()->GetNE();
-   Vector errors(ne);
-   u.ComputeElementL2Errors(ex,errors);
-   for (int i = 0; i<ne; i++)
-   {
-      double error = errors(i);
-      int order = fes->GetElementOrder(i);
-      if (error < min_thresh && order > 1)
-      {
-         fes->SetElementOrder(i,order-1);
-      }
-      else if (error > max_thresh && order < 2)
-      {
-         fes->SetElementOrder(i, order+1);
-      }
-      else
-      {
-         // do nothing
-      }
-   }
-
-   fes->Update(false);
-
-   PRefinementTransferOperator * T = new PRefinementTransferOperator(fes_old,*fes);
-
-   GridFunction u_fine(fes);
-   T->Mult(u,u_fine);
-
-   // copy the orders to the old space
-   for (int i = 0; i<ne; i++)
-   {
-      int order = fes->GetElementOrder(i);
-      fes_old.SetElementOrder(i,order);
-      orders_gf(i) = order;
-   }
-   fes_old.Update(false);
-
-   delete T;
-
-   // update old gridfuntion;
-   u = u_fine;
-
-}
-
-void Hrefine2(GridFunction &u, Coefficient & ex_coeff, double min_thresh,
-              double max_thresh)
-{
-   FiniteElementSpace * fes = u.FESpace();
-   Mesh * mesh = fes->GetMesh();
-   int ne = mesh->GetNE();
-   Vector errors(ne);
-   u.ComputeElementL2Errors(ex_coeff,errors);
-
-   Array<int> actions(ne);
-   for (int i = 0; i<ne; i++)
-   {
-      double error = errors(i);
-      if (error > max_thresh)
-      {
-         actions[i] = 1;
-      }
-      else if (error < min_thresh)
-      {
-         actions[i] = -1;
-      }
-      else
-      {
-         actions[i] = 0;
-      }
-   }
-   Refine(actions,u,1);
-
-   // construct a list of possible ref actions
-   // Array<int> actions(ne);
-   // for (int i = 0; i<ne; i++)
-   // {
-   //    double error = errors(i);
-   //    if (error > max_thresh && mesh->ncmesh->GetElementDepth(i) < 1)
-   //    {
-   //       actions[i] = 1;
-   //    }
-   //    else
-   //    {
-   //       actions[i] = 0;
-   //    }
-   // }
-
-   // // list of possible dref actions
-   // Array<int> derefactions(ne); derefactions = 0;
-   // const Table & dref_table = mesh->ncmesh->GetDerefinementTable();
-   // for (int i = 0; i<dref_table.Size(); i++)
-   // {
-   //    int size = dref_table.RowSize(i);
-   //    const int * row = dref_table.GetRow(i);
-   //    double error = 0.;
-   //    for (int j = 0; j<size; j++)
-   //    {
-   //       error += errors[row[j]];
-   //    }
-   //    if (error < min_thresh)
-   //    {
-   //       for (int j = 0; j<size; j++)
-   //       {
-   //          actions[row[j]] += -1;
-   //       }
-   //    }
-   // }
-
-   // // now refine the elements that have score >0 and deref the elements that have score < 0
-   // Array<Refinement> elements_to_refine;
-   // for (int i = 0; i<ne; i++)
-   // {
-   //    if (actions[i] > 0)
-   //    {
-   //       elements_to_refine.Append(Refinement(i,0b01));
-   //    }
-   // }
-
-   // mesh->GeneralRefinement(elements_to_refine);
-   // fes->Update();
-   // u.Update();
-
-   // // map old actions to new mesh
-   // Array<int> new_actions(mesh->GetNE());
-   // if (mesh->GetLastOperation() == mesh->REFINE)
-   // {
-   //    const CoarseFineTransformations &tr = mesh->GetRefinementTransforms();
-   //    Table coarse2fine;
-   //    tr.MakeCoarseToFineTable(coarse2fine);
-   //    new_actions = 1;
-   //    for (int i = 0; i<coarse2fine.Size(); i++)
-   //    {
-   //       if (coarse2fine.RowSize(i) == 1)
-   //       {
-   //          int * el = coarse2fine.GetRow(i);
-   //          new_actions[el[0]] = actions[i];
-   //       }
-   //    }
-   // }
-   // else
-   // {
-   //    new_actions = actions;
-   // }
-
-   // // create a dummy error vector
-   // Vector new_errors(mesh->GetNE());
-   // new_errors = infinity();
-   // for (int i = 0; i< new_errors.Size(); i++)
-   // {
-   //    if (new_actions[i] < 0)
-   //    {
-   //       new_errors[i] = 0.;
-   //    }
-   // }
-
-   // // any threshold would do here
-   // mesh->DerefineByError(new_errors,min_thresh);
-
-   // fes->Update();
-   // u.Update();
-}
-
-
-Table * Refine(Array<int> ref_actions, GridFunction &u, int depth_limit)
-{
-   FiniteElementSpace * fes = u.FESpace();
-   Mesh * mesh = fes->GetMesh();
-   int ne = mesh->GetNE();
-
-
-   //  ovewrite to no action if an element is marked for refinement but it exceeds the depth limit
-   for (int i = 0; i<ne; i++)
-   {
-      int depth = mesh->ncmesh->GetElementDepth(i);
-      if (depth >= depth_limit && ref_actions[i] == 1)
-      {
-         ref_actions[i] = 0;
-      }
-   }
-
-   // current policy to map agent_actions to actions
-   // 1. All elements that are marked for refinement are to perform the refinement
-   // 2. All of the "siblings" (i) of a marked element for refinement are assigned action=max(0,agent_actions[i])
-   //   i.e., a) if the action is to be refined then they are refined
-   //         b) if the action is to be derefined or no action then they get no action
-   // 3. If among the "siblings" there is no refinement action then the group is marked
-   //    for derefinement if the majority (including a tie) of the siblings are marked for derefinement
-   //    otherwise they are marked for no action
-   //  h-refine:   action =  1
-   //  h-derefine: action = -1
-   //  do nothing: action =  0
-
-   Array<int> actions(ne);
-   Array<int> actions_marker(ne);
-   actions_marker = 0;
-
-   const Table & deref_table = mesh->ncmesh->GetDerefinementTable();
-
-   for (int i = 0; i<deref_table.Size(); i++)
-   {
-      int n = deref_table.RowSize(i);
-      const int * row = deref_table.GetRow(i);
-      int sum_of_actions = 0;
-      bool ref_flag = false;
-      for (int j = 0; j<n; j++)
-      {
-         int action = ref_actions[row[j]];
-         sum_of_actions+=action;
-         if (action == 1)
-         {
-            ref_flag = true;
-            break;
-         }
-      }
-      if (ref_flag)
-      {
-         for (int j = 0; j<n; j++)
-         {
-            actions[row[j]] = max(0,ref_actions[row[j]]);
-            actions_marker[row[j]] = 1;
-         }
-      }
-      else
-      {
-         bool dref_flag = (2*abs(sum_of_actions) >= n) ? true : false;
-         for (int j = 0; j<n; j++)
-         {
-            actions[row[j]] = (dref_flag) ? -1 : 0;
-            actions_marker[row[j]] = 1;
-         }
-      }
-   }
-
-   for (int i = 0; i<ne; i++)
-   {
-      if (actions_marker[i] != 1)
-      {
-         if (ref_actions[i] == -1)
-         {
-            actions[i] = 0;
-         }
-         else
-         {
-            actions[i] = ref_actions[i];
-         }
-      }
-   }
-
-   // now the actions array holds feasible actions of -1,0,1
-   Array<Refinement> refinements;
-   for (int i = 0; i<ne; i++)
-   {
-      if (actions[i] == 1) {refinements.Append(Refinement(i,0b11));}
-   }
-   if (refinements.Size())
-   {
-      mesh->GeneralRefinement(refinements);
-      fes->Update();
-      u.Update();
-      ne = mesh->GetNE();
-   }
-
-   Table * ref_table = nullptr;
-   Table * dref_table = nullptr;
-   // now the derefinements
-   Array<int> new_actions(ne);
-   if (refinements.Size())
-   {
-      new_actions = 1;
-      const CoarseFineTransformations & tr = mesh->GetRefinementTransforms();
-      ref_table = new Table();
-      tr.MakeCoarseToFineTable(*ref_table);
-      for (int i = 0; i<ref_table->Size(); i++)
-      {
-         int n = ref_table->RowSize(i);
-         if (n == 1)
-         {
-            int * row = ref_table->GetRow(i);
-            new_actions[row[0]] = actions[i];
-         }
-      }
-   }
-   else
-   {
-      new_actions = actions;
-   }
-
-   Vector dummy_errors(ne);
-   dummy_errors = 1.0;
-   for (int i = 0; i<ne; i++)
-   {
-      if (new_actions[i] < 0)
-      {
-         dummy_errors[i] = 0.;
-      }
-   }
-   mesh->DerefineByError(dummy_errors,0.5);
-
-   fes->Update();
-   u.Update();
-
-   if (mesh->GetNE() < ne)
-   {
-      const CoarseFineTransformations & tr =
-         mesh->ncmesh->GetDerefinementTransforms();
-      Table coarse_to_fine_table;
-      tr.MakeCoarseToFineTable(coarse_to_fine_table);
-      dref_table = Transpose(coarse_to_fine_table);
-   }
-
-   // Build combined table of mesh modifications
-   Table * T = nullptr;
-   if (ref_table && dref_table)
-   {
-      T = Mult(*ref_table, * dref_table);
-      delete dref_table;
-      delete ref_table;
-   }
-   else if (ref_table)
-   {
-      T = ref_table;
-      delete dref_table;
-   }
-   else if (dref_table)
-   {
-      T= dref_table;
-      delete ref_table;
-   }
-   else
-   {
-      // do nothing: no mesh modifications happened
-   }
-
-   return T;
-}
@@ -1,957 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <algorithm>
-
-using namespace std;
-using namespace mfem;
-
-double sx, sy;
-
-Vector vel;
-
-
-void Prefine(FiniteElementSpace & fes_old,
-             GridFunction &u, GridFunction &pref_gf, GridFunction &orders_gf,
-             double min_thresh, double max_thresh);
-
-// void Prefine(FiniteElementSpace & fes_old,
-//              GridFunction &u, Coefficient &gf_ex, GridFunction &orders_gf,
-//              double min_thresh, double max_thresh);
-
-void Hrefine(GridFunction &u, Coefficient &gf_ex, double min_thresh,
-             double max_thresh);
-
-// void Hrefine2(GridFunction &u, Coefficient &gf_ex, double min_thresh,
-//   double max_thresh);
-Table * Hrefine2(GridFunction &u, GridFunction &u_ref, Table * refT,
-                 Coefficient &gf_ex, double min_thresh,
-                 double max_thresh);
-
-Table * Refine(Array<int> ref_actions, GridFunction &u, int depth_limit = 100);
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v);
-
-// Initial condition
-double u0_function(const Vector &x, double);
-
-// Inflow boundary condition
-double inflow_function(const Vector &x);
-
-class FE_Evolution : public TimeDependentOperator
-{
-private:
-   BilinearForm &M, &K;
-   const Vector &b;
-   Solver *M_prec;
-   CGSolver M_solver;
-
-   mutable Vector z;
-
-public:
-   FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_);
-   void Update();
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~FE_Evolution();
-};
-
-enum ref_kind
-{
-   order,        // p refinement
-   geometric     // h refinement
-};
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   const char *mesh_file = "../data/periodic-hexagon.mesh";
-   int ref_levels = 2;
-   int order = 1;
-   sx = 1.0;
-   sy = 1.0;
-   double t_final = 1.0;
-   double dt = 0.002;
-   bool visualization = true;
-   int vis_steps = 5;
-   int refmode = 0;
-   int precision = 8;
-   ref_kind ref_mode = ref_kind::geometric;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&mesh_file, "-m", "--mesh",
-                  "Mesh file to use.");
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&t_final, "-tf", "--t-final",
-                  "Final time; start time is 0.");
-   args.AddOption(&dt, "-dt", "--time-step",
-                  "Time step.");
-   args.AddOption(&sx, "-sx", "--sx",
-                  "mesh length in x direction");
-   args.AddOption(&sy, "-sy", "--sy",
-                  "mesh length in y direction");
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&vis_steps, "-vs", "--visualization-steps",
-                  "Visualize every n-th timestep.");
-   args.AddOption(&refmode, "-rm", "--refinement-mode",
-                  "0: 'p', 1: 'h' ");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-   ref_mode = (ref_kind)refmode;
-
-   Mesh mesh0 = Mesh::MakeCartesian2D(32,32,mfem::Element::QUADRILATERAL,false,sx,
-                                      sy);
-   std::vector<Vector> translations = {Vector({sx,0.0}), Vector({0.0,sy})};
-   Mesh mesh = Mesh::MakePeriodic(mesh0,
-                                  mesh0.CreatePeriodicVertexMapping(translations));
-
-   mesh.EnsureNCMesh();
-
-   // compute reference solution
-   Mesh ref_mesh(mesh);
-
-   int dim = mesh.Dimension();
-
-   ODESolver *ode_solver = new RK4Solver;
-   ODESolver *ref_ode_solver = new RK4Solver;
-   ODESolver *pref_ode_solver = new RK4Solver;
-
-   // 5. Define the discontinuous DG finite element space of the given
-   //    polynomial order on the refined mesh.
-   DG_FECollection fec(order, dim, BasisType::GaussLobatto);
-   FiniteElementSpace fes(&mesh, &fec);
-   FiniteElementSpace ref_fes(&ref_mesh, &fec);
-   FiniteElementSpace fes_old(&mesh, &fec);
-   FiniteElementSpace pref_fes(&mesh, &fec);
-
-   for (int i = 0; i<mesh.GetNE(); i++)
-   {
-      int order = pref_fes.GetElementOrder(i);
-      pref_fes.SetElementOrder(i,order+2);
-   }
-   pref_fes.Update(false);
-
-
-   cout << "Number of unknowns: " << fes.GetVSize() << endl;
-
-   VectorFunctionCoefficient velocity(dim, velocity_function);
-   FunctionCoefficient inflow(inflow_function);
-   FunctionCoefficient u0(u0_function);
-
-   BilinearForm m(&fes);
-   BilinearForm k(&fes);
-   m.AddDomainIntegrator(new MassIntegrator);
-   constexpr double alpha = -1.0;
-   k.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
-   k.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-   k.AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-
-   LinearForm b(&fes);
-   b.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha,-0.5));
-
-   m.Assemble();
-   int skip_zeros = 0;
-   k.Assemble(skip_zeros);
-   b.Assemble();
-   m.Finalize();
-   k.Finalize(skip_zeros);
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u(&fes);
-   u0.SetTime(0.);
-   u.ProjectCoefficient(u0);
-
-   // reference solution (href)
-   BilinearForm m_ref(&ref_fes);
-   BilinearForm k_ref(&ref_fes);
-   m_ref.AddDomainIntegrator(new MassIntegrator);
-   k_ref.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
-   k_ref.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-   k_ref.AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-
-   LinearForm b_ref(&ref_fes);
-   b_ref.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha,-0.5));
-
-   m_ref.Assemble();
-   k_ref.Assemble(skip_zeros);
-   b_ref.Assemble();
-   m_ref.Finalize();
-   k_ref.Finalize(skip_zeros);
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u_ref(&ref_fes);
-   u_ref.ProjectCoefficient(u0);
-
-
-   Array<int> refinements(ref_mesh.GetNE());
-   refinements = 1;
-
-   Table * T1 = Refine(refinements, u_ref, 2);
-   refinements.SetSize(ref_mesh.GetNE());
-   refinements = 1;
-   Table * T2 = Refine(refinements, u_ref, 2);
-
-   Table * refT = Mult(*T1,*T2);
-
-   m_ref.Update();
-   m_ref.Assemble();
-   m_ref.Finalize();
-   k_ref.Update();
-   k_ref.Assemble(skip_zeros);
-   k_ref.Finalize(skip_zeros);
-   b_ref.Update();
-   b_ref.Assemble();
-
-
-
-   // reference solution (pref)
-   BilinearForm m_pref(&pref_fes);
-   BilinearForm k_pref(&pref_fes);
-   m_pref.AddDomainIntegrator(new MassIntegrator);
-   k_pref.AddDomainIntegrator(new ConvectionIntegrator(velocity, alpha));
-   k_pref.AddInteriorFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-   k_pref.AddBdrFaceIntegrator(
-      new NonconservativeDGTraceIntegrator(velocity, alpha));
-
-   LinearForm b_pref(&pref_fes);
-   b_pref.AddBdrFaceIntegrator(
-      new BoundaryFlowIntegrator(inflow, velocity, alpha,-0.5));
-
-   m_pref.Assemble();
-   k_pref.Assemble(skip_zeros);
-   b_pref.Assemble();
-   m_pref.Finalize();
-   k_pref.Finalize(skip_zeros);
-
-   // 7. Define the initial conditions, save the corresponding grid function to
-   //    a file and (optionally) save data in the VisIt format and initialize
-   //    GLVis visualization.
-   GridFunction u_pref(&pref_fes);
-   u_pref.ProjectCoefficient(u0);
-   m_ref.Assemble();
-   m_ref.Finalize();
-   k_ref.Assemble(skip_zeros);
-   k_ref.Finalize(skip_zeros);
-
-
-   L2_FECollection orders_fec(0,dim);
-   FiniteElementSpace orders_fes(&mesh,&orders_fec);
-   GridFunction orders_gf(&orders_fes);
-   for (int i = 0; i<mesh.GetNE(); i++) { orders_gf(i) = order; }
-
-
-   socketstream sout;
-   // socketstream s_refout;
-   socketstream meshout;
-   if (visualization)
-   {
-      char vishost[] = "localhost";
-      int  visport   = 19916;
-      sout.open(vishost, visport);
-      // s_refout.open(vishost, visport);
-      meshout.open(vishost, visport);
-      if (!sout)
-      {
-         cout << "Unable to connect to GLVis server at "
-              << vishost << ':' << visport << endl;
-         visualization = false;
-         cout << "GLVis visualization disabled.\n";
-      }
-      else
-      {
-         sout.precision(precision);
-         sout << "solution\n" << mesh << u;
-         sout << flush;
-         // s_refout.precision(precision);
-         // s_refout << "solution\n" << ref_mesh << u_ref;
-         // s_refout << flush;
-
-         meshout.precision(precision);
-         meshout << "solution\n" << mesh << orders_gf;
-         meshout << flush;
-         cin.get();
-      }
-   }
-
-
-   // 8. Define the time-dependent evolution operator describing the ODE
-   //    right-hand side, and perform time-integration (looping over the time
-   //    iterations, ti, with a time-step dt).
-   FE_Evolution adv(m, k, b);
-   FE_Evolution ref_adv(m_ref, k_ref, b_ref);
-   FE_Evolution pref_adv(m_pref, k_pref, b_pref);
-
-   double t = 0.0;
-   adv.SetTime(t);
-   double ref_t = 0.0;
-   ref_adv.SetTime(ref_t);
-
-   double pref_t = 0.0;
-   pref_adv.SetTime(pref_t);
-
-   FunctionCoefficient u_ex(u0_function);
-
-   ode_solver->Init(adv);
-   ref_ode_solver->Init(ref_adv);
-   pref_ode_solver->Init(pref_adv);
-
-   bool done = false;
-   for (int ti = 0; !done; )
-   {
-      double dt_real = min(dt, t_final - t);
-
-      ode_solver->Step(u, t, dt_real);
-      ref_ode_solver->Step(u_ref, ref_t, dt_real);
-      pref_ode_solver->Step(u_pref, pref_t, dt_real);
-      ti++;
-
-      done = (t >= t_final - 1e-8*dt);
-
-      if (done || ti % vis_steps == 0)
-      {
-         cout << "time step: " << ti << ", time: " << t << endl;
-         u_ex.SetTime(t);
-         // Prefine(fes_old,u,u_ex, orders_gf, 5e-5, 5e-4);
-
-         mfem::out << "Global L2 Error = " << u.ComputeL2Error(u_ex) << std::endl;
-         // Prefine(fes_old,u,u_ex, orders_gf, 5e-5, 5e-4);
-         // Hrefine2(u,u_ex, 5e-5, 5e-4);
-         // mfem::out << "refT size = " << refT->Size() << " x " << refT->Width() << endl;
-         if (ref_mode == ref_kind::geometric)
-         {
-            refT = Hrefine2(u,u_ref, refT, u_ex, 5e-5, 5e-4);
-         }
-         else
-         {
-            // Prefine(fes_old,u,u_ex, orders_gf, 5e-5, 5e-4);
-            Prefine(fes_old,u,u_pref, orders_gf, 5e-5, 5e-4);
-         }
-         // mfem::out << "refT size = " << refT->Size() << " x " << refT->Width() << endl;
-         // mfem::out << "number of elements = " << mesh.GetNE() << endl;
-
-
-         m.Update();
-         m.Assemble();
-         m.Finalize();
-         k.Update();
-         k.Assemble(skip_zeros);
-         k.Finalize(skip_zeros);
-         b.Update();
-         b.Assemble();
-         adv.Update();
-         ode_solver->Init(adv);
-
-
-         // ref_adv.Update();
-         // ref_ode_solver->Init(ref_adv);
-         if (visualization)
-         {
-            // GridFunction gf_ex(&fes);
-            // gf_ex.ProjectCoefficient(u_ex);
-            GridFunction * pr_u = ProlongToMaxOrder(&u);
-            sout << "solution\n" << mesh << *pr_u << flush;
-            // s_refout << "solution\n" << ref_mesh << u_ref << flush;
-
-            meshout << "solution\n" << mesh << orders_gf << flush;
-         }
-      }
-   }
-
-   // 10. Free the used memory.
-   delete ode_solver;
-   delete ref_ode_solver;
-   delete pref_ode_solver;
-
-   return 0;
-}
-
-
-// Implementation of class FE_Evolution
-FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
-{
-   Array<int> ess_tdof_list;
-   M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.SetOperator(M);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Update()
-{
-   height = M.Height();
-   width = M.Width();
-   z.SetSize(M.Height());
-
-   Array<int> ess_tdof_list;
-   delete M_prec;
-   M_prec = new OperatorJacobiSmoother(M, ess_tdof_list);
-   M_solver.SetPreconditioner(*M_prec);
-   M_solver.SetOperator(M);
-   M_solver.iterative_mode = false;
-   M_solver.SetRelTol(1e-9);
-   M_solver.SetAbsTol(0.0);
-   M_solver.SetMaxIter(100);
-   M_solver.SetPrintLevel(0);
-}
-
-void FE_Evolution::Mult(const Vector &x, Vector &y) const
-{
-   // y = M^{-1} (K x + b)
-   K.Mult(x, z);
-   z += b;
-   M_solver.Mult(z, y);
-}
-
-FE_Evolution::~FE_Evolution()
-{
-   delete M_prec;
-}
-
-// Velocity coefficient
-void velocity_function(const Vector &x, Vector &v)
-{
-   v.SetSize(2);
-   v(0) = 1.;
-   v(1) = 1.;
-}
-
-// Initial condition
-double u0_function(const Vector &x, double t)
-{
-   // give x0, y0;
-
-   // Rotation matrix
-   double theta = M_PI/4;
-   //
-
-   double x0 = 0.5;
-   double y0 = 0.5;
-   double w = 100.;
-   double c = 1.;
-   double ds = c*t;
-   Vector a(2);
-   a(0) = cos(theta);
-   a(1) = sin(theta);
-   // double xx = x(0) - a(0)*ds;
-   // double yy = x(1) - a(1)*ds;
-   double xx = x(0) - ds;
-   double yy = x(1) - ds;
-
-   double tol = 1e-6;
-   if (xx>= sx+tol || xx<= 0.0-tol)
-   {
-      xx -= floor(xx/sx) * sx;
-   }
-   if (yy>= sy+tol || yy<= 0.0-tol)
-   {
-      yy -= floor(yy/sy) * sy;
-   }
-
-
-
-   // double d = (xx-x0)*a(0) + (yy-y0)*a(1);
-   // double d1 = (xx-x0-0.5)*a(0) + (yy-y0-0.5)*a(1);
-   // double d2 = (xx-x0+0.5)*a(0) + (yy-y0+0.5)*a(1);
-   // return 1. + exp(-w*(d*d)) + exp(-w*(d1*d1)) + exp(-w*(d2*d2));
-   double dr_x = (xx-x0)*(xx-x0);
-   double dr_y = (yy-y0)*(yy-y0);
-   return 1. + exp(-w*(dr_x+dr_y));
-   // return 1. + exp(-w*(dr_x));
-}
-
-// Inflow boundary condition (zero for the problems considered in this example)
-double inflow_function(const Vector &x)
-{
-   return 1.0;
-}
-
-
-// void Prefine(FiniteElementSpace & fes_old,
-//              GridFunction &u, Coefficient &ex, GridFunction &orders_gf,
-//              double min_thresh, double max_thresh)
-
-void Prefine(FiniteElementSpace & fes_old,
-             GridFunction &u, GridFunction &pref_gf, GridFunction &orders_gf,
-             double min_thresh, double max_thresh)
-{
-   // get element errors
-   FiniteElementSpace * fes = u.FESpace();
-   int ne = fes->GetMesh()->GetNE();
-   Vector errors(ne);
-
-   GridFunction pru(pref_gf.FESpace());
-   PRefinementTransferOperator * P = new PRefinementTransferOperator(*fes,
-                                                                     *pref_gf.FESpace());
-   P->Mult(u,pru);
-   delete P;
-   pru-=pref_gf;
-   ConstantCoefficient zero(0.0);
-   pru.ComputeElementL2Errors(zero,errors);
-
-   // u.ComputeElementL2Errors(ex,errors);
-   for (int i = 0; i<ne; i++)
-   {
-      double error = errors(i);
-      int order = fes->GetElementOrder(i);
-      if (error < min_thresh && order > 1)
-      {
-         fes->SetElementOrder(i,order-1);
-      }
-      else if (error > max_thresh && order < 2)
-      {
-         fes->SetElementOrder(i, order+1);
-      }
-      else
-      {
-         // do nothing
-      }
-   }
-
-   fes->Update(false);
-
-   PRefinementTransferOperator * T = new PRefinementTransferOperator(fes_old,*fes);
-
-   GridFunction u_fine(fes);
-   T->Mult(u,u_fine);
-
-   // copy the orders to the old space
-   for (int i = 0; i<ne; i++)
-   {
-      int order = fes->GetElementOrder(i);
-      fes_old.SetElementOrder(i,order);
-      orders_gf(i) = order;
-   }
-   fes_old.Update(false);
-
-   delete T;
-
-   // update old gridfuntion;
-   u = u_fine;
-
-}
-
-// void Hrefine2(GridFunction &u, Coefficient & ex_coeff, double min_thresh,
-//               double max_thresh)
-Table * Hrefine2(GridFunction &u, GridFunction &u_ref, Table * refT,
-                 Coefficient & ex_coeff, double min_thresh,
-                 double max_thresh)
-{
-   FiniteElementSpace * fes = u.FESpace();
-   Mesh * mesh = fes->GetMesh();
-   int ne = mesh->GetNE();
-   Vector errors(ne);
-   // u.ComputeElementL2Errors(ex_coeff,errors);
-
-   // copy the fespace, refine it up to element depth 2 and calculate the errors
-   // copy mesh
-   Mesh fine_mesh(*mesh);
-
-   FiniteElementSpace fes_copy(&fine_mesh,fes->FEColl());
-   GridFunction u_fine(&fes_copy);
-   // copy data;
-   u_fine = u;
-   Array<int>refinements(fine_mesh.GetNE());
-   refinements = 1;
-   Table * T1 = Refine(refinements,u_fine,2);
-   refinements.SetSize(fine_mesh.GetNE());
-   refinements = 1;
-   Table * T2 = Refine(refinements,u_fine,2);
-   Table * T = Mult(*T1, *T2);
-
-   delete T1;
-   delete T2;
-
-   // constract map
-   int n = T->Size();
-   int m = T->Width();
-   Array<int> elem_map(m);
-   for (int i = 0; i< n; i++)
-   {
-      int nr = T->RowSize(i);
-      int * row = T->GetRow(i);
-      int * ref_row = refT->GetRow(i);
-      for (int j = 0; j<nr ; j++ )
-      {
-         elem_map[row[j]] = ref_row[j];
-      }
-   }
-
-
-   // Table *fine2refT = Transpose(*Mult(*Transpose(*T), *refT));
-
-
-   // char vishost[] = "localhost";
-   // int  visport   = 19916;
-   // socketstream pr_out(vishost, visport);
-   // pr_out << "solution\n" << fine_mesh << u_fine << flush;
-
-   // calculate error
-
-   GridFunction diff(u_fine);
-   // this needs to change for reordering
-   diff-= u_ref;
-
-   ConstantCoefficient zero(0.0);
-   Vector fine_errors(fine_mesh.GetNE());
-   diff.ComputeElementL2Errors(zero,fine_errors);
-
-   // combine fine errors to current mesh;
-   // Table *Tt = Transpose(*T);
-   // mfem::out << "Tt->Size = " << Tt->Size() << endl;
-   // mfem::out << "errors = " << errors.Size() << endl;
-   for (int i = 0; i<T->Size(); i++)
-   {
-      int m = T->RowSize(i);
-      int *row = T->GetRow(i);
-      double err = 0.;
-      for (int j = 0; j<m; j++)
-      {
-         err += fine_errors[row[j]]*fine_errors[row[j]];
-      }
-      errors[i] = sqrt(err);
-   }
-
-
-
-   // cin.get();
-
-   // compute element errors by
-   // 1. refine the mesh up to mesh limit 2
-   // 2. Prolongate the current solution to the refined mesh
-   // 3. Calculate errors and combine them (for the coarse elements)
-   // 4. Derifine mesh
-
-   //copy the mesh
-   // Mesh * ref_mesh = new Mesh(*mesh);
-   // Array<int> ref_actions(ne);
-   // ref_actions = 1;
-   // NCMesh * ref_ncmesh = ref_mesh->ncmesh;
-
-
-
-
-   Array<int> actions(ne);
-   for (int i = 0; i<ne; i++)
-   {
-      double error = errors(i);
-      if (error > max_thresh)
-      {
-         actions[i] = 1;
-      }
-      else if (error < min_thresh)
-      {
-         actions[i] = -1;
-      }
-      else
-      {
-         actions[i] = 0;
-      }
-   }
-
-   Table * T3 = Refine(actions,u,1);
-
-
-   if (T3)
-   {
-      Table *Ttt = Mult(*Transpose(*T3), *refT);
-      delete T3;
-      delete refT;
-      refT = Ttt;
-   }
-
-   return refT;
-
-   // construct a list of possible ref actions
-   // Array<int> actions(ne);
-   // for (int i = 0; i<ne; i++)
-   // {
-   //    double error = errors(i);
-   //    if (error > max_thresh && mesh->ncmesh->GetElementDepth(i) < 1)
-   //    {
-   //       actions[i] = 1;
-   //    }
-   //    else
-   //    {
-   //       actions[i] = 0;
-   //    }
-   // }
-
-   // // list of possible dref actions
-   // Array<int> derefactions(ne); derefactions = 0;
-   // const Table & dref_table = mesh->ncmesh->GetDerefinementTable();
-   // for (int i = 0; i<dref_table.Size(); i++)
-   // {
-   //    int size = dref_table.RowSize(i);
-   //    const int * row = dref_table.GetRow(i);
-   //    double error = 0.;
-   //    for (int j = 0; j<size; j++)
-   //    {
-   //       error += errors[row[j]];
-   //    }
-   //    if (error < min_thresh)
-   //    {
-   //       for (int j = 0; j<size; j++)
-   //       {
-   //          actions[row[j]] += -1;
-   //       }
-   //    }
-   // }
-
-   // // now refine the elements that have score >0 and deref the elements that have score < 0
-   // Array<Refinement> elements_to_refine;
-   // for (int i = 0; i<ne; i++)
-   // {
-   //    if (actions[i] > 0)
-   //    {
-   //       elements_to_refine.Append(Refinement(i,0b01));
-   //    }
-   // }
-
-   // mesh->GeneralRefinement(elements_to_refine);
-   // fes->Update();
-   // u.Update();
-
-   // // map old actions to new mesh
-   // Array<int> new_actions(mesh->GetNE());
-   // if (mesh->GetLastOperation() == mesh->REFINE)
-   // {
-   //    const CoarseFineTransformations &tr = mesh->GetRefinementTransforms();
-   //    Table coarse2fine;
-   //    tr.MakeCoarseToFineTable(coarse2fine);
-   //    new_actions = 1;
-   //    for (int i = 0; i<coarse2fine.Size(); i++)
-   //    {
-   //       if (coarse2fine.RowSize(i) == 1)
-   //       {
-   //          int * el = coarse2fine.GetRow(i);
-   //          new_actions[el[0]] = actions[i];
-   //       }
-   //    }
-   // }
-   // else
-   // {
-   //    new_actions = actions;
-   // }
-
-   // // create a dummy error vector
-   // Vector new_errors(mesh->GetNE());
-   // new_errors = infinity();
-   // for (int i = 0; i< new_errors.Size(); i++)
-   // {
-   //    if (new_actions[i] < 0)
-   //    {
-   //       new_errors[i] = 0.;
-   //    }
-   // }
-
-   // // any threshold would do here
-   // mesh->DerefineByError(new_errors,min_thresh);
-
-   // fes->Update();
-   // u.Update();
-}
-
-
-Table * Refine(Array<int> ref_actions, GridFunction &u, int depth_limit)
-{
-   FiniteElementSpace * fes = u.FESpace();
-   Mesh * mesh = fes->GetMesh();
-   int ne = mesh->GetNE();
-
-
-   //  ovewrite to no action if an element is marked for refinement but it exceeds the depth limit
-   for (int i = 0; i<ne; i++)
-   {
-      int depth = mesh->ncmesh->GetElementDepth(i);
-      if (depth >= depth_limit && ref_actions[i] == 1)
-      {
-         ref_actions[i] = 0;
-      }
-   }
-
-   // current policy to map agent_actions to actions
-   // 1. All elements that are marked for refinement are to perform the refinement
-   // 2. All of the "siblings" (i) of a marked element for refinement are assigned action=max(0,agent_actions[i])
-   //   i.e., a) if the action is to be refined then they are refined
-   //         b) if the action is to be derefined or no action then they get no action
-   // 3. If among the "siblings" there is no refinement action then the group is marked
-   //    for derefinement if the majority (including a tie) of the siblings are marked for derefinement
-   //    otherwise they are marked for no action
-   //  h-refine:   action =  1
-   //  h-derefine: action = -1
-   //  do nothing: action =  0
-
-   Array<int> actions(ne);
-   Array<int> actions_marker(ne);
-   actions_marker = 0;
-
-   const Table & deref_table = mesh->ncmesh->GetDerefinementTable();
-
-   for (int i = 0; i<deref_table.Size(); i++)
-   {
-      int n = deref_table.RowSize(i);
-      const int * row = deref_table.GetRow(i);
-      int sum_of_actions = 0;
-      bool ref_flag = false;
-      for (int j = 0; j<n; j++)
-      {
-         int action = ref_actions[row[j]];
-         sum_of_actions+=action;
-         if (action == 1)
-         {
-            ref_flag = true;
-            break;
-         }
-      }
-      if (ref_flag)
-      {
-         for (int j = 0; j<n; j++)
-         {
-            actions[row[j]] = max(0,ref_actions[row[j]]);
-            actions_marker[row[j]] = 1;
-         }
-      }
-      else
-      {
-         bool dref_flag = (2*abs(sum_of_actions) >= n) ? true : false;
-         for (int j = 0; j<n; j++)
-         {
-            actions[row[j]] = (dref_flag) ? -1 : 0;
-            actions_marker[row[j]] = 1;
-         }
-      }
-   }
-
-   for (int i = 0; i<ne; i++)
-   {
-      if (actions_marker[i] != 1)
-      {
-         if (ref_actions[i] == -1)
-         {
-            actions[i] = 0;
-         }
-         else
-         {
-            actions[i] = ref_actions[i];
-         }
-      }
-   }
-
-   // now the actions array holds feasible actions of -1,0,1
-   Array<Refinement> refinements;
-   for (int i = 0; i<ne; i++)
-   {
-      if (actions[i] == 1) {refinements.Append(Refinement(i,0b11));}
-   }
-   if (refinements.Size())
-   {
-      mesh->GeneralRefinement(refinements);
-      fes->Update();
-      u.Update();
-      ne = mesh->GetNE();
-   }
-
-   Table * ref_table = nullptr;
-   Table * dref_table = nullptr;
-   // now the derefinements
-   Array<int> new_actions(ne);
-   if (refinements.Size())
-   {
-      new_actions = 1;
-      const CoarseFineTransformations & tr = mesh->GetRefinementTransforms();
-      ref_table = new Table();
-      tr.MakeCoarseToFineTable(*ref_table);
-      for (int i = 0; i<ref_table->Size(); i++)
-      {
-         int n = ref_table->RowSize(i);
-         if (n == 1)
-         {
-            int * row = ref_table->GetRow(i);
-            new_actions[row[0]] = actions[i];
-         }
-      }
-   }
-   else
-   {
-      new_actions = actions;
-   }
-
-   Vector dummy_errors(ne);
-   dummy_errors = 1.0;
-   for (int i = 0; i<ne; i++)
-   {
-      if (new_actions[i] < 0)
-      {
-         dummy_errors[i] = 0.;
-      }
-   }
-   mesh->DerefineByError(dummy_errors,0.5);
-
-   fes->Update();
-   u.Update();
-
-   if (mesh->GetNE() < ne)
-   {
-      const CoarseFineTransformations & tr =
-         mesh->ncmesh->GetDerefinementTransforms();
-      Table coarse_to_fine_table;
-      tr.MakeCoarseToFineTable(coarse_to_fine_table);
-      dref_table = Transpose(coarse_to_fine_table);
-   }
-
-   // Build combined table of mesh modifications
-   Table * T = nullptr;
-   if (ref_table && dref_table)
-   {
-      T = Mult(*ref_table, * dref_table);
-      delete dref_table;
-      delete ref_table;
-   }
-   else if (ref_table)
-   {
-      T = ref_table;
-      delete dref_table;
-   }
-   else if (dref_table)
-   {
-      T= dref_table;
-      delete ref_table;
-   }
-   else
-   {
-      // do nothing: no mesh modifications happened
-   }
-
-   return T;
-}
@@ -30,6 +30,7 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
+//             * ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -37,9 +38,13 @@
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
 //               ex1 -pa -d ceed-cpu -o 4 -a
+//               ex1 -pa -d ceed-cpu -m ../data/square-mixed.mesh
+//               ex1 -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
 //             * ex1 -pa -d ceed-cuda
 //             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -73,6 +78,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
+   bool fa = false;
   const char *device_config = "cpu";
   bool visualization = true;
   bool algebraic_ceed = false;
@@ -87,6 +93,8 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
@@ -184,6 +192,7 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
@@ -305,6 +305,8 @@ int main(int argc, char *argv[])
         vis_w.precision(8);
         visualize(vis_w, mesh, &x, &w, "Elastic energy density", true);
      }
+      cout << "GLVis visualization paused."
+           << " Press space (in the GLVis window) to resume it.\n";
   }

   double ee0 = oper.ElasticEnergy(x.GetTrueVector());
@@ -351,6 +351,11 @@ int main(int argc, char *argv[])
         vis_w.precision(8);
         visualize(vis_w, pmesh, &x_gf, &w_gf, "Elastic energy density", true);
      }
+      if (myid == 0)
+      {
+         cout << "GLVis visualization paused."
+              << " Press space (in the GLVis window) to resume it.\n";
+      }
   }

   double ee0 = oper.ElasticEnergy(x_gf);
@@ -43,9 +43,9 @@
 #include <sstream>
 #include <iostream>

-// Classes FE_Evolution, RiemannSolver, DomainIntegrator and FaceIntegrator
+// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
 // shared between the serial and parallel version of the example.
-#include "fem/auxiliary.hpp"
+#include "ex18.hpp"

 // Choice for the problem setup. See InitialCondition in ex18.hpp.
 int problem;
@@ -53,84 +53,21 @@ int problem;
 // Equation constant parameters.
 const int num_equation = 4;
 const double specific_heat_ratio = 1.4;
-const double gas_constant = 8.3145;
+const double gas_constant = 1.0;

 // Maximum characteristic speed (updated by integrators)
 double max_char_speed;

-// Initial condition
-void InitialCondition(const Vector &x, Vector &y)
-{
-   MFEM_ASSERT(x.Size() == 2, "");
-
-   double radius = 0, Minf = 0, beta = 0;
-   if (problem == 1)
-   {
-      // "Fast vortex"
-      radius = 0.2;
-      Minf = 0.5;
-      beta = 1. / 5.;
-   }
-   else if (problem == 2)
-   {
-      // "Slow vortex"
-      radius = 0.2;
-      Minf = 0.05;
-      beta = 1. / 50.;
-   }
-   else
-   {
-      mfem_error("Cannot recognize problem."
-                 "Options are: 1 - fast vortex, 2 - slow vortex");
-   }
-
-   const double xc = 0.0, yc = 0.0;
-
-   // Nice units
-   const double vel_inf = 1.;
-   const double den_inf = 1.;
-
-   // Derive remainder of background state from this and Minf
-   const double pres_inf = (den_inf / specific_heat_ratio) * (vel_inf / Minf) *
-                           (vel_inf / Minf);
-   const double temp_inf = pres_inf / (den_inf * gas_constant);
-
-   double r2rad = 0.0;
-   r2rad += (x(0) - xc) * (x(0) - xc);
-   r2rad += (x(1) - yc) * (x(1) - yc);
-   r2rad /= (radius * radius);
-
-   const double shrinv1 = 1.0 / (specific_heat_ratio - 1.);
-
-   const double velX = vel_inf * (1 - beta * (x(1) - yc) / radius * exp(
-                                     -0.5 * r2rad));
-   const double velY = vel_inf * beta * (x(0) - xc) / radius * exp(-0.5 * r2rad);
-   const double vel2 = velX * velX + velY * velY;
-
-   const double specific_heat = gas_constant * specific_heat_ratio * shrinv1;
-   const double temp = temp_inf - 0.5 * (vel_inf * beta) *
-                       (vel_inf * beta) / specific_heat * exp(-r2rad);
-
-   const double den = den_inf * pow(temp/temp_inf, shrinv1);
-   const double pres = den * gas_constant * temp;
-   const double energy = shrinv1 * pres / den + 0.5 * vel2;
-
-   y(0) = den;
-   y(1) = den * velX;
-   y(2) = den * velY;
-   y(3) = den * energy;
-}
-
 int main(int argc, char *argv[])
 {
   // 1. Parse command-line options.
   problem = 1;
   const char *mesh_file = "../data/periodic-square.mesh";
   int ref_levels = 1;
-   int order = 2;
+   int order = 3;
   int ode_solver_type = 4;
   double t_final = 2.0;
-   double dt = 0.001;
+   double dt = -0.01;
   double cfl = 0.3;
   bool visualization = true;
   int vis_steps = 50;
@@ -256,13 +193,13 @@ int main(int argc, char *argv[])
   Aflux.Assemble();

   NonlinearForm A(&vfes);
-   RiemannSolver rsolver(specific_heat_ratio, num_equation);
-   A.AddInteriorFaceIntegrator(new FaceIntegrator(rsolver, dim, num_equation));
+   RiemannSolver rsolver;
+   A.AddInteriorFaceIntegrator(new FaceIntegrator(rsolver, dim));

   // 8. Define the time-dependent evolution operator describing the ODE
   //    right-hand side, and perform time-integration (looping over the time
   //    iterations, ti, with a time-step dt).
-   EulerSystem euler(vfes, A, Aflux.SpMat(), specific_heat_ratio, num_equation);
+   FE_Evolution euler(vfes, A, Aflux.SpMat());

   // Visualize the density
   socketstream sout;
@@ -308,6 +245,17 @@ int main(int argc, char *argv[])
   double t = 0.0;
   euler.SetTime(t);
   ode_solver->Init(euler);
+
+   if (cfl > 0)
+   {
+      // Find a safe dt, using a temporary vector. Calling Mult() computes the
+      // maximum char speed at all quadrature points on all faces.
+      Vector z(A.Width());
+      max_char_speed = 0.;
+      A.Mult(sol, z);
+      dt = cfl * hmin / max_char_speed / (2*order+1);
+   }
+
   // Integrate in time.
   bool done = false;
   for (int ti = 0; !done; )
@@ -315,6 +263,10 @@ int main(int argc, char *argv[])
      double dt_real = min(dt, t_final - t);

      ode_solver->Step(sol, t, dt_real);
+      if (cfl > 0)
+      {
+         dt = cfl * hmin / max_char_speed / (2*order+1);
+      }
      ti++;

      done = (t >= t_final - 1e-8*dt);
@@ -43,7 +43,7 @@
 #include <sstream>
 #include <iostream>

-// Classes FE_Evolution, RiemannSolver, DomainIntegrator and FaceIntegrator
+// Classes FE_Evolution, RiemannSolver, and FaceIntegrator
 // shared between the serial and parallel version of the example.
 #include "ex18.hpp"

@@ -219,7 +219,7 @@ int main(int argc, char *argv[])
   // 9. Set up the nonlinear form corresponding to the DG discretization of the
   //    flux divergence, and assemble the corresponding mass matrix.
   MixedBilinearForm Aflux(&dfes, &fes);
-   Aflux.AddDomainIntegrator(new DomainIntegrator(dim));
+   Aflux.AddDomainIntegrator(new TransposeIntegrator(new GradientIntegrator()));
   Aflux.Assemble();

   ParNonlinearForm A(&vfes);
@@ -30,13 +30,18 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
+//             * mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
 //               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
 //             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -74,6 +79,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
+   bool fa = false;
   const char *device_config = "cpu";
   bool visualization = true;
   bool algebraic_ceed = false;
@@ -88,6 +94,8 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
@@ -211,6 +219,7 @@ int main(int argc, char *argv[])
   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
@@ -182,7 +182,7 @@ int main(int argc, char *argv[])
   }
   for (int level = 0; level < order_refinements; ++level)
   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection((int)std::pow(2, level+1), dim));
      fespaces.AddOrderRefinedLevel(collections.Last());
   }

@@ -219,7 +219,7 @@ int main(int argc, char *argv[])
   }
   for (int level = 0; level < order_refinements; ++level)
   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection((int)std::pow(2, level+1), dim));
      fespaces->AddOrderRefinedLevel(collections.Last());
   }

@@ -3,34 +3,63 @@
 // Compile with: make ex33
 //
 // Sample runs:  ex33 -m ../data/square-disc.mesh -alpha 0.33 -o 2
+//               ex33 -m ../data/square-disc.mesh -alpha 4.5 -o 3
+//               ex33 -m ../data/star.mesh -alpha 1.4 -o 3
 //               ex33 -m ../data/star.mesh -alpha 0.99 -o 3
 //               ex33 -m ../data/inline-quad.mesh -alpha 0.5 -o 3
+//               ex33 -m ../data/amr-quad.mesh -alpha 1.5 -o 3
 //               ex33 -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
+//               ex33 -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
 //               ex33 -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
+//               ex33 -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
+//
+// Verification runs:
+//    ex33 -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
+//    ex33 -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
+//    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
+//    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
+//
+//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
+//        for all alpha.
 //
 // Description:
 //
 //  In this example we solve the following fractional PDE with MFEM:
 //
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
 //
-//  To solve this FPDE, we rely on a rational approximation [2] of the normal
-//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
-//  boundary conditions). Namely, we first approximate the operator
+//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
+//  N is given by floor(α). By doing so, we obtain
 //
-//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
+//
+//  We first compute the right hand side by solving the integer order PDE
+//
+//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
+//
+//  The remaining FPDE is then given by
+//
+//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
+//
+//  We rely on a rational approximation [2] of the normal linear operator
+//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
+//  and (a-N) in (0,1). We approximate the operator
+//
+//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
 //
 //  where I is the L2-identity operator and the coefficients c_i and d_i
 //  are generated offline to a prescribed accuracy in a pre-processing step.
 //  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve N+1
+//  that this partial fractional expansion derives from. We then solve M+1
 //  independent integer-order PDEs,
 //
-//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
+//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
 //
 //  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
 //
-//    u ≈ Σ_{i=0}^N u_i.
+//    u ≈ Σ_{i=0}^M u_i.
+//
+//  (If alpha is an integer, we stop after the first PDE was solved.)
 //
 // References:
 //
@@ -47,6 +76,8 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <math.h>
+#include <string>

 #include "ex33.hpp"

@@ -59,8 +90,9 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int num_refs = 3;
-   bool visualization = true;
   double alpha = 0.5;
+   bool visualization = true;
+   bool verification = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -75,6 +107,9 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
+                  "--no-verification",
+                  "Use sinusoidal function (f) for analytic comparison.");
   args.Parse();
   if (!args.Good())
   {
@@ -84,9 +119,31 @@ int main(int argc, char *argv[])
   args.PrintOptions(cout);

   Array<double> coeffs, poles;
+   int progress_steps = 1;

-   // 2. Compute the coefficients that define the integer-order PDEs.
-   ComputePartialFractionApproximation(alpha,coeffs,poles);
+   // 2. Compute the rational expansion coefficients that define the
+   //    integer-order PDEs.
+   const int power_of_laplace = floor(alpha);
+   double exponent_to_approximate = alpha - power_of_laplace;
+   bool integer_order = false;
+   // Check if alpha is an integer or not.
+   if (abs(exponent_to_approximate) > 1e-12)
+   {
+      mfem::out << "Approximating the fractional exponent "
+                << exponent_to_approximate
+                << endl;
+      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
+                                          poles);
+
+      // If the example is build without LAPACK, the exponent_to_approximate
+      // might be modified by the function call above.
+      alpha = exponent_to_approximate + power_of_laplace;
+   }
+   else
+   {
+      integer_order = true;
+      mfem::out << "Treating integer order PDE." << endl;
+   }

   // 3. Read the mesh from the given mesh file.
   Mesh mesh(mesh_file, 1, 1);
@@ -99,8 +156,8 @@ int main(int argc, char *argv[])
   }

   // 5. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new H1_FECollection(order, dim);
-   FiniteElementSpace fespace(&mesh, fec);
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec);
   cout << "Number of finite element unknowns: "
        << fespace.GetTrueVSize() << endl;

@@ -114,79 +171,234 @@ int main(int argc, char *argv[])
   }

   // 7. Define diffusion coefficient, load, and solution GridFunction.
-   ConstantCoefficient f(1.0);
+   auto func = [&alpha](const Vector &x)
+   {
+      double val = 1.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         val *= sin(M_PI*x(i));
+      }
+      return pow(x.Size()*pow(M_PI,2), alpha) * val;
+   };
+   FunctionCoefficient f(func);
   ConstantCoefficient one(1.0);
   GridFunction u(&fespace);
-   u = 0.;
+   GridFunction x(&fespace);
+   GridFunction g(&fespace);
+   u = 0.0;
+   x = 0.0;
+   g = 0.0;

   // 8. Prepare for visualization.
   char vishost[] = "localhost";
   int  visport   = 19916;
-   socketstream xout, uout;
-   ostringstream oss_x, oss_u;
-   if (visualization)
+
+   // 9. Set up the linear form b(.) for integer-order PDE solves.
+   LinearForm b(&fespace);
+   if (verification)
   {
-      xout.open(vishost, visport);
-      xout.precision(8);
-      uout.open(vishost, visport);
-      uout.precision(8);
+      // This statement is only relevant for the verification of the code. It
+      // uses a different f such that an analytic solution is known and easy
+      // to compare with the numerical one. The FPDE becomes:
+      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
+      // -> u(x,y) = sin(\pi x) sin(\pi y)
+      b.AddDomainIntegrator(new DomainLFIntegrator(f));
   }
-
-   for (int i = 0; i < coeffs.Size(); i++)
+   else
   {
-      // 9. Set up the linear form b(.) for integer-order PDE solve.
-      LinearForm b(&fespace);
-      ProductCoefficient cf(coeffs[i], f);
-      b.AddDomainIntegrator(new DomainLFIntegrator(cf));
-      b.Assemble();
+      b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   }
+   b.Assemble();

-      // 10. Define GridFunction for integer-order PDE solve.
-      GridFunction x(&fespace);
-      x = 0.0;
+   // ------------------------------------------------------------------------
+   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
+   // ------------------------------------------------------------------------

-      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
-      BilinearForm a(&fespace);
-      a.AddDomainIntegrator(new DiffusionIntegrator(one));
-      ConstantCoefficient c2(-poles[i]);
-      a.AddDomainIntegrator(new MassIntegrator(c2));
-      a.Assemble();
+   if (power_of_laplace > 0)
+   {
+      // 10.1 Compute Stiffnes Matrix
+      BilinearForm k(&fespace);
+      k.AddDomainIntegrator(new DiffusionIntegrator(one));
+      k.Assemble();

-      // 12. Assemble the bilinear form and the corresponding linear system.
-      OperatorPtr A;
+      // 10.2 Compute Mass Matrix
+      BilinearForm m(&fespace);
+      m.AddDomainIntegrator(new MassIntegrator(one));
+      m.Assemble();
+      SparseMatrix mass;
+      Array<int> empty;
+      m.FormSystemMatrix(empty, mass);
+
+      // 10.3 Form the system of equations
      Vector B, X;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+      OperatorPtr Op;
+      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
+      GSSmoother M((SparseMatrix&)(*Op));

-      // 13. Solve the linear system A X = B.
-      GSSmoother M((SparseMatrix&)(*A));
-
-      mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
-                << " u = " << coeffs[i] << " f " << endl;
-      PCG(*A, M, B, X, 3, 200, 1e-12, 0.0);
-
-      // 14. Recover the solution as a finite element grid function.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 15. Accumulate integer-order PDE solutions.
-      u+=x;
-
-      // 16. Send the solutions by socket to a GLVis server.
-      if (visualization)
+      mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
+                << " ( f ) " << endl;
+      for (int i = 0; i < power_of_laplace; i++)
      {
-         oss_x.str(""); oss_x.clear();
-         oss_x << "Solution of PDE -Δ u + " << -poles[i]
-               << " u = " << coeffs[i] << " f";
-         xout << "solution\n" << mesh << x
-              << "window_title '" << oss_x.str() << "'" << flush;
+         // 10.4 Solve the linear system Op X = B (N times).
+         PCG(*Op, M, B, X, 3, 300, 1e-12, 0.0);

-         oss_u.str(""); oss_u.clear();
-         oss_u << "Solution of fractional PDE -Δ^" << alpha
-               << " u = f";
-         uout << "solution\n" << mesh << u
-              << "window_title '" << oss_u.str() << "'" << flush;
+         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
+         if (i == power_of_laplace - 1)
+         {
+            // Needed for visualization and solution verification.
+            k.RecoverFEMSolution(X, b, g);
+            if (integer_order && verification)
+            {
+               // For an integer order PDE, g is also our solution u.
+               u+=g;
+            }
+            if (visualization)
+            {
+               socketstream fout;
+               ostringstream oss_f;
+               fout.open(vishost, visport);
+               fout.precision(8);
+               oss_f.str(""); oss_f.clear();
+               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
+                     << power_of_laplace
+                     << " g = f";
+               fout << "solution\n" << mesh << g
+                    << "window_title '" << oss_f.str() << "'" << flush;
+            }
+         }
+
+         // 10.6 Prepare for next iteration (primal / dual space)
+         mass.Mult(X, B);
+         X.SetSubVectorComplement(ess_tdof_list,0.0);
+      }
+
+      // 10.7 Extract solution for the next step. The b now corresponds to the
+      //      function g in the PDE.
+      const SparseMatrix * R = fespace.GetRestrictionMatrix();
+      if (R)
+      {
+         R->MultTranspose(B,b);
+      }
+      else
+      {
+         b = B;
      }
   }

-   // 17. Free the used memory.
-   delete fec;
+   // ------------------------------------------------------------------------
+   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
+   //     up the solutions.
+   // ------------------------------------------------------------------------
+   if (!integer_order)
+   {
+      // Setup visualization.
+      socketstream xout, uout;
+      ostringstream oss_x, oss_u;
+      if (visualization)
+      {
+         xout.open(vishost, visport);
+         xout.precision(8);
+         uout.open(vishost, visport);
+         uout.precision(8);
+      }
+      // Iterate over all expansion coefficient that contribute to the
+      // solution.
+      for (int i = 0; i < coeffs.Size(); i++)
+      {
+         mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                   << " u = " << coeffs[i] << " g " << endl;
+
+
+         // 11.1 Reset GridFunction for integer-order PDE solve.
+         x = 0.0;
+
+         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
+         BilinearForm a(&fespace);
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         ConstantCoefficient d_i(-poles[i]);
+         a.AddDomainIntegrator(new MassIntegrator(d_i));
+         a.Assemble();
+
+         // 11.3 Assemble the bilinear form and the corresponding linear system.
+         OperatorPtr A;
+         Vector B, X;
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+         // 11.4 Solve the linear system A X = B.
+         GSSmoother M((SparseMatrix&)(*A));
+
+         PCG(*A, M, B, X, 3, 300, 1e-12, 0.0);
+
+         // 11.5 Recover the solution as a finite element grid function.
+         a.RecoverFEMSolution(X, b, x);
+
+         // 11.6 Accumulate integer-order PDE solutions.
+         x *= coeffs[i];
+         u += x;
+
+         // 11.7 Send fractional PDE solution to a GLVis server.
+         if (visualization)
+         {
+            oss_x.str(""); oss_x.clear();
+            oss_x << "Step " << progress_steps
+                  << ": Solution of PDE -Δ u + " << -poles[i]
+                  << " u = " << coeffs[i] << " g";
+            xout << "solution\n" << mesh << x
+                 << "window_title '" << oss_x.str() << "'" << flush;
+
+            oss_u.str(""); oss_u.clear();
+            oss_u << "Step " << progress_steps + 1
+                  << ": Solution of fractional PDE (-Δ)^" << alpha
+                  << " u = f";
+            uout << "solution\n" << mesh << u
+                 << "window_title '" << oss_u.str() << "'"
+                 << flush;
+         }
+      }
+   }
+
+   // ------------------------------------------------------------------------
+   // 12. (optional) Verify the solution.
+   // ------------------------------------------------------------------------
+   if (verification)
+   {
+      auto solution = [] (const Vector &x)
+      {
+         double val = 1.0;
+         for (int i=0; i<x.Size(); i++)
+         {
+            val *= sin(M_PI*x(i));
+         }
+         return val;
+      };
+      FunctionCoefficient sol(solution);
+      double l2_error = u.ComputeL2Error(sol);
+
+      string analytic_solution,expected_mesh;
+      switch (dim)
+      {
+         case 1:
+            analytic_solution = "sin(π x)";
+            expected_mesh = "inline_segment.mesh";
+            break;
+         case 2:
+            analytic_solution = "sin(π x) sin(π y)";
+            expected_mesh = "inline_quad.mesh";
+            break;
+         default:
+            analytic_solution = "sin(π x) sin(π y) sin(π z)";
+            expected_mesh = "inline_hex.mesh";
+            break;
+      }
+
+      mfem::out << "\n" << string(80,'=')
+                << "\n\nSolution Verification in "<< dim << "D \n\n"
+                << "Analytic solution : " << analytic_solution << "\n"
+                << "Expected mesh     : " << expected_mesh <<"\n"
+                << "Your mesh         : " << mesh_file << "\n"
+                << "L2 error          : " << l2_error << "\n\n"
+                << string(80,'=') << endl;
+   }
+
   return 0;
 }
@@ -32,6 +32,7 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <string>

 using namespace std;
 using namespace mfem;
@@ -249,6 +250,13 @@ void PartialFractionExpansion(double scale, Array<double> & poles,
   coeffs.SetSize(psize);
   coeffs = scale;

+   // Note: C p(z)/q(z) = Σ_i c_i / (z - p_i) results in an system of equations
+   // where the N unknowns are the coefficients c_i. After multiplying the
+   // system with q(z), the coefficients c_i can be computed analytically by
+   // choosing N values for z. Choosing z_j = = p_j diagonalizes the system and
+   // one can obtain an analytic form for the c_i coefficients. The result is
+   // implemented in the code block below.
+
   for (int i=0; i<psize; i++)
   {
      double tmp_numer=1.0;
@@ -305,9 +313,12 @@ void ComputePartialFractionApproximation(double & alpha,
   if (print_warning)
   {
      mfem::out
-            << "\nMFEM is compiled without LAPACK.\nUsing precomputed values for PartialFractionApproximation. \n"
-            << "Only alpha = 0.33, 0.5, and 0.99 are available.\nThe default is alpha = 0.5."
-            << std::endl;
+            << "\n" << string(80, '=')
+            << "\nMFEM is compiled without LAPACK."
+            << "\nUsing precomputed values for PartialFractionApproximation."
+            << "\nOnly alpha = 0.33, 0.5, and 0.99 are available."
+            << "\nThe default is alpha = 0.5.\n" << string(80, '=') << "\n"
+            << endl;
   }
   const double eps = std::numeric_limits<double>::epsilon();

@@ -351,7 +362,7 @@ void ComputePartialFractionApproximation(double & alpha,

   if (print_warning)
   {
-      mfem::out << "Using precomputed values for alpha = "
+      mfem::out << "=> Using precomputed values for alpha = "
                << alpha << "\n" << std::endl;
   }

@@ -3,34 +3,63 @@
 // Compile with: make ex33p
 //
 // Sample runs:  mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 0.33 -o 2
+//               mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 4.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 1.4 -o 3
 //               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 0.99 -o 3
 //               mpirun -np 4 ex33p -m ../data/inline-quad.mesh -alpha 0.5 -o 3
-//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
+//               mpirun -np 4 ex33p -m ../data/amr-quad.mesh -alpha 1.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3 -r 2
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
 //               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
+//               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
+//
+// Verification runs:
+//    mpirun -np 4 ex33p -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
+
+//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
+//        for all alpha.
 //
 // Description:
 //
 //  In this example we solve the following fractional PDE with MFEM:
 //
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
 //
-//  To solve this FPDE, we rely on a rational approximation [2] of the normal
-//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
-//  boundary conditions). Namely, we first approximate the operator
+//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
+//  N is given by floor(α). By doing so, we obtain
 //
-//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
+//
+//  We first compute the right hand side by solving the integer order PDE
+//
+//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
+//
+//  The remaining FPDE is then given by
+//
+//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
+//
+//  We rely on a rational approximation [2] of the normal linear operator
+//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
+//  and (a-N) in (0,1). We approximate the operator
+//
+//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
 //
 //  where I is the L2-identity operator and the coefficients c_i and d_i
 //  are generated offline to a prescribed accuracy in a pre-processing step.
 //  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve N+1
+//  that this partial fractional expansion derives from. We then solve M+1
 //  independent integer-order PDEs,
 //
-//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
+//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
 //
 //  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
 //
-//    u ≈ Σ_{i=0}^N u_i.
+//    u ≈ Σ_{i=0}^M u_i.
+//
+//  (If alpha is an integer, we stop after the first PDE was solved.)
 //
 // References:
 //
@@ -47,6 +76,8 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <math.h>
+#include <string>

 #include "ex33.hpp"

@@ -65,9 +96,9 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int num_refs = 3;
-   bool visualization = true;
-   bool visualize_x = false;
   double alpha = 0.5;
+   bool visualization = true;
+   bool verification = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,12 +110,12 @@ int main(int argc, char *argv[])
                  "Number of uniform refinements");
   args.AddOption(&alpha, "-alpha", "--alpha",
                  "Fractional exponent");
-   args.AddOption(&visualize_x, "-vis_x", "--visualize_x", "-no-vis_x",
-                  "--no-visualization_x",
-                  "Enable or disable GLVis visualization of each integer-order PDE solution.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
-                  "Enable or disable GLVis visualization of the fractional PDE solution.");
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
+                  "--no-verification",
+                  "Use sinusoidal function (f) for analytic comparison.");
   args.Parse();
   if (!args.Good())
   {
@@ -97,61 +128,51 @@ int main(int argc, char *argv[])
   }

   Array<double> coeffs, poles;
+   int progress_steps = 1;

-   // 2. Compute the coefficients that define the integer-order PDEs.
-   ComputePartialFractionApproximation(alpha,coeffs,poles);
-
-   int num_par_solves;
-   int max_par_solves = max(1,num_procs/2);
-   for (num_par_solves=max_par_solves; num_par_solves>0; num_par_solves--)
+   // 2. Compute the rational expansion coefficients that define the
+   //    integer-order PDEs.
+   const int power_of_laplace = floor(alpha);
+   double exponent_to_approximate = alpha - power_of_laplace;
+   bool integer_order = false;
+   // Check if alpha is an integer or not.
+   if (abs(exponent_to_approximate) > 1e-12)
   {
-      if (num_procs%num_par_solves==0 && num_par_solves<coeffs.Size())
+      if (Mpi::Root())
      {
-         break;
+         mfem::out << "Approximating the fractional exponent "
+                   << exponent_to_approximate
+                   << endl;
+      }
+      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
+                                          poles);
+
+      // If the example is build without LAPACK, the exponent_to_approximate
+      // might be modified by the function call above.
+      alpha = exponent_to_approximate + power_of_laplace;
+   }
+   else
+   {
+      integer_order = true;
+      if (Mpi::Root())
+      {
+         mfem::out << "Treating integer order PDE." << endl;
      }
   }
-   if (num_par_solves == 1) {num_par_solves = num_procs;}

-   int solver_ranks = num_procs/num_par_solves;
-
-   // 3. Split the MPI communicator:
-   //    row_comm is used for parallel partition of the mesh
-   //    col_comm is used for independent integer-order solves
-   int row_color = myid / solver_ranks; // Determine color based on row
-   int col_color = myid % solver_ranks; // Determine color based on col
-
-   MPI_Comm row_comm, col_comm;
-   MPI_Comm_split(MPI_COMM_WORLD, row_color, myid, &row_comm);
-   MPI_Comm_split(MPI_COMM_WORLD, col_color, myid, &col_comm);
-
-   int row_rank, row_size, col_rank, col_size;
-   MPI_Comm_rank(row_comm, &row_rank);
-   MPI_Comm_size(row_comm, &row_size);
-   MPI_Comm_rank(col_comm, &col_rank);
-   MPI_Comm_size(col_comm, &col_size);
-
-   if (Mpi::Root())
-   {
-      mfem::out << "\nTotal number of MPI ranks = " << num_procs << endl;
-      mfem::out << "Number of independent parallel solves = " << col_size << endl;
-      mfem::out << "Number of MPI ranks within each solve = " << row_size
-                <<"\n" << endl;
-   }
-
-   // 4. Read the mesh from the given mesh file.
+   // 3. Read the mesh from the given mesh file.
   Mesh mesh(mesh_file, 1, 1);
   int dim = mesh.Dimension();

-   // 5. Refine the mesh to increase the resolution.
+   // 4. Refine the mesh to increase the resolution.
   for (int i = 0; i < num_refs; i++)
   {
      mesh.UniformRefinement();
   }
-
-   ParMesh pmesh(row_comm, mesh);
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
   mesh.Clear();

-   // 6. Define a finite element space on the mesh.
+   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(&pmesh, &fec);
   if (Mpi::Root())
@@ -160,7 +181,7 @@ int main(int argc, char *argv[])
           << fespace.GetTrueVSize() << endl;
   }

-   // 7. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   Array<int> ess_tdof_list;
   if (pmesh.bdr_attributes.Size())
   {
@@ -169,120 +190,250 @@ int main(int argc, char *argv[])
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 8. Define diffusion coefficient, load, and solution GridFunction.
-   ConstantCoefficient f(1.0);
+   // 7. Define diffusion coefficient, load, and solution GridFunction.
+   auto func = [&alpha](const Vector &x)
+   {
+      double val = 1.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         val *= sin(M_PI*x(i));
+      }
+      return pow(x.Size()*pow(M_PI,2), alpha) * val;
+   };
+   FunctionCoefficient f(func);
   ConstantCoefficient one(1.0);
   ParGridFunction u(&fespace);
   ParGridFunction x(&fespace);
+   ParGridFunction g(&fespace);
   u = 0.0;
+   x = 0.0;
+   g = 0.0;
+
+   // 8. Prepare for visualization.
+   char vishost[] = "localhost";
+   int  visport   = 19916;

   // 9. Set up the linear form b(.) for integer-order PDE solves.
   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(f));
+   if (verification)
+   {
+      // This statement is only relevant for the verification of the code. It
+      // uses a different f such that an analytic solution is known and easy
+      // to compare with the numerical one. The FPDE becomes:
+      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
+      // -> u(x,y) = sin(\pi x) sin(\pi y)
+      b.AddDomainIntegrator(new DomainLFIntegrator(f));
+   }
+   else
+   {
+      b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   }
   b.Assemble();

-   int my_coeff_size = max(coeffs.Size()/col_size,1);
-   int ibeg = col_rank*my_coeff_size;
-   if (ibeg + 2*my_coeff_size > coeffs.Size())
+   // ------------------------------------------------------------------------
+   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
+   // ------------------------------------------------------------------------
+
+   if (power_of_laplace > 0)
   {
-      my_coeff_size = coeffs.Size()-col_rank*my_coeff_size;
-   }
-   else if (ibeg > coeffs.Size() - 1)
-   {
-      my_coeff_size = 0;
-   }
+      // 10.1 Compute Stiffnes Matrix
+      ParBilinearForm k(&fespace);
+      k.AddDomainIntegrator(new DiffusionIntegrator(one));
+      k.Assemble();

-   int iend = ibeg+my_coeff_size;
+      // 10.2 Compute Mass Matrix
+      ParBilinearForm m(&fespace);
+      m.AddDomainIntegrator(new MassIntegrator(one));
+      m.Assemble();
+      HypreParMatrix mass;
+      Array<int> empty;
+      m.FormSystemMatrix(empty, mass);

-
-   for (int i = ibeg; i < iend; i++)
-   {
-      // 10. Reset GridFunction for integer-order PDE solve.
-      x = 0.0;
-
-      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
-      ParBilinearForm a(&fespace);
-      a.AddDomainIntegrator(new DiffusionIntegrator(one));
-      ConstantCoefficient d_i(-poles[i]);
-      a.AddDomainIntegrator(new MassIntegrator(d_i));
-      a.Assemble();
-
-      // 12. Assemble the bilinear form and the corresponding linear system.
-      OperatorPtr A;
+      // 10.3 Form the system of equations
      Vector B, X;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-      // 13. Solve the linear system A X = B.
-      HypreBoomerAMG * prec = new HypreBoomerAMG;
-      prec->SetPrintLevel(-1);
-
-      int print_level = (col_rank==0) ? 3 : 0;
-      if (Mpi::Root())
-      {
-         mfem::out << "\nMPI rank " <<  myid
-                   << ": Solving PDE -Δ u + " << -poles[i]
-                   << " u = " << coeffs[i] << " f " << endl;
-      }
-      CGSolver cg(row_comm);
+      OperatorPtr Op;
+      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
+      HypreBoomerAMG prec;
+      prec.SetPrintLevel(-1);
+      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-12);
      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(print_level);
-      cg.SetPreconditioner(*prec);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(prec);
+      cg.SetOperator(*Op);

-      // 14. Recover the solution as a finite element grid function.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 15. Accumulate integer-order PDE solutions.
-      x *= coeffs[i];
-      u += x;
-
-      // 16. Send integer-order PDE solutions to a GLVis server.
-      if (visualize_x)
+      if (Mpi::Root())
      {
-         if (col_rank > 0 && i < iend-1)
+         mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
+                   << " ( f ) " << endl;
+      }
+      for (int i = 0; i < power_of_laplace; i++)
+      {
+         // 10.4 Solve the linear system Op X = B (N times).
+         cg.Mult(B, X);
+         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
+         if (i == power_of_laplace - 1)
         {
-            MPI_Status status;
-            MPI_Recv(nullptr,0,MPI_INT, col_rank-1,0,col_comm,&status);
+            // Needed for visualization and solution verification.
+            k.RecoverFEMSolution(X, b, g);
+            if (integer_order && verification)
+            {
+               // For an integer order PDE, g is also our solution u.
+               u+=g;
+            }
+            if (visualization)
+            {
+               socketstream fout;
+               ostringstream oss_f;
+               fout.open(vishost, visport);
+               fout.precision(8);
+               oss_f.str(""); oss_f.clear();
+               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
+                     << power_of_laplace
+                     << " g = f";
+               fout << "parallel " << num_procs << " " << myid << "\n"
+                    << "solution\n" << pmesh << g
+                    << "window_title '" << oss_f.str() << "'" << flush;
+            }
         }
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream xout(vishost, visport);
+
+         // 10.6 Prepare for next iteration (primal / dual space)
+         mass.Mult(X, B);
+         X.SetSubVectorComplement(ess_tdof_list,0.0);
+      }
+
+      // 10.7 Extract solution for the next step. The b now corresponds to the
+      //      function g in the PDE.
+      const SparseMatrix* rm = fespace.GetRestrictionMatrix();
+      rm->MultTranspose(B, b);
+   }
+
+   // ------------------------------------------------------------------------
+   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
+   //     up the solutions.
+   // ------------------------------------------------------------------------
+   if (!integer_order)
+   {
+      // Setup visualization.
+      socketstream xout, uout;
+      ostringstream oss_x, oss_u;
+      if (visualization)
+      {
+         xout.open(vishost, visport);
         xout.precision(8);
-         ostringstream oss;
-         oss << "Solution of PDE -Δ u + " << -poles[i]
-             << " u = " << coeffs[i] << " f" ;
-         xout << "parallel " << row_size << " " << row_rank << "\n";
-         xout << "solution\n" << pmesh << x
-              << "window_title '" << oss.str() << "'" << flush;
-         if (col_rank < col_size-1)
+         uout.open(vishost, visport);
+         uout.precision(8);
+      }
+      // Iterate over all expansion coefficient that contribute to the
+      // solution.
+      for (int i = 0; i < coeffs.Size(); i++)
+      {
+         if (Mpi::Root())
         {
-            MPI_Send(nullptr,0,MPI_INT,col_rank+1,0,col_comm);
+            mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                      << " u = " << coeffs[i] << " g " << endl;
+         }
+
+         // 11.1 Reset GridFunction for integer-order PDE solve.
+         x = 0.0;
+
+         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
+         ParBilinearForm a(&fespace);
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         ConstantCoefficient d_i(-poles[i]);
+         a.AddDomainIntegrator(new MassIntegrator(d_i));
+         a.Assemble();
+
+         // 11.3 Assemble the bilinear form and the corresponding linear system.
+         OperatorPtr A;
+         Vector B, X;
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+         // 11.4 Solve the linear system A X = B.
+         HypreBoomerAMG prec;
+         prec.SetPrintLevel(-1);
+
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-12);
+         cg.SetMaxIter(2000);
+         cg.SetPrintLevel(3);
+         cg.SetPreconditioner(prec);
+         cg.SetOperator(*A);
+         cg.Mult(B, X);
+
+         // 11.5 Recover the solution as a finite element grid function.
+         a.RecoverFEMSolution(X, b, x);
+
+         // 11.6 Accumulate integer-order PDE solutions.
+         x *= coeffs[i];
+         u += x;
+
+         // 11.7 Send fractional PDE solution to a GLVis server.
+         if (visualization)
+         {
+            oss_x.str(""); oss_x.clear();
+            oss_x << "Step " << progress_steps
+                  << ": Solution of PDE -Δ u + " << -poles[i]
+                  << " u = " << coeffs[i] << " g";
+            xout << "parallel " << num_procs << " " << myid << "\n"
+                 << "solution\n" << pmesh << x
+                 << "window_title '" << oss_x.str() << "'" << flush;
+
+            oss_u.str(""); oss_u.clear();
+            oss_u << "Step " << progress_steps + 1
+                  << ": Solution of fractional PDE (-Δ)^" << alpha
+                  << " u = f";
+            uout << "parallel " << num_procs << " " << myid << "\n"
+                 << "solution\n" << pmesh << u
+                 << "window_title '" << oss_u.str() << "'"
+                 << flush;
         }
      }
   }

-   // 17. Accumulate for the fractional PDE solution
-   MPI_Allreduce(MPI_IN_PLACE, u.GetData(), u.Size(),
-                 MPI_DOUBLE, MPI_SUM,col_comm);
-
-   // 18. Send fractional PDE solution to a GLVis server.
-   if (visualization)
+   // ------------------------------------------------------------------------
+   // 12. (optional) Verify the solution.
+   // ------------------------------------------------------------------------
+   if (verification)
   {
-      if (col_rank == 0)
+      auto solution = [] (const Vector &x)
      {
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream uout(vishost, visport);
-         uout.precision(8);
-         ostringstream oss;
-         oss << "Solution of fractional PDE -Δ^" << alpha
-             << " u = f" ;
-         uout << "parallel " << row_size << " " << row_rank << "\n";
-         uout << "solution\n" << pmesh << u
-              << "window_title '" << oss.str() << "'" << flush;
+         double val = 1.0;
+         for (int i=0; i<x.Size(); i++)
+         {
+            val *= sin(M_PI*x(i));
+         }
+         return val;
+      };
+      FunctionCoefficient sol(solution);
+      double l2_error = u.ComputeL2Error(sol);
+
+      if (Mpi::Root())
+      {
+         string analytic_solution,expected_mesh;
+         switch (dim)
+         {
+            case 1:
+               analytic_solution = "sin(π x)";
+               expected_mesh = "inline_segment.mesh";
+               break;
+            case 2:
+               analytic_solution = "sin(π x) sin(π y)";
+               expected_mesh = "inline_quad.mesh";
+               break;
+            default:
+               analytic_solution = "sin(π x) sin(π y) sin(π z)";
+               expected_mesh = "inline_hex.mesh";
+               break;
+         }
+
+         mfem::out << "\n" << string(80,'=')
+                   << "\n\nSolution Verification in "<< dim << "D \n\n"
+                   << "Analytic solution : " << analytic_solution << "\n"
+                   << "Expected mesh     : " << expected_mesh <<"\n"
+                   << "Your mesh         : " << mesh_file << "\n"
+                   << "L2 error          : " << l2_error << "\n\n"
+                   << string(80,'=') << endl;
      }
   }

@@ -1,220 +0,0 @@
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <algorithm>
-
-using namespace std;
-using namespace mfem;
-
-
-
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   bool visualization = true;
-
-   int precision = 8;
-   cout.precision(precision);
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(cout);
-      return 1;
-   }
-   args.PrintOptions(cout);
-
-
-   Mesh mesh = Mesh::MakeCartesian2D(2,2,mfem::Element::QUADRILATERAL);
-   mesh.EnsureNCMesh();
-
-   Array<Table * > map_table;
-   // in case of a refinement we need the Transpose of the coarse2fine Table
-   // in case of a derefinement we need the coarse2fine table
-
-   int ref = 2;
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-
-   if (visualization)
-   {
-      socketstream sout(vishost, visport);
-      sout.precision(precision);
-      sout << "mesh\n" << mesh << flush;
-   }
-
-   Table temp;
-
-   Array<int> refinements;
-   refinements.Append(0);
-   refinements.Append(1);
-   refinements.Append(3);
-   mesh.GeneralRefinement(refinements);
-   if (visualization)
-   {
-      socketstream sout(vishost, visport);
-      sout.precision(precision);
-      sout << "mesh\n" << mesh << flush;
-   }
-   const CoarseFineTransformations & tr = mesh.GetRefinementTransforms();
-   tr.MakeCoarseToFineTable(temp);
-   map_table.Append(Transpose(temp));
-
-   refinements.SetSize(0);
-   refinements.Append(7);
-   mesh.GeneralRefinement(refinements);
-   if (visualization)
-   {
-      socketstream sout(vishost, visport);
-      sout.precision(precision);
-      sout << "mesh\n" << mesh << flush;
-   }
-   const CoarseFineTransformations & tr1 = mesh.GetRefinementTransforms();
-   tr1.MakeCoarseToFineTable(temp);
-   map_table.Append(Transpose(temp));
-
-
-   // // for (int i = 0; i<ref; i++)
-   // // {
-   // //    mesh.RandomRefinement(0.5);
-   // //    if (visualization)
-   // //    {
-   // //       socketstream sout(vishost, visport);
-   // //       sout.precision(precision);
-   // //       sout << "mesh\n" << mesh << flush;
-   // //    }
-   // //    const CoarseFineTransformations & tr = mesh.GetRefinementTransforms();
-   // //    tr.MakeCoarseToFineTable(temp);
-   // //    map_table.Append(Transpose(temp));
-   // // }
-
-   // // derefine
-   Vector errors(mesh.GetNE());
-   errors = 1;
-   errors[12] = 0.;  errors[14] = 0.;
-   errors[13] = 0.;  errors[15] = 0.;
-   // errors[21] = 0.; errors[18] = 0.;
-   // errors[19] = 0.; errors[20] = 0.;
-   mesh.DerefineByError(errors,0.3);
-
-   if (visualization)
-   {
-      socketstream sout1(vishost, visport);
-      sout1.precision(precision);
-      sout1 << "mesh\n" << mesh << flush;
-   }
-
-   const CoarseFineTransformations &tr2 = mesh.ncmesh->GetDerefinementTransforms();
-   map_table.Append(new Table);
-   tr2.MakeCoarseToFineTable(*map_table.Last());
-
-   for (int j = 0; j<mesh.GetNE(); j++)
-   {
-      // while the element depth > 0 for the element
-      mfem::out << "Refinement history for element: " << std::setw(4) << j << ": ";
-      int row = j;
-      for (int i = map_table.Size()-1 ; i>=0; i--)
-      {
-         row = map_table[i]->GetRow(row)[0];
-         if (i == 0)
-         {
-            mfem::out << std::setw(4) << row << endl;
-         }
-         else
-         {
-            mfem::out << std::setw(4) << row ;
-         }
-      }
-   }
-
-   // example ...
-   // create the maps from mesh 1 to mesh 3 (after one ref and 1 dref)
-   // 1. specify the newly created elements (either from ref or from dref)
-   // 2. mark the elements that are deleted
-   // 3. provide the map from the unmodified elements to their new numbers
-
-   // for the above scenario
-   Table & ref_table = *Transpose(*map_table[1]);
-   Table & dref_table = *Transpose(*map_table[2]);
-
-
-   Table * T = Mult(ref_table,dref_table);
-
-
-   mfem::out << "ref_table = " << endl;
-   ref_table.Print();
-
-   mfem::out << "dref_table = " << endl;
-   dref_table.Print();
-
-   mfem::out << "combined_table = " << endl;
-   T->Print();
-
-
-   Table * Tt = Transpose(*T);
-
-   Array<int> old_elems_map(T->Size()); // -1 if is to be deleted
-   Array<int> new_elems;
-
-   // loop though the old elements
-   for (int i = 0; i<T->Size(); i++)
-   {
-      // check row size
-      int n = T->RowSize(i);
-      int * row = T->GetRow(i);
-      if (n == 1)
-      {
-         // check the size of the transpose row
-         int m = Tt->RowSize(row[0]);
-         if (m == 1)
-         {
-            // the element is left unchanged
-            mfem::out << "Element number = " << i << " is mapped to element number = " <<
-                      row[0] << endl;
-            old_elems_map[i] = row[0];
-         }
-         else
-         {
-            mfem::out << "Element number = " << i <<
-                      " is derefined (deleted). Create new element: " << row[0] << endl;
-            old_elems_map[i] = -1;
-            new_elems.Append(row[0]);
-         }
-      }
-      else
-      {
-         mfem::out << "Element number = " << i <<
-                   " is refined (deleted). Create new elements = " ;
-         old_elems_map[i] = -1;
-         for (int j = 0; j<n; j++)
-         {
-            mfem::out << row[j];
-            new_elems.Append(row[j]);
-            if (j == n-1)
-            {
-               mfem::out << endl;
-            }
-            else
-            {
-               mfem::out << ", ";
-            }
-         }
-      }
-   }
-
-   // new_elems.Sort();
-   new_elems.Unique();
-
-   mfem::out << "elements map = " ; old_elems_map.Print(cout,old_elems_map.Size());
-   mfem::out << "new_elements = " ; new_elems.Print(cout,new_elems.Size());
-
-
-   return 0;
-}
-
@@ -58,15 +58,17 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
-  ceed/algebraic.cpp
-  ceed/full-assembly.cpp
-  ceed/solvers-atpmg.cpp
-  ceed/convection.cpp
-  ceed/diffusion.cpp
-  ceed/nlconvection.cpp
-  ceed/mass.cpp
-  ceed/operator.cpp
-  ceed/util.cpp
+  ceed/interface/basis.cpp
+  ceed/interface/restriction.cpp
+  ceed/interface/operator.cpp
+  ceed/interface/util.cpp
+  ceed/integrators/convection/convection.cpp
+  ceed/integrators/diffusion/diffusion.cpp
+  ceed/integrators/nlconvection/nlconvection.cpp
+  ceed/integrators/mass/mass.cpp
+  ceed/solvers/algebraic.cpp
+  ceed/solvers/full-assembly.cpp
+  ceed/solvers/solvers-atpmg.cpp
  linearform.cpp
  linearform_ext.cpp
  lininteg.cpp
@@ -128,7 +130,6 @@ set(SRCS
  tmop_amr.cpp
  gslib.cpp
  transfer.cpp
-  auxiliary.cpp
  )

 set(HDRS
@@ -159,17 +160,20 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
-  ceed/algebraic.hpp
-  ceed/full-assembly.hpp
-  ceed/solvers-atpmg.hpp
-  ceed/coefficient.hpp
-  ceed/convection.hpp
-  ceed/diffusion.hpp
-  ceed/integrator.hpp
-  ceed/mass.hpp
-  ceed/nlconvection.hpp
-  ceed/operator.hpp
-  ceed/util.hpp
+  ceed/interface/basis.hpp
+  ceed/interface/integrator.hpp
+  ceed/interface/interface.hpp
+  ceed/interface/operator.hpp
+  ceed/interface/restriction.hpp
+  ceed/interface/util.hpp
+  ceed/integrators/convection/convection.hpp
+  ceed/integrators/diffusion/diffusion.hpp
+  ceed/integrators/mass/mass.hpp
+  ceed/integrators/nlconvection/nlconvection.hpp
+  ceed/interface/coefficient.hpp
+  ceed/solvers/algebraic.hpp
+  ceed/solvers/full-assembly.hpp
+  ceed/solvers/solvers-atpmg.hpp
  linearform.hpp
  linearform_ext.hpp
  lininteg.hpp
@@ -207,7 +211,6 @@ set(HDRS
  tmop_amr.hpp
  gslib.hpp
  transfer.hpp
-  auxiliary.hpp
  )

 if (MFEM_USE_SIDRE)
@@ -1,426 +0,0 @@
-//                  MFEM Example 18 - Serial/Parallel Shared Code
-
-#include "mfem.hpp"
-using namespace std;
-using namespace mfem;
-
-
-// Time-dependent operator for the right-hand side of the ODE representing the
-// DG weak form.
-class EulerSystem : public TimeDependentOperator
-{
-private:
-   const int dim;
-   int num_equation;
-   double specific_heat_ratio;
-
-   FiniteElementSpace * vfes;
-   Operator &A;
-   SparseMatrix &Aflux;
-   std::vector<DenseMatrix> Me_inv;
-
-   mutable Vector state;
-   mutable DenseMatrix f;
-   mutable DenseTensor flux;
-   mutable Vector z;
-
-   void GetFlux(const DenseMatrix &state_, DenseTensor &flux_) const;
-
-public:
-   EulerSystem(FiniteElementSpace &vfes_,
-               Operator &A_, SparseMatrix &Aflux_,
-               double specific_heat_ratio_, int num_equation_);
-
-   virtual void Mult(const Vector &x, Vector &y) const;
-
-   virtual ~EulerSystem() {}
-
-};
-
-// Implements a simple Rusanov flux
-class RiemannSolver
-{
-public:
-   Vector flux1;
-   Vector flux2;
-
-   int num_equation;
-   double specific_heat_ratio;
-   RiemannSolver(double specific_heat_ratio_, int num_equation_);
-   double Eval(const Vector &state1, const Vector &state2,
-               const Vector &nor, Vector &flux);
-};
-
-// Interior face term: <F.n(u),[w]>
-class FaceIntegrator : public NonlinearFormIntegrator
-{
-public:
-   RiemannSolver rsolver;
-   int num_equation;
-   Vector shape1;
-   Vector shape2;
-   Vector funval1;
-   Vector funval2;
-   Vector nor;
-   Vector fluxN;
-
-   FaceIntegrator(RiemannSolver &rsolver_, const int dim, double num_equation_);
-
-   virtual void AssembleFaceVector(const FiniteElement &el1,
-                                   const FiniteElement &el2,
-                                   FaceElementTransformations &Tr,
-                                   const Vector &elfun, Vector &elvect);
-};
-
-// Implementation of class FE_Evolution
-EulerSystem::EulerSystem(FiniteElementSpace &vfes_,
-                         Operator &A_, SparseMatrix &Aflux_,
-                         double specific_heat_ratio_,
-                         int num_equation_)
-   : TimeDependentOperator(A_.Height()),
-     dim(vfes_.GetFE(0)->GetDim()),
-     vfes(&vfes_),
-     specific_heat_ratio(specific_heat_ratio_),
-     num_equation(num_equation_),
-     A(A_),
-     Aflux(Aflux_),
-     state(num_equation),
-     f(num_equation, dim),
-     flux(vfes->GetNDofs(), dim, num_equation),
-     z(A.Height())
-{
-   MassIntegrator mi;
-
-   for (int i = 0; i < vfes->GetNE(); i++) {
-      // Standard local assembly and inversion for energy mass matrices.
-      int dof = vfes->GetFE(i)->GetDof();
-      DenseMatrix Me(dof);
-      DenseMatrixInverse inv(&Me);
-
-      DenseMatrix inv_mi = DenseMatrix(vfes->GetFE(i)->GetDof(), vfes->GetFE(i)->GetDof());
-
-      mi.AssembleElementMatrix(*vfes->GetFE(i), *vfes->GetElementTransformation(i), Me);
-      inv.Factor();
-      inv.GetInverseMatrix(inv_mi);
-
-      Me_inv.push_back(inv_mi);
-   }
-}
-
-void EulerSystem::Mult(const Vector &x, Vector &y) const
-{
-   // 1. Create the vector z with the face terms -<F.n(u), [w]>.
-   A.Mult(x, z);
-
-   // 2. Add the element terms.
-   // i.  computing the flux approximately as a grid function by interpolating
-   //     at the solution nodes.
-   // ii. multiplying this grid function by a (constant) mixed bilinear form for
-   //     each of the num_equation, computing (F(u), grad(w)) for each equation.
-
-   DenseMatrix xmat(x.GetData(), vfes->GetNDofs(), num_equation);
-   GetFlux(xmat, flux);
-
-   for (int k = 0; k < num_equation; k++) {
-      Vector fk(flux(k).GetData(), dim * vfes->GetNDofs());
-      Vector zk(z.GetData() + k * vfes->GetNDofs(), vfes->GetNDofs());
-      Aflux.AddMult(fk, zk);
-   }
-
-   // 3. Multiply element-wise by the inverse mass matrices.
-   Vector zval;
-   Array<int> vdofs;
-
-   for (int i = 0; i < vfes->GetNE(); i++) {
-      int dof = vfes->GetFE(i)->GetDof();
-      DenseMatrix zmat, ymat(dof, num_equation);
-
-      // Return the vdofs ordered byNODES
-      vfes->GetElementVDofs(i, vdofs);
-      z.GetSubVector(vdofs, zval);
-      zmat.UseExternalData(zval.GetData(), dof, num_equation);
-      mfem::Mult(Me_inv[i], zmat, ymat);
-      y.SetSubVector(vdofs, ymat.GetData());
-   }
-}
-
-// Physicality check (at end)
-bool StateIsPhysical(const Vector &state, const int dim);
-
-// Pressure (EOS) computation
-inline double ComputePressure(const Vector &state, int num_equation,
-                              double specific_heat_ratio)
-{
-   const int udim = num_equation - 2;
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, udim);
-   const double den_energy = state(num_equation - 1);
-
-   double den_vel2 = 0;
-   for (int d = 0; d < udim; d++) {
-      den_vel2 += den_vel(d)*den_vel(d);
-   }
-   den_vel2 /= den;
-
-   return (specific_heat_ratio-1.0)*(den_energy - 0.5*den_vel2);
-}
-
-// Compute the vector flux F(u)
-void ComputeFlux(const Vector &state, int dim, DenseMatrix &flux, 
-                 double specific_heat_ratio, int num_equation)
-{
-   const int udim = num_equation - 2;
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, udim);
-   const double den_energy = state(num_equation - 1);
-
-   MFEM_ASSERT(StateIsPhysical(state, dim), "");
-
-   const double pres = ComputePressure(state, num_equation, specific_heat_ratio);
-   const double H = (den_energy + pres)/den;
-
-   // Hard-code quasi-1D cases
-   if (num_equation == 3) {
-      // Set x-flux
-      flux(0, 0) = den_vel(0);
-      flux(1, 0) = den_vel(0)*den_vel(0)/den + pres;
-      flux(2, 0) = den_vel(0)*H;
-
-      // Zero other components
-      for (int d = 1; d < dim; d++) {
-         for (int eq = 0; eq < num_equation; eq++){
-            flux(eq, d) = 0.0;
-         }
-      }
-   }
-   else {
-      MFEM_ASSERT(num_equation == dim + 2, "2D/3D solutions must be of size dim+2.")
-      for (int d = 0; d < dim; d++) {
-         flux(0, d) = den_vel(d);
-         for (int i = 0; i < dim; i++) {
-            flux(1+i, d) = den_vel(i) * den_vel(d) / den;
-         }
-         flux(1+d, d) += pres;
-      }
-      for (int d = 0; d < dim; d++) {
-         flux(1+dim, d) = den_vel(d) * H;
-      }
-   }
-}
-
-// Compute the scalar F(u).n
-void ComputeFluxDotN(const Vector &state, const Vector &nor, Vector &fluxN, 
-                     double specific_heat_ratio, int num_equation)
-{
-   const int udim = num_equation - 2;
-   // NOTE: nor in general is not a unit normal
-   const int dim = nor.Size();
-   MFEM_ASSERT(StateIsPhysical(state, dim), "");
-
-   DenseMatrix flux = DenseMatrix(num_equation, dim);
-   ComputeFlux(state, dim, flux, specific_heat_ratio, num_equation);
-
-   for (int i = 0; i < num_equation; i++) {
-      fluxN(i) = 0.0;
-      for (int d = 0; d < dim; d++) {
-         fluxN(i) += nor(d)*flux(i,d);
-      }
-   }
-}
-
-// Compute the maximum characteristic speed.
-inline double ComputeMaxCharSpeed(const Vector &state, const int dim,
-                                  double specific_heat_ratio, int num_equation)
-{
-   const int udim = num_equation - 2;
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, udim);
-
-   double den_vel2 = 0;
-   for (int d = 0; d < udim; d++) {
-      den_vel2 += den_vel(d)*den_vel(d);
-   }
-   den_vel2 /= den;
-
-   const double pres = ComputePressure(state, num_equation, specific_heat_ratio);
-   const double sound = sqrt(specific_heat_ratio*pres/den);
-   const double vel = sqrt(den_vel2/den);
-
-   return vel + sound;
-}
-
-// Compute the flux at solution nodes.
-void EulerSystem::GetFlux(const DenseMatrix &x_, DenseTensor &flux_) const
-{
-   const int flux_dof = flux_.SizeI();
-   const int flux_dim = flux_.SizeJ();
-
-   for (int i = 0; i < flux_dof; i++) {
-      for (int k = 0; k < num_equation; k++) {
-         state(k) = x_(i, k);
-      }
-
-      ComputeFlux(state, flux_dim, f, specific_heat_ratio, num_equation);
-
-      for (int d = 0; d < flux_dim; d++) {
-         for (int k = 0; k < num_equation; k++) {
-            flux_(i, d, k) = f(k, d);
-         }
-      }
-   }
-}
-
-// Implementation of class RiemannSolver
-RiemannSolver::RiemannSolver(double specific_heat_ratio_, int num_equation_) :
-   specific_heat_ratio(specific_heat_ratio_),
-   num_equation(num_equation_),
-   flux1(num_equation_),
-   flux2(num_equation_) { }
-
-double RiemannSolver::Eval(const Vector &state1, const Vector &state2,
-                           const Vector &nor, Vector &flux)
-{
-   // NOTE: nor in general is not a unit normal
-   const int dim = nor.Size();
-
-   MFEM_ASSERT(StateIsPhysical(state1, dim), "");
-   MFEM_ASSERT(StateIsPhysical(state2, dim), "");
-
-   const double maxE1 = ComputeMaxCharSpeed(state1, dim, specific_heat_ratio, num_equation);
-   const double maxE2 = ComputeMaxCharSpeed(state2, dim, specific_heat_ratio, num_equation);
-
-   const double maxE = max(maxE1, maxE2);
-
-   ComputeFluxDotN(state1, nor, flux1, specific_heat_ratio, num_equation);
-   ComputeFluxDotN(state2, nor, flux2, specific_heat_ratio, num_equation);
-
-   double normag = 0;
-   for (int i = 0; i < dim; i++) {
-      normag += nor(i) * nor(i);
-   }
-   normag = sqrt(normag);
-
-   for (int i = 0; i < num_equation; i++) {
-      flux(i) = 0.5 * (flux1(i) + flux2(i))
-                - 0.5 * maxE * (state2(i) - state1(i)) * normag;
-   }
-
-   return maxE;
-}
-
-
-// Implementation of class FaceIntegrator
-FaceIntegrator::FaceIntegrator(RiemannSolver &rsolver_, const int dim, double num_equation_) :
-   rsolver(rsolver_),
-   num_equation(num_equation_),
-   funval1(num_equation),
-   funval2(num_equation),
-   nor(dim),
-   fluxN(num_equation) { }
-
-void FaceIntegrator::AssembleFaceVector(const FiniteElement &el1,
-                                        const FiniteElement &el2,
-                                        FaceElementTransformations &Tr,
-                                        const Vector &elfun, Vector &elvect)
-{
-   // Compute the term <F.n(u),[w]> on the interior faces.
-   const int dof1 = el1.GetDof();
-   const int dof2 = el2.GetDof();
-
-   shape1.SetSize(dof1);
-   shape2.SetSize(dof2);
-
-   elvect.SetSize((dof1 + dof2) * num_equation);
-   elvect = 0.0;
-
-   DenseMatrix elfun1_mat(elfun.GetData(), dof1, num_equation);
-   DenseMatrix elfun2_mat(elfun.GetData() + dof1 * num_equation, dof2,
-                          num_equation);
-
-   DenseMatrix elvect1_mat(elvect.GetData(), dof1, num_equation);
-   DenseMatrix elvect2_mat(elvect.GetData() + dof1 * num_equation, dof2,
-                           num_equation);
-
-   // Integration order calculation from DGTraceIntegrator
-   int intorder;
-   if (Tr.Elem2No >= 0)
-      intorder = (min(Tr.Elem1->OrderW(), Tr.Elem2->OrderW()) +
-                  2*max(el1.GetOrder(), el2.GetOrder()));
-   else {
-      intorder = Tr.Elem1->OrderW() + 2*el1.GetOrder();
-   }
-
-   if (el1.Space() == FunctionSpace::Pk) {
-      intorder++;
-   }
-
-   const IntegrationRule *ir = &IntRules.Get(Tr.GetGeometryType(), intorder);
-   for (int i = 0; i < ir->GetNPoints(); i++) {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions on both elements at the face
-      el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
-      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-
-      // Interpolate elfun at the point
-      elfun1_mat.MultTranspose(shape1, funval1);
-      elfun2_mat.MultTranspose(shape2, funval2);
-
-      // Get the normal vector and the flux on the face
-      CalcOrtho(Tr.Jacobian(), nor);
-      const double mcs = rsolver.Eval(funval1, funval2, nor, fluxN);
-
-      fluxN *= ip.weight;
-      for (int k = 0; k < num_equation; k++) {
-         for (int s = 0; s < dof1; s++) {
-            elvect1_mat(s, k) -= fluxN(k) * shape1(s);
-         }
-         for (int s = 0; s < dof2; s++) {
-            elvect2_mat(s, k) += fluxN(k) * shape2(s);
-         }
-      }
-   }
-}
-
-// Check that the state is physical - enabled in debug mode
-bool StateIsPhysical(const Vector &state, const int dim)
-{
-   const double den = state(0);
-   const Vector den_vel(state.GetData() + 1, dim);
-   const double den_energy = state(1 + dim);
-
-   if (den < 0) {
-      cout << "Negative density: ";
-      for (int i = 0; i < state.Size(); i++) {
-         cout << state(i) << " ";
-      }
-      cout << endl;
-      return false;
-   }
-   if (den_energy <= 0) {
-      cout << "Negative energy: ";
-      for (int i = 0; i < state.Size(); i++) {
-         cout << state(i) << " ";
-      }
-      cout << endl;
-      return false;
-   }
-
-   double den_vel2 = 0;
-   for (int i = 0; i < dim; i++) { den_vel2 += den_vel(i) * den_vel(i); }
-   den_vel2 /= den;
-
-   const double int_energy = den_energy - 0.5 * den_vel2;
-
-   if (int_energy <= 0) {
-      cout << "Negative internal energy: " << int_energy << ", state: ";
-      for (int i = 0; i < state.Size(); i++) {
-         cout << state(i) << " ";
-      }
-      cout << endl;
-      return false;
-   }
-   return true;
-}
@@ -305,38 +305,66 @@ public:
   /// Finalizes the matrix initialization.
   virtual void Finalize(int skip_zeros = 1);

-   /// Returns a const reference to the sparse matrix.
+   /** @brief Returns a const reference to the sparse matrix:  \f$ M \f$
+
+       This will fail if HasSpMat() is false. */
   const SparseMatrix &SpMat() const
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }

-   /// Returns a reference to the sparse matrix:  \f$ M \f$
+   /** @brief Returns a reference to the sparse matrix:  \f$ M \f$
+
+       This will fail if HasSpMat() is false. */
   SparseMatrix &SpMat()
   {
      MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
      return *mat;
   }

+   /** @brief Returns true if the sparse matrix is not null, false otherwise.
+
+       @sa SpMat(). */
+   bool HasSpMat()
+   {
+      return mat != nullptr;
+   }
+
+
   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
        to it.  Used for transfering ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

-   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
+       \f$ M_e \f$
+
+       This will fail if HasSpMatElim() is false. */
   const SparseMatrix &SpMatElim() const
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
      return *mat_e;
   }

-   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
+   /** @brief Returns a reference to the sparse matrix of eliminated b.c.:
+       \f$ M_e \f$
+
+       This will fail if HasSpMatElim() is false. */
   SparseMatrix &SpMatElim()
   {
      MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
      return *mat_e;
   }

+   /**  @brief Returns true if the sparse matrix of eliminated b.c.s is not null,
+        false otherwise.
+
+        @sa SpMatElim(). */
+   bool HasSpMatElim()
+   {
+      return mat_e != nullptr;
+   }
+
   /// Adds new Domain Integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);
   /// Adds new Domain Integrator restricted to certain elements specified by
@@ -410,6 +438,14 @@ public:
   virtual const Operator *GetOutputRestriction() const
   { return GetRestriction(); }

+   /// @brief Compute serial RAP operator and store it in @a A as a SparseMatrix.
+   void SerialRAP(OperatorHandle &A)
+   {
+      MFEM_ASSERT(mat, "SerialRAP requires the SparseMatrix to be assembled.");
+      ConformingAssemble();
+      A.Reset(mat, false);
+   }
+
   /** @brief Form the linear system A X = B, corresponding to this bilinear
       form and the linear form @a b(.). */
   /** This method applies any necessary transformations to the linear system
@@ -16,7 +16,7 @@
 #include "bilinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pgridfunc.hpp"
-#include "ceed/util.hpp"
+#include "ceed/interface/util.hpp"

 namespace mfem
 {
@@ -251,6 +251,7 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)

 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 {
+   if ( Device::Allows(Backend::CEED_MASK) ) { return; }
   ElementDofOrdering ordering = UsesTensorBasis(*a->FESpace())?
                                 ElementDofOrdering::LEXICOGRAPHIC:
                                 ElementDofOrdering::NATIVE;
@@ -956,6 +957,57 @@ void FABilinearFormExtension::Assemble()
   }
 }

+
+void FABilinearFormExtension::RAP(OperatorHandle &A)
+{
+#ifdef MFEM_USE_MPI
+   if ( auto pa = dynamic_cast<ParBilinearForm*>(a) )
+   {
+      pa->ParallelRAP(*pa->mat, A);
+   }
+   else
+#endif
+   {
+      a->SerialRAP(A);
+   }
+}
+
+void FABilinearFormExtension::EliminateBC(const Array<int> &ess_dofs,
+                                          OperatorHandle &A)
+{
+#ifdef MFEM_USE_MPI
+   if ( dynamic_cast<ParBilinearForm*>(a) )
+   {
+      A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
+                                          DiagonalPolicy::DIAG_ONE);
+   }
+   else
+#endif
+   {
+      A.As<SparseMatrix>()->EliminateBC(ess_dofs,
+                                        DiagonalPolicy::DIAG_ONE);
+   }
+}
+
+void FABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_dofs,
+                                               OperatorHandle &A)
+{
+   RAP(A);
+   EliminateBC(ess_dofs, A);
+}
+
+void FABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                               Vector &x, Vector &b,
+                                               OperatorHandle &A,
+                                               Vector &X, Vector &B,
+                                               int copy_interior)
+{
+   Operator *A_out;
+   Operator::FormLinearSystem(ess_tdof_list, x, b, A_out, X, B, copy_interior);
+   delete A_out;
+   FormSystemMatrix(ess_tdof_list, A);
+}
+
 void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
 {
 #ifdef MFEM_USE_MPI
@@ -125,6 +125,15 @@ public:
   FABilinearFormExtension(BilinearForm *form);

   void Assemble();
+   void RAP(OperatorHandle &A);
+   /** @note Always does `DIAG_ONE` policy to be consistent with
+       `Operator::FormConstrainedSystemOperator`. */
+   void EliminateBC(const Array<int> &ess_dofs, OperatorHandle &A);
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0);
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;

@@ -1672,7 +1672,8 @@ void VectorFEDivergenceIntegrator::AssembleElementMatrix2(
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
      trial_fe.CalcDivShape(ip, divshape);
-      test_fe.CalcShape(ip, shape);
+      Trans.SetIntPoint(&ip);
+      test_fe.CalcPhysShape(Trans, shape);
      double w = ip.weight;
      if (Q)
      {
@@ -15,6 +15,7 @@
 #include "../config/config.hpp"
 #include "nonlininteg.hpp"
 #include "fespace.hpp"
+#include "ceed/interface/util.hpp"

 namespace mfem
 {
@@ -2397,8 +2398,9 @@ public:
    scalar function given by FiniteElement through standard transformation.
    Here, u is the trial function and p is the test function.

-    Note: the element matrix returned by AssembleElementMatrix2 does NOT depend
-    on the ElementTransformation Trans. */
+    Note: if the test space does not have map type INTEGRAL, then the element
+    matrix returned by AssembleElementMatrix2 will not depend on the
+    ElementTransformation Trans. */
 class VectorFEDivergenceIntegrator : public BilinearFormIntegrator
 {
 protected:
@@ -2735,7 +2737,8 @@ private:

 public:
   DivDivIntegrator() { Q = NULL; }
-   DivDivIntegrator(Coefficient &q) : Q(&q) { }
+   DivDivIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q) { }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/convection.hpp"
+#include "ceed/integrators/convection/convection.hpp"

 using namespace std;

@@ -30,7 +30,16 @@ void ConvectionIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFConvectionIntegrator(fes, *ir, Q, alpha);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFConvectionIntegrator(*this, fes, Q, alpha);
+      }
+      else
+      {
+         ceedOp = new ceed::MFConvectionIntegrator(fes, *ir, Q, alpha);
+      }
      return;
   }
   MFEM_ABORT("Error: ConvectionIntegrator::AssembleMF only implemented with"
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/convection.hpp"
+#include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"

 namespace mfem
@@ -1386,7 +1386,16 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAConvectionIntegrator(*this, fes, Q, alpha);
+      }
+      else
+      {
+         ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
+      }
      return;
   }
   const int dims = el.GetDim();
@@ -1497,6 +1506,7 @@ static void PAConvectionApply(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPAConvectionApply3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
         case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
         case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
         case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -1548,6 +1558,7 @@ static void PAConvectionApplyT(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPAConvectionApplyT3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
         case 0x23: return SmemPAConvectionApplyT3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
         case 0x24: return SmemPAConvectionApplyT3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
         case 0x26: return SmemPAConvectionApplyT3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -136,6 +136,9 @@ static void PADGTraceSetup(const int dim,

 void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
 {
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+
   nf = fes.GetNFbyType(type);
   if (nf==0) { return; }
   // Assumes tensor-product elements
@@ -153,7 +156,7 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   geom = mesh->GetFaceGeometricFactors(
             *ir,
             FaceGeometricFactors::DETERMINANTS |
-             FaceGeometricFactors::NORMALS, type);
+             FaceGeometricFactors::NORMALS, type, mt);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
@@ -695,6 +698,7 @@ static void PADGTraceApply(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPADGTraceApply3D<2,2,1>(NF,B,Bt,op,x,y);
         case 0x23: return SmemPADGTraceApply3D<2,3,1>(NF,B,Bt,op,x,y);
         case 0x34: return SmemPADGTraceApply3D<3,4,2>(NF,B,Bt,op,x,y);
         case 0x45: return SmemPADGTraceApply3D<4,5,2>(NF,B,Bt,op,x,y);
@@ -1124,6 +1128,7 @@ static void PADGTraceApplyTranspose(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPADGTraceApplyTranspose3D<2,2>(NF,B,Bt,op,x,y);
         case 0x23: return SmemPADGTraceApplyTranspose3D<2,3>(NF,B,Bt,op,x,y);
         case 0x34: return SmemPADGTraceApplyTranspose3D<3,4>(NF,B,Bt,op,x,y);
         case 0x45: return SmemPADGTraceApplyTranspose3D<4,5>(NF,B,Bt,op,x,y);
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -33,7 +33,16 @@ void DiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
      MFEM_VERIFY(!VQ && !MQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
-      ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFDiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: DiffusionIntegrator::AssembleMF only implemented with"
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -271,18 +271,21 @@ void PADiffusionSetup3D(const int Q1D,
                  D(qx,qy,qz,1,e) = D12; // 1,2
                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3

-                  const double D21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);

                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);

-                  D(qx,qy,qz,3,e) = symmetric ? D22 : D21; // 2,2 or 2,1
                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3

-                  if (!symmetric)
+                  if (symmetric)
                  {
+                     D(qx,qy,qz,3,e) = D22; // 2,2
+                  }
+                  else
+                  {
+                     D(qx,qy,qz,3,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
                     D(qx,qy,qz,8,e) = D33; // 3,3
@@ -365,7 +368,16 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
      MFEM_VERIFY(!VQ && !MQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
-      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   const int dims = el.GetDim();
@@ -24,18 +24,20 @@ namespace mfem

 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
                   Vector &coeff_,
                   Vector &op)
 {
+   const bool symmetric = (coeffDim != 4);
   const int NQ = Q1D*Q1D;
   auto W = w.Read();

   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto coeff = Reshape(coeff_.Read(), NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 3, NE);
+   auto C = Reshape(coeff_.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 3 : 4, NE);

   MFEM_FORALL(e, NE,
   {
@@ -45,28 +47,60 @@ void PAHdivSetup2D(const int Q1D,
         const double J21 = J(q,1,0,e);
         const double J12 = J(q,0,1,e);
         const double J22 = J(q,1,1,e);
-         const double c_detJ = W[q] * coeff(q, e) / ((J11*J22)-(J21*J12));
-         // (c/detJ) J^T J
-         y(q,0,e) = c_detJ * (J11*J11 + J21*J21); // 1,1
-         y(q,1,e) = c_detJ * (J11*J12 + J21*J22); // 1,2
-         y(q,2,e) = c_detJ * (J12*J12 + J22*J22); // 2,2
+         const double c_detJ = W[q] / ((J11*J22)-(J21*J12));
+
+         // (1/detJ) J^T C J
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
+         {
+            const double C11 = C(0,q,e);
+            const double C12 = C(1,q,e);
+            const double C21 = symmetric ? C12 : C(2,q,e);
+            const double C22 = symmetric ? C(2,q,e) : C(3,q,e);
+            const double R11 = C11*J11 + C12*J21;
+            const double R21 = C21*J11 + C22*J21;
+            const double R12 = C11*J12 + C12*J22;
+            const double R22 = C21*J12 + C22*J22;
+
+            y(q,0,e) = c_detJ * (J11*R11 + J21*R21); // 1,1
+            y(q,1,e) = c_detJ * (J11*R12 + J21*R22); // 1,2
+
+            if (symmetric)
+            {
+               y(q,2,e) = c_detJ * (J12*R12 + J22*R22); // 2,2
+            }
+            else
+            {
+               y(q,2,e) = c_detJ * (J12*R11 + J22*R21); // 2,1
+               y(q,3,e) = c_detJ * (J12*R12 + J22*R22); // 2,2
+            }
+         }
+         else // Vector or scalar coefficient
+         {
+            const double C1 = C(0,q,e);
+            const double C2 = (coeffDim == 2 ? C(1,q,e) : C1);
+            y(q,0,e) = c_detJ * (J11*C1*J11 + J21*C2*J21); // 1,1
+            y(q,1,e) = c_detJ * (J11*C1*J12 + J21*C2*J22); // 1,2
+            y(q,2,e) = c_detJ * (J12*C1*J12 + J22*C2*J22); // 2,2
+         }
      }
   });
 }

 // PA H(div) Mass Assemble 3D kernel
 void PAHdivSetup3D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
                   Vector &coeff_,
                   Vector &op)
 {
+   const bool symmetric = (coeffDim != 9);
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto coeff = Reshape(coeff_.Read(), NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 6, NE);
+   auto C = Reshape(coeff_.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 6 : 9, NE);

   MFEM_FORALL(e, NE,
   {
@@ -84,14 +118,58 @@ void PAHdivSetup3D(const int Q1D,
         const double detJ = J11 * (J22 * J33 - J32 * J23) -
         /* */               J21 * (J12 * J33 - J32 * J13) +
         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double c_detJ = W[q] * coeff(q, e) / detJ;
-         // (c/detJ) J^T J
-         y(q,0,e) = c_detJ * (J11*J11 + J21*J21 + J31*J31); // 1,1
-         y(q,1,e) = c_detJ * (J12*J11 + J22*J21 + J32*J31); // 2,1
-         y(q,2,e) = c_detJ * (J13*J11 + J23*J21 + J33*J31); // 3,1
-         y(q,3,e) = c_detJ * (J12*J12 + J22*J22 + J32*J32); // 2,2
-         y(q,4,e) = c_detJ * (J13*J12 + J23*J22 + J33*J32); // 3,2
-         y(q,5,e) = c_detJ * (J13*J13 + J23*J23 + J33*J33); // 3,3
+         const double c_detJ = W[q] / detJ;
+
+         // (1/detJ) J^T C J
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         {
+            double M[3][3];
+            M[0][0] = C(0, q, e);
+            M[0][1] = C(1, q, e);
+            M[0][2] = C(2, q, e);
+            M[1][0] = (!symmetric) ? C(3, q, e) : M[0][1];
+            M[1][1] = (!symmetric) ? C(4, q, e) : C(3, q, e);
+            M[1][2] = (!symmetric) ? C(5, q, e) : C(4, q, e);
+            M[2][0] = (!symmetric) ? C(6, q, e) : M[0][2];
+            M[2][1] = (!symmetric) ? C(7, q, e) : M[1][2];
+            M[2][2] = (!symmetric) ? C(8, q, e) : C(5, q, e);
+
+            int idx = 0;
+            for (int i=0; i<3; ++i)
+               for (int j = (symmetric ? i : 0); j<3; ++j)
+               {
+                  y(q,idx,e) = 0.0;
+                  for (int k=0; k<3; ++k)
+                  {
+                     double MJ_kj = 0.0;
+                     for (int l=0; l<3; ++l)
+                     {
+                        MJ_kj += M[k][l] * J(q,l,j,e);
+                     }
+
+                     y(q,idx,e) += J(q,k,i,e) * MJ_kj;
+                  }
+
+                  y(q,idx,e) *= c_detJ;
+                  idx++;
+               }
+         }
+         else  // Vector or scalar coefficient version
+         {
+            int idx = 0;
+            for (int i=0; i<3; ++i)
+               for (int j=i; j<3; ++j)
+               {
+                  y(q,idx,e) = 0.0;
+                  for (int k=0; k<3; ++k)
+                  {
+                     y(q,idx,e) += J(q,k,i,e) * C(coeffDim == 3 ? k : 0, q, e) * J(q,k,j,e);
+                  }
+
+                  y(q,idx,e) *= c_detJ;
+                  idx++;
+               }
+         }
      }
   });
 }
@@ -99,6 +177,7 @@ void PAHdivSetup3D(const int Q1D,
 void PAHdivMassApply2D(const int D1D,
                       const int Q1D,
                       const int NE,
+                       const bool symmetric,
                       const Array<double> &Bo_,
                       const Array<double> &Bc_,
                       const Array<double> &Bot_,
@@ -115,7 +194,7 @@ void PAHdivMassApply2D(const int D1D,
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
   auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
   auto Bct = Reshape(Bct_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 3, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
   auto x = Reshape(x_.Read(), 2*(D1D-1)*D1D, NE);
   auto y = Reshape(y_.ReadWrite(), 2*(D1D-1)*D1D, NE);

@@ -178,11 +257,12 @@ void PAHdivMassApply2D(const int D1D,
         {
            const double O11 = op(qx,qy,0,e);
            const double O12 = op(qx,qy,1,e);
-            const double O22 = op(qx,qy,2,e);
+            const double O21 = symmetric ? O12 : op(qx,qy,2,e);
+            const double O22 = symmetric ? op(qx,qy,2,e) : op(qx,qy,3,e);
            const double massX = mass[qy][qx][0];
            const double massY = mass[qy][qx][1];
            mass[qy][qx][0] = (O11*massX)+(O12*massY);
-            mass[qy][qx][1] = (O12*massX)+(O22*massY);
+            mass[qy][qx][1] = (O21*massX)+(O22*massY);
         }
      }

@@ -225,9 +305,179 @@ void PAHdivMassApply2D(const int D1D,
   }); // end of element loop
 }

+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAHdivMassApply2D(const int NE,
+                           const bool symmetric,
+                           const Array<double> &Bo_,
+                           const Array<double> &Bc_,
+                           const Array<double> &Bot_,
+                           const Array<double> &Bct_,
+                           const Vector &op_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(Bot_);
+   MFEM_CONTRACT_VAR(Bct_);
+
+   static constexpr int VDIM = 2;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
+   const auto x = Reshape(x_.Read(), D1D*(D1D-1), VDIM, NE);
+   auto y = y_.ReadWrite();
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, VDIM,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED double smo[MQ1*(MD1-1)];
+      DeviceMatrix Bo(smo, D1D-1, Q1D);
+
+      MFEM_SHARED double smc[MQ1*MD1];
+      DeviceMatrix Bc(smc, D1D, Q1D);
+
+      MFEM_SHARED double sm0[VDIM*MDQ*MDQ];
+      MFEM_SHARED double sm1[VDIM*MDQ*MDQ];
+      DeviceMatrix X(sm0, D1D*(D1D-1), VDIM);
+      DeviceCube QD(sm1, Q1D, D1D, VDIM);
+      DeviceCube QQ(sm0, Q1D, Q1D, VDIM);
+
+      // Load X, Bo and Bc into shared memory
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               if (qx < D1D && dy < (D1D-1)) { X(qx + dy*D1D,vd) = x(qx+dy*D1D,vd,e); }
+               if (tidz == 0)
+               {
+                  if (dy < (D1D-1)) { Bo(dy,qx) = bo(qx,dy); }
+                  Bc(dy,qx) = bc(qx,dy);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply B operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceCube Xxy(X, nx, ny, VDIM);
+         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double dq = 0.0;
+               for (int dx = 0; dx < nx; ++dx)
+               {
+                  dq += Xxy(dx,dy,vd) * Bx(dx,qx);
+               }
+               QD(qx,dy,vd) = dq;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix By = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double qq = 0.0;
+               for (int dy = 0; dy < ny; ++dy)
+               {
+                  qq += QD(qx,dy,vd) * By(dy,qy);
+               }
+               QQ(qx,qy,vd) = qq;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply D operator
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double Qx = QQ(qx,qy,0);
+               const double Qy = QQ(qx,qy,1);
+
+               const double D11 = D(qx,qy,0,e);
+               const double D12 = D(qx,qy,1,e);
+               const double D21 = symmetric ? D12 : D(qx,qy,2,e);
+               const double D22 = symmetric ? D(qx,qy,2,e) : D(qx,qy,3,e);
+
+               QQ(qx,qy,0) = D11*Qx + D12*Qy;
+               QQ(qx,qy,1) = D21*Qx + D22*Qy;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double qd = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
+               }
+               QD(dx,qy,vd) = qd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
+         DeviceTensor<4> Yxy(y, nx, ny, VDIM, NE);
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double dd = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  dd += QD(dx,qy,vd) * Bty(dy,qy);
+               }
+               Yxy(dx,dy,vd,e) += dd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
@@ -238,7 +488,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,

   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 3, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
   auto diag = Reshape(diag_.ReadWrite(), 2*(D1D-1)*D1D, NE);

   MFEM_FORALL(e, NE,
@@ -259,7 +509,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
               for (int qy = 0; qy < Q1D; ++qy)
               {
                  const double wy = (c == 1) ? Bc(qy,dy) : Bo(qy,dy);
-                  mass[qx] += wy*wy*((c == 0) ? op(qx,qy,0,e) : op(qx,qy,2,e));
+                  mass[qx] += wy*wy*((c == 0) ? op(qx,qy,0,e) : op(qx,qy,symmetric ? 2 : 3,e));
               }
            }

@@ -283,6 +533,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 void PAHdivMassAssembleDiagonal3D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
@@ -294,7 +545,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,

   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto diag = Reshape(diag_.ReadWrite(), 3*(D1D-1)*(D1D-1)*D1D, NE);

   MFEM_FORALL(e, NE,
@@ -307,7 +558,8 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
         const int D1Dy = (c == 1) ? D1D : D1D - 1;
         const int D1Dx = (c == 0) ? D1D : D1D - 1;

-         const int opc = (c == 0) ? 0 : ((c == 1) ? 3 : 5);
+         const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                         (symmetric ? 5 : 8));

         double mass[HDIV_MAX_Q1D];

@@ -350,6 +602,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
 void PAHdivMassApply3D(const int D1D,
                       const int Q1D,
                       const int NE,
+                       const bool symmetric,
                       const Array<double> &Bo_,
                       const Array<double> &Bc_,
                       const Array<double> &Bot_,
@@ -366,7 +619,7 @@ void PAHdivMassApply3D(const int D1D,
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
   auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
   auto Bct = Reshape(Bct_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto x = Reshape(x_.Read(), 3*(D1D-1)*(D1D-1)*D1D, NE);
   auto y = Reshape(y_.ReadWrite(), 3*(D1D-1)*(D1D-1)*D1D, NE);

@@ -461,15 +714,19 @@ void PAHdivMassApply3D(const int D1D,
               const double O11 = op(qx,qy,qz,0,e);
               const double O12 = op(qx,qy,qz,1,e);
               const double O13 = op(qx,qy,qz,2,e);
-               const double O22 = op(qx,qy,qz,3,e);
-               const double O23 = op(qx,qy,qz,4,e);
-               const double O33 = op(qx,qy,qz,5,e);
+               const double O21 = symmetric ? O12 : op(qx,qy,qz,3,e);
+               const double O22 = symmetric ? op(qx,qy,qz,3,e) : op(qx,qy,qz,4,e);
+               const double O23 = symmetric ? op(qx,qy,qz,4,e) : op(qx,qy,qz,5,e);
+               const double O31 = symmetric ? O13 : op(qx,qy,qz,6,e);
+               const double O32 = symmetric ? O23 : op(qx,qy,qz,7,e);
+               const double O33 = symmetric ? op(qx,qy,qz,5,e) : op(qx,qy,qz,8,e);
+
               const double massX = mass[qz][qy][qx][0];
               const double massY = mass[qz][qy][qx][1];
               const double massZ = mass[qz][qy][qx][2];
               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
-               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
-               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
            }
         }
      }
@@ -537,6 +794,337 @@ void PAHdivMassApply3D(const int D1D,
   }); // end of element loop
 }

+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAHdivMassApply3D(const int NE,
+                           const bool symmetric,
+                           const Array<double> &Bo_,
+                           const Array<double> &Bc_,
+                           const Array<double> &Bot_,
+                           const Array<double> &Bct_,
+                           const Vector &op_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(Bot_);
+   MFEM_CONTRACT_VAR(Bct_);
+
+   static constexpr int VDIM = 3;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   const auto x = Reshape(x_.Read(), D1D*(D1D-1)*(D1D-1), VDIM, NE);
+   auto y = y_.ReadWrite();
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, VDIM,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED double smo[MQ1*(MD1-1)];
+      DeviceMatrix Bo(smo, D1D-1, Q1D);
+
+      MFEM_SHARED double smc[MQ1*MD1];
+      DeviceMatrix Bc(smc, D1D, Q1D);
+
+      MFEM_SHARED double sm0[VDIM*MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[VDIM*MDQ*MDQ*MDQ];
+      DeviceMatrix X(sm0, D1D*(D1D-1)*(D1D-1), VDIM);
+      DeviceTensor<4> QDD(sm1, Q1D, D1D, D1D, VDIM);
+      DeviceTensor<4> QQD(sm0, Q1D, Q1D, D1D, VDIM);
+      DeviceTensor<4> QQQ(sm1, Q1D, Q1D, Q1D, VDIM);
+      DeviceTensor<4> DQQ(sm0, D1D, Q1D, Q1D, VDIM);
+      DeviceTensor<4> DDQ(sm1, D1D, D1D, Q1D, VDIM);
+
+      // Load X into shared memory
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         MFEM_FOREACH_THREAD(dz,y,D1D-1)
+         {
+            MFEM_FOREACH_THREAD(dy,x,D1D-1)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  X(dx+(dy+dz*(D1D-1))*D1D,vd) = x(dx+(dy+dz*(D1D-1))*D1D,vd,e);
+               }
+            }
+         }
+      }
+      // Load Bo and Bc into shared memory
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D-1)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bo(d,q) = bo(q,d);
+            }
+         }
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bc(d,q) = bc(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply B operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceTensor<4> Xxyz(X, nx, ny, nz, VDIM);
+         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < nx; ++dx)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += Xxyz(dx,dy,dz,vd) * Bx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { QDD(qx,dy,dz,vd) = u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceMatrix By = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dy = 0; dy < ny; ++dy)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += QDD(qx,dy,dz,vd) * By(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { QQD(qx,qy,dz,vd) = u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceMatrix Bz = (vd == 2) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += QQD(qx,qy,dz,vd) * Bz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { QQQ(qx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply D operator
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  const double Qx = QQQ(qx,qy,qz,0);
+                  const double Qy = QQQ(qx,qy,qz,1);
+                  const double Qz = QQQ(qx,qy,qz,2);
+
+                  const double D11 = D(qx,qy,qz,0,e);
+                  const double D12 = D(qx,qy,qz,1,e);
+                  const double D13 = D(qx,qy,qz,2,e);
+                  const double D21 = symmetric ? D12 : D(qx,qy,qz,3,e);
+                  const double D22 = symmetric ? D(qx,qy,qz,3,e) : D(qx,qy,qz,4,e);
+                  const double D23 = symmetric ? D(qx,qy,qz,4,e) : D(qx,qy,qz,5,e);
+                  const double D31 = symmetric ? D13 : D(qx,qy,qz,6,e);
+                  const double D32 = symmetric ? D23 : D(qx,qy,qz,7,e);
+                  const double D33 = symmetric ? D(qx,qy,qz,5,e) : D(qx,qy,qz,8,e);
+
+                  QQQ(qx,qy,qz,0) = D11*Qx + D12*Qy + D13*Qz;
+                  QQQ(qx,qy,qz,1) = D21*Qx + D22*Qy + D23*Qz;
+                  QQQ(qx,qy,qz,2) = D31*Qx + D32*Qy + D33*Qz;
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceTensor<5> Yxyz(y, nx, ny, nz, VDIM, NE);
+         DeviceMatrix Btz = (vd == 2) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<double> &Bo,
+                     const Array<double> &Bc,
+                     const Array<double> &Bot,
+                     const Array<double> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
 // PA H(div) div-div assemble 2D kernel
 // NOTE: this is identical to PACurlCurlSetup3D
 static void PADivDivSetup2D(const int Q1D,
@@ -626,7 +1214,7 @@ static void PADivDivApply2D(const int D1D,
   {
      double div[MAX_Q1D][MAX_Q1D];

-      // div[qy][qx] will be computed as du_x/dx + duy_/dy
+      // div[qy][qx] will be computed as du_x/dx + du_y/dy

      for (int qy = 0; qy < Q1D; ++qy)
      {
@@ -1209,6 +1797,13 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      }
   }

+   if (test_el->GetMapType() == FiniteElement::INTEGRAL)
+   {
+      const GeometricFactors *geom =
+         mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS);
+      coeff /= geom->detJ;
+   }
+
   if (trial_el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
   {
      PADivL2Setup3D(quad1D, ne, ir->GetWeights(), coeff, pa_data);
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -31,7 +31,16 @@ void MassIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: MassIntegrator::AssembleMF only implemented with"
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -38,7 +38,16 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   int map_type = el.GetMapType();
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -149,7 +149,16 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   const int dims = el.GetDim();
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/diffusion.hpp"
+#include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;

@@ -30,7 +30,19 @@ void VectorDiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      MFEM_VERIFY(!VQ && !MQ,
+                  "Only scalar coefficient supported for DiffusionIntegrator"
+                  " with libCEED");
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFDiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: VectorDiffusionIntegrator::AssembleMF only implemented"
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -34,7 +34,16 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   dim = mesh->Dimension();
@@ -12,7 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "ceed/mass.hpp"
+#include "ceed/integrators/mass/mass.hpp"

 using namespace std;

@@ -34,7 +34,16 @@ void VectorMassIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: VectorMassIntegrator::AssembleMF only implemented with"
@@ -11,6 +11,7 @@

 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
+#include "gridfunc.hpp"

 namespace mfem
 {
@@ -89,6 +90,7 @@ void SmemPAHcurlMassApply3D(const int D1D,
                            Vector &y);

 void PAHdivSetup2D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
@@ -96,6 +98,7 @@ void PAHdivSetup2D(const int Q1D,
                   Vector &op);

 void PAHdivSetup3D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
@@ -149,6 +152,7 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
@@ -157,32 +161,24 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 void PAHdivMassAssembleDiagonal3D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
                                  Vector &diag_);

-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const Array<double> &Bo_,
-                       const Array<double> &Bc_,
-                       const Array<double> &Bot_,
-                       const Array<double> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
-
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const Array<double> &Bo_,
-                       const Array<double> &Bc_,
-                       const Array<double> &Bot_,
-                       const Array<double> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<double> &Bo,
+                     const Array<double> &Bc,
+                     const Array<double> &Bot,
+                     const Array<double> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);

 void PAHcurlL2Setup(const int NQ,
                    const int coeffDim,
@@ -818,68 +814,79 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
                      Device::GetMemoryType());

-   Vector coeff(coeffDim * ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
+   Vector coeff;
+
+   auto *qf_c = dynamic_cast<QuadratureFunctionCoefficient*>(Q);
+   if (qf_c)
   {
-      Vector DM(DQ ? coeffDim : 0);
-      DenseMatrix M;
-      DenseSymmetricMatrix SM;
+      const QuadratureFunction &qf = qf_c->GetQuadFunction();
+      qf.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction&>(qf), 0);
+   }
+   else
+   {
+      coeff.SetSize(coeffDim * ne * nq);
+      coeff = 1.0;
+      auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+      if (Q || DQ || MQ)
+      {
+         Vector DM(DQ ? coeffDim : 0);
+         DenseMatrix M;
+         DenseSymmetricMatrix SM;

-      if (DQ)
-      {
-         MFEM_VERIFY(coeffDim == dim, "");
-      }
-      if (SMQ)
-      {
-         MFEM_VERIFY(SMQ->GetSize() == dim, "");
-         SM.SetSize(dim);
-      }
-      else if (MQ)
-      {
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-         M.SetSize(dim);
-      }
-
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
+         if (DQ)
         {
-            if (SMQ)
-            {
-               SMQ->Eval(SM, *tr, ir->IntPoint(p));
-               int cnt = 0;
-               for (int i=0; i<dim; ++i)
-                  for (int j=i; j<dim; ++j, ++cnt)
-                  {
-                     coeffh(cnt, p, e) = SM(i,j);
-                  }
-            }
-            else if (MQ)
-            {
-               MQ->Eval(M, *tr, ir->IntPoint(p));
+            MFEM_VERIFY(coeffDim == dim, "");
+         }
+         if (SMQ)
+         {
+            MFEM_VERIFY(SMQ->GetSize() == dim, "");
+            SM.SetSize(dim);
+         }
+         else if (MQ)
+         {
+            MFEM_VERIFY(coeffDim == MQdim, "");
+            MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+            M.SetSize(dim);
+         }

-               for (int i=0; i<dim; ++i)
-                  for (int j=0; j<dim; ++j)
-                  {
-                     coeffh(j+(i*dim), p, e) = M(i,j);
-                  }
-            }
-            else if (DQ)
+         for (int e=0; e<ne; ++e)
+         {
+            ElementTransformation *tr = mesh->GetElementTransformation(e);
+            for (int p=0; p<nq; ++p)
            {
-               DQ->Eval(DM, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
+               if (SMQ)
               {
-                  coeffh(i, p, e) = DM[i];
+                  SMQ->Eval(SM, *tr, ir->IntPoint(p));
+                  int cnt = 0;
+                  for (int i=0; i<dim; ++i)
+                     for (int j=i; j<dim; ++j, ++cnt)
+                     {
+                        coeffh(cnt, p, e) = SM(i,j);
+                     }
+               }
+               else if (MQ)
+               {
+                  MQ->Eval(M, *tr, ir->IntPoint(p));
+
+                  for (int i=0; i<dim; ++i)
+                     for (int j=0; j<dim; ++j)
+                     {
+                        coeffh(j+(i*dim), p, e) = M(i,j);
+                     }
+               }
+               else if (DQ)
+               {
+                  DQ->Eval(DM, *tr, ir->IntPoint(p));
+                  for (int i=0; i<coeffDim; ++i)
+                  {
+                     coeffh(i, p, e) = DM[i];
+                  }
+               }
+               else
+               {
+                  coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
            }
         }
      }
@@ -897,12 +904,12 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   }
   else if (trial_div && test_div && dim == 3)
   {
-      PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 2)
   {
-      PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup2D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
@@ -963,7 +970,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
      else if (trial_fetype == mfem::FiniteElement::DIV &&
               test_fetype == trial_fetype)
      {
-         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne,
+         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
                                      mapsO->B, mapsC->B, pa_data, diag);
      }
      else
@@ -971,7 +978,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
         MFEM_ABORT("Unknown kernel.");
      }
   }
-   else
+   else // 2D
   {
      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
      {
@@ -981,7 +988,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
      else if (trial_fetype == mfem::FiniteElement::DIV &&
               test_fetype == trial_fetype)
      {
-         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne,
+         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne, symmetric,
                                      mapsO->B, mapsC->B, pa_data, diag);
      }
      else
@@ -1034,8 +1041,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_div && test_div)
      {
-         PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                           mapsC->Bt, pa_data, x, y);
+         PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
+                         mapsC->Bt, pa_data, x, y);
      }
      else if (trial_curl && test_div)
      {
@@ -1056,7 +1063,7 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
         MFEM_ABORT("Unknown kernel.");
      }
   }
-   else
+   else // 2D
   {
      if (trial_curl && test_curl)
      {
@@ -1065,8 +1072,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_div && test_div)
      {
-         PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                           mapsC->Bt, pa_data, x, y);
+         PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
+                         mapsC->Bt, pa_data, x, y);
      }
      else if ((trial_curl && test_div) || (trial_div && test_curl))
      {
@@ -1111,6 +1118,11 @@ void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

   if (symmetricSpaces)
   {
+      if (MQ && dynamic_cast<SymmetricMatrixCoefficient*>(MQ) == NULL)
+      {
+         MFEM_ABORT("VectorFEMassIntegrator transpose not implemented for asymmetric MatrixCoefficient");
+      }
+
      this->AddMultPA(x, y);
   }
 }
@@ -1,222 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_LIBCEED_COEFF
-#define MFEM_LIBCEED_COEFF
-
-#include "../../config/config.hpp"
-#ifdef MFEM_USE_CEED
-#include "util.hpp"
-#include "../../linalg/vector.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../mesh/mesh.hpp"
-#include "../gridfunc.hpp"
-#include "../../linalg/dtensor.hpp"
-
-namespace mfem
-{
-
-class Mesh;
-class IntegrationRule;
-class Coefficient;
-class VectorCoefficient;
-class GridFunction;
-
-namespace ceed
-{
-
-struct Coefficient
-{
-   const int ncomp;
-   Coefficient(int ncomp_) : ncomp(ncomp_) { }
-   virtual bool IsConstant() const { return true; }
-   virtual ~Coefficient() { }
-};
-
-struct VariableCoefficient : Coefficient
-{
-   CeedVector coeffVector = nullptr;
-   const CeedEvalMode emode;
-   VariableCoefficient(int ncomp_, CeedEvalMode emode_)
-      : Coefficient(ncomp_), emode(emode_) { }
-   virtual bool IsConstant() const override { return false; }
-   ~VariableCoefficient()
-   {
-      CeedVectorDestroy(&coeffVector);
-   }
-};
-
-struct GridCoefficient : VariableCoefficient
-{
-   const mfem::GridFunction &gf;
-   CeedBasis basis;
-   CeedElemRestriction restr;
-   GridCoefficient(const mfem::GridFunction &gf_)
-      : VariableCoefficient(gf_.VectorDim(), CEED_EVAL_INTERP),
-        gf(gf_)
-   {
-      InitVector(gf, coeffVector);
-   }
-};
-
-struct QuadCoefficient : VariableCoefficient
-{
-   mfem::Vector coeff;
-   CeedElemRestriction restr;
-   QuadCoefficient(int ncomp_) : VariableCoefficient(ncomp_, CEED_EVAL_NONE) { }
-};
-
-/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
-    mfem::Coefficient @a Q, an mfem::Mesh @a mesh, and an mfem::IntegrationRule
-    @a ir. */
-template <typename Context>
-void InitCoefficient(mfem::Coefficient *Q, mfem::Mesh &mesh,
-                     const mfem::IntegrationRule &ir,
-                     Coefficient*& coeff_ptr, Context &ctx)
-{
-   if ( Q == nullptr )
-   {
-      Coefficient *ceedCoeff = new Coefficient(1);
-      ctx.coeff = 1.0;
-      coeff_ptr = ceedCoeff;
-   }
-   else if (ConstantCoefficient *const_coeff =
-               dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      Coefficient *ceedCoeff = new Coefficient(1);
-      ctx.coeff = const_coeff->constant;
-      coeff_ptr = ceedCoeff;
-   }
-   else if (GridFunctionCoefficient* gf_coeff =
-               dynamic_cast<GridFunctionCoefficient*>(Q))
-   {
-      GridCoefficient *ceedCoeff =
-         new GridCoefficient(*gf_coeff->GetGridFunction());
-      coeff_ptr = ceedCoeff;
-   }
-   else if (QuadratureFunctionCoefficient *cQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
-      const int ne = mesh.GetNE();
-      const int nq = ir.GetNPoints();
-      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      ceedCoeff->coeff.MakeRef(const_cast<mfem::QuadratureFunction &>(qFun),0);
-      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
-      coeff_ptr = ceedCoeff;
-   }
-   else
-   {
-      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
-      const int ne = mesh.GetNE();
-      const int nq = ir.GetNPoints();
-      ceedCoeff->coeff.SetSize(nq * ne);
-      auto C = Reshape(ceedCoeff->coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir.IntPoint(q));
-         }
-      }
-      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
-      coeff_ptr = ceedCoeff;
-   }
-}
-
-
-/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
-    mfem::VectorCoefficient @a Q, an mfem::Mesh @a mesh, and an
-    mfem::IntegrationRule @a ir. */
-template <typename Context>
-void InitCoefficient(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
-                     const mfem::IntegrationRule &ir,
-                     Coefficient *&coeff_ptr, Context &ctx)
-{
-   if (VectorConstantCoefficient *const_coeff =
-          dynamic_cast<VectorConstantCoefficient*>(VQ))
-   {
-      const int vdim = const_coeff->GetVDim();
-      const mfem::Vector &val = const_coeff->GetVec();
-      Coefficient *ceedCoeff = new Coefficient(vdim);
-      for (int i = 0; i < vdim; i++)
-      {
-         ctx.coeff[i] = val[i];
-      }
-      coeff_ptr = ceedCoeff;
-   }
-   else if (VectorGridFunctionCoefficient* vgf_coeff =
-               dynamic_cast<VectorGridFunctionCoefficient*>(VQ))
-   {
-      GridCoefficient *ceedCoeff =
-         new GridCoefficient(*vgf_coeff->GetGridFunction());
-      coeff_ptr = ceedCoeff;
-   }
-   else if (VectorQuadratureFunctionCoefficient *cQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(VQ))
-   {
-      QuadCoefficient *ceedCoeff = new QuadCoefficient(cQ->GetVDim());
-      const int dim = mesh.Dimension();
-      const int ne = mesh.GetNE();
-      const int nq = ir.GetNPoints();
-      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      ceedCoeff->coeff.MakeRef(const_cast<mfem::QuadratureFunction &>(qFun),0);
-      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
-      coeff_ptr = ceedCoeff;
-   }
-   else
-   {
-      const int dim = mesh.Dimension();
-      QuadCoefficient *ceedCoeff = new QuadCoefficient(dim);
-      const int ne = mesh.GetNE();
-      const int nq = ir.GetNPoints();
-      ceedCoeff->coeff.SetSize(dim * nq * ne);
-      auto C = Reshape(ceedCoeff->coeff.HostWrite(), dim, nq, ne);
-      mfem::DenseMatrix Q_ir;
-      for (int e = 0; e < ne; ++e)
-      {
-         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
-         VQ->Eval(Q_ir, T, ir);
-         for (int q = 0; q < nq; ++q)
-         {
-            for (int i = 0; i < dim; ++i)
-            {
-               C(i,q,e) = Q_ir(i,q);
-            }
-         }
-      }
-      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
-      coeff_ptr = ceedCoeff;
-   }
-}
-
-} // namespace ceed
-
-} // namespace mfem
-
-#endif
-
-#endif // MFEM_LIBCEED_COEFF
@@ -11,7 +11,7 @@

 #include "convection.hpp"

-#include "../../config/config.hpp"
+#include "../../../../config/config.hpp"
 #ifdef MFEM_USE_CEED
 #include "convection_qf.h"
 #endif
@@ -28,7 +28,7 @@ struct ConvectionOperatorInfo : public OperatorInfo
   ConvectionContext ctx;
   ConvectionOperatorInfo(int dim, double alpha)
   {
-      header = "/convection_qf.h";
+      header = "/integrators/convection/convection_qf.h";
      build_func_const = ":f_build_conv_const";
      build_qf_const = &f_build_conv_const;
      build_func_quad = ":f_build_conv_quad";
@@ -62,6 +62,20 @@ PAConvectionIntegrator::PAConvectionIntegrator(
 #endif
 }

+MixedPAConvectionIntegrator::MixedPAConvectionIntegrator(
+   const ConvectionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::VectorCoefficient *Q,
+   const double alpha)
+{
+#ifdef MFEM_USE_CEED
+   ConvectionOperatorInfo info(fes.GetMesh()->Dimension(), alpha);
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFConvectionIntegrator::MFConvectionIntegrator(
   const mfem::FiniteElementSpace &fes,
   const mfem::IntegrationRule &irm,
@@ -77,6 +91,20 @@ MFConvectionIntegrator::MFConvectionIntegrator(
 #endif
 }

+MixedMFConvectionIntegrator::MixedMFConvectionIntegrator(
+   const ConvectionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::VectorCoefficient *Q,
+   const double alpha)
+{
+#ifdef MFEM_USE_CEED
+   ConvectionOperatorInfo info(fes.GetMesh()->Dimension(), alpha);
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -12,8 +12,9 @@
 #ifndef MFEM_LIBCEED_CONV_HPP
 #define MFEM_LIBCEED_CONV_HPP

-#include "integrator.hpp"
-#include "../fespace.hpp"
+#include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
+#include "../../../fespace.hpp"

 namespace mfem
 {
@@ -26,21 +27,39 @@ class PAConvectionIntegrator : public PAIntegrator
 {
 public:
   PAConvectionIntegrator(const mfem::FiniteElementSpace &fes,
-                          const mfem::IntegrationRule &irm,
+                          const mfem::IntegrationRule &ir,
                          mfem::VectorCoefficient *Q,
                          const double alpha);
 };

+class MixedPAConvectionIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPAConvectionIntegrator(const ConvectionIntegrator &integ,
+                               const mfem::FiniteElementSpace &fes,
+                               mfem::VectorCoefficient *Q,
+                               const double alpha);
+};
+
 /// Represent a ConvectionIntegrator with AssemblyLevel::None using libCEED.
 class MFConvectionIntegrator : public MFIntegrator
 {
 public:
   MFConvectionIntegrator(const mfem::FiniteElementSpace &fes,
-                          const mfem::IntegrationRule &irm,
+                          const mfem::IntegrationRule &ir,
                          mfem::VectorCoefficient *Q,
                          const double alpha);
 };

+class MixedMFConvectionIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFConvectionIntegrator(const ConvectionIntegrator &integ,
+                               const mfem::FiniteElementSpace &fes,
+                               mfem::VectorCoefficient *Q,
+                               const double alpha);
+};
+
 }

 }
@@ -11,7 +11,7 @@

 #include "diffusion.hpp"

-#include "../../config/config.hpp"
+#include "../../../../config/config.hpp"
 #ifdef MFEM_USE_CEED
 #include "diffusion_qf.h"
 #endif
@@ -28,7 +28,7 @@ struct DiffusionOperatorInfo : public OperatorInfo
   DiffusionContext ctx;
   DiffusionOperatorInfo(int dim)
   {
-      header = "/diffusion_qf.h";
+      header = "/integrators/diffusion/diffusion_qf.h";
      build_func_const = ":f_build_diff_const";
      build_qf_const = &f_build_diff_const;
      build_func_quad = ":f_build_diff_quad";
@@ -60,6 +60,32 @@ PADiffusionIntegrator::PADiffusionIntegrator(
 #endif
 }

+MixedPADiffusionIntegrator::MixedPADiffusionIntegrator(
+   const DiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedPADiffusionIntegrator::MixedPADiffusionIntegrator(
+   const VectorDiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFDiffusionIntegrator::MFDiffusionIntegrator(
   const mfem::FiniteElementSpace &fes,
   const mfem::IntegrationRule &irm,
@@ -74,6 +100,32 @@ MFDiffusionIntegrator::MFDiffusionIntegrator(
 #endif
 }

+MixedMFDiffusionIntegrator::MixedMFDiffusionIntegrator(
+   const DiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedMFDiffusionIntegrator::MixedMFDiffusionIntegrator(
+   const VectorDiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -12,8 +12,9 @@
 #ifndef MFEM_LIBCEED_DIFF_HPP
 #define MFEM_LIBCEED_DIFF_HPP

-#include "integrator.hpp"
-#include "../fespace.hpp"
+#include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
+#include "../../../fespace.hpp"

 namespace mfem
 {
@@ -26,19 +27,43 @@ class PADiffusionIntegrator : public PAIntegrator
 {
 public:
   PADiffusionIntegrator(const mfem::FiniteElementSpace &fes,
-                         const mfem::IntegrationRule &irm,
+                         const mfem::IntegrationRule &ir,
                         mfem::Coefficient *Q);
 };

+class MixedPADiffusionIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPADiffusionIntegrator(const DiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+
+   MixedPADiffusionIntegrator(const VectorDiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+};
+
 /// Represent a DiffusionIntegrator with AssemblyLevel::None using libCEED.
 class MFDiffusionIntegrator : public MFIntegrator
 {
 public:
   MFDiffusionIntegrator(const mfem::FiniteElementSpace &fes,
-                         const mfem::IntegrationRule &irm,
+                         const mfem::IntegrationRule &ir,
                         mfem::Coefficient *Q);
 };

+class MixedMFDiffusionIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFDiffusionIntegrator(const DiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+
+   MixedMFDiffusionIntegrator(const VectorDiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+};
+
 }

 }
@@ -11,7 +11,7 @@

 #include "mass.hpp"

-#include "../../config/config.hpp"
+#include "../../../../config/config.hpp"
 #ifdef MFEM_USE_CEED
 #include "mass_qf.h"
 #endif
@@ -28,7 +28,7 @@ struct MassOperatorInfo : public OperatorInfo
   MassContext ctx;
   MassOperatorInfo()
   {
-      header = "/mass_qf.h";
+      header = "/integrators/mass/mass_qf.h";
      build_func_const = ":f_build_mass_const";
      build_qf_const = &f_build_mass_const;
      build_func_quad = ":f_build_mass_quad";
@@ -59,6 +59,30 @@ PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
 #endif
 }

+MixedPAMassIntegrator::MixedPAMassIntegrator(const MassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedPAMassIntegrator::MixedPAMassIntegrator(const VectorMassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
                                   const mfem::IntegrationRule &irm,
                                   mfem::Coefficient *Q)
@@ -72,6 +96,30 @@ MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
 #endif
 }

+MixedMFMassIntegrator::MixedMFMassIntegrator(const MassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedMFMassIntegrator::MixedMFMassIntegrator(const VectorMassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -12,8 +12,9 @@
 #ifndef MFEM_LIBCEED_MASS_HPP
 #define MFEM_LIBCEED_MASS_HPP

-#include "integrator.hpp"
-#include "../fespace.hpp"
+#include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
+#include "../../../fespace.hpp"

 namespace mfem
 {
@@ -26,19 +27,43 @@ class PAMassIntegrator : public PAIntegrator
 {
 public:
   PAMassIntegrator(const mfem::FiniteElementSpace &fes,
-                    const mfem::IntegrationRule &irm,
+                    const mfem::IntegrationRule &ir,
                    mfem::Coefficient *Q);
 };

+class MixedPAMassIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPAMassIntegrator(const MassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+
+   MixedPAMassIntegrator(const VectorMassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+};
+
 /// Represent a MassIntegrator with AssemblyLevel::None using libCEED.
 class MFMassIntegrator : public MFIntegrator
 {
 public:
   MFMassIntegrator(const mfem::FiniteElementSpace &fes,
-                    const mfem::IntegrationRule &irm,
+                    const mfem::IntegrationRule &ir,
                    mfem::Coefficient *Q);
 };

+class MixedMFMassIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFMassIntegrator(const MassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+
+   MixedMFMassIntegrator(const VectorMassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+};
+
 }

 }
@@ -11,7 +11,7 @@

 #include "nlconvection.hpp"

-#include "../../config/config.hpp"
+#include "../../../../config/config.hpp"
 #ifdef MFEM_USE_CEED
 #include "nlconvection_qf.h"
 #endif
@@ -28,7 +28,7 @@ struct NLConvectionOperatorInfo : public OperatorInfo
   NLConvectionContext ctx;
   NLConvectionOperatorInfo(int dim)
   {
-      header = "/nlconvection_qf.h";
+      header = "/integrators/nlconvection/nlconvection_qf.h";
      build_func_const = ":f_build_conv_const";
      build_qf_const = &f_build_conv_const;
      build_func_quad = ":f_build_conv_quad";
@@ -60,6 +60,19 @@ PAVectorConvectionNLFIntegrator::PAVectorConvectionNLFIntegrator(
 #endif
 }

+MixedPAVectorConvectionNLIntegrator::MixedPAVectorConvectionNLIntegrator(
+   const VectorConvectionNLFIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   NLConvectionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFVectorConvectionNLFIntegrator::MFVectorConvectionNLFIntegrator(
   const mfem::FiniteElementSpace &fes,
   const mfem::IntegrationRule &irm,
@@ -74,6 +87,19 @@ MFVectorConvectionNLFIntegrator::MFVectorConvectionNLFIntegrator(
 #endif
 }

+MixedMFVectorConvectionNLIntegrator::MixedMFVectorConvectionNLIntegrator(
+   const VectorConvectionNLFIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   NLConvectionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -12,8 +12,9 @@
 #ifndef MFEM_LIBCEED_NLCONV_HPP
 #define MFEM_LIBCEED_NLCONV_HPP

-#include "integrator.hpp"
-#include "../fespace.hpp"
+#include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
+#include "../../../fespace.hpp"

 namespace mfem
 {
@@ -31,6 +32,15 @@ public:
                                   mfem::Coefficient *coeff);
 };

+class MixedPAVectorConvectionNLIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPAVectorConvectionNLIntegrator(
+      const VectorConvectionNLFIntegrator &integ,
+      const mfem::FiniteElementSpace &fes,
+      mfem::Coefficient *Q);
+};
+
 /** Represent a VectorConvectionNLFIntegrator with AssemblyLevel::None
    using libCEED. */
 class MFVectorConvectionNLFIntegrator : public MFIntegrator
@@ -41,6 +51,15 @@ public:
                                   mfem::Coefficient *coeff);
 };

+class MixedMFVectorConvectionNLIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFVectorConvectionNLIntegrator(
+      const VectorConvectionNLFIntegrator &integ,
+      const mfem::FiniteElementSpace &fes,
+      mfem::Coefficient *Q);
+};
+
 }

 }
@@ -327,13 +327,13 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
            const CeedScalar A33 = J11 * J22 - J12 * J21;
            const CeedScalar w = qw[i] * coeff;
            const CeedScalar qd00 = w * A11;
-            const CeedScalar qd01 = w * A21;
-            const CeedScalar qd02 = w * A31;
-            const CeedScalar qd10 = w * A12;
+            const CeedScalar qd10 = w * A21;
+            const CeedScalar qd20 = w * A31;
+            const CeedScalar qd01 = w * A12;
            const CeedScalar qd11 = w * A22;
-            const CeedScalar qd12 = w * A32;
-            const CeedScalar qd20 = w * A13;
-            const CeedScalar qd21 = w * A23;
+            const CeedScalar qd21 = w * A32;
+            const CeedScalar qd02 = w * A13;
+            const CeedScalar qd12 = w * A23;
            const CeedScalar qd22 = w * A33;
            const CeedScalar u0   = u[i + Q * 0];
            const CeedScalar u1   = u[i + Q * 1];
@@ -440,13 +440,13 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
            const CeedScalar A33 = J11 * J22 - J12 * J21;
            const CeedScalar w = qw[i] * c[i];
            const CeedScalar qd00 = w * A11;
-            const CeedScalar qd01 = w * A21;
-            const CeedScalar qd02 = w * A31;
-            const CeedScalar qd10 = w * A12;
+            const CeedScalar qd10 = w * A21;
+            const CeedScalar qd20 = w * A31;
+            const CeedScalar qd01 = w * A12;
            const CeedScalar qd11 = w * A22;
-            const CeedScalar qd12 = w * A32;
-            const CeedScalar qd20 = w * A13;
-            const CeedScalar qd21 = w * A23;
+            const CeedScalar qd21 = w * A32;
+            const CeedScalar qd02 = w * A13;
+            const CeedScalar qd12 = w * A23;
            const CeedScalar qd22 = w * A33;
            const CeedScalar u0   = u[i + Q * 0];
            const CeedScalar u1   = u[i + Q * 1];
@@ -0,0 +1,156 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../gridfunc.hpp"
+#include "util.hpp"
+
+namespace mfem
+{
+
+namespace ceed
+{
+
+#ifdef MFEM_USE_CEED
+
+static CeedElemTopology GetCeedTopology(Geometry::Type geom)
+{
+   switch (geom)
+   {
+      case Geometry::SEGMENT:
+         return CEED_TOPOLOGY_LINE;
+      case Geometry::TRIANGLE:
+         return CEED_TOPOLOGY_TRIANGLE;
+      case Geometry::SQUARE:
+         return CEED_TOPOLOGY_QUAD;
+      case Geometry::TETRAHEDRON:
+         return CEED_TOPOLOGY_TET;
+      case Geometry::CUBE:
+         return CEED_TOPOLOGY_HEX;
+      case Geometry::PRISM:
+         return CEED_TOPOLOGY_PRISM;
+      case Geometry::PYRAMID:
+         return CEED_TOPOLOGY_PYRAMID;
+      default:
+         MFEM_ABORT("This type of element is not supported");
+         return CEED_TOPOLOGY_PRISM; // Silence warning
+   }
+}
+
+static void InitNonTensorBasis(const mfem::FiniteElementSpace &fes,
+                               const mfem::FiniteElement &fe,
+                               const mfem::IntegrationRule &ir,
+                               Ceed ceed, CeedBasis *basis)
+{
+   const mfem::DofToQuad &maps = fe.GetDofToQuad(ir, mfem::DofToQuad::FULL);
+   mfem::Mesh *mesh = fes.GetMesh();
+   const int dim = mesh->Dimension();
+   const int ndofs = maps.ndof;
+   const int nqpts = maps.nqpt;
+   mfem::DenseMatrix qX(dim,nqpts);
+   mfem::Vector qW(nqpts);
+   for (int i = 0; i < nqpts; i++)
+   {
+      const mfem::IntegrationPoint &ip = ir.IntPoint(i);
+      qX(0,i) = ip.x;
+      if (dim>1) { qX(1,i) = ip.y; }
+      if (dim>2) { qX(2,i) = ip.z; }
+      qW(i) = ip.weight;
+   }
+   CeedBasisCreateH1(ceed, GetCeedTopology(fe.GetGeomType()),
+                     fes.GetVDim(), ndofs, nqpts,
+                     maps.Bt.GetData(), maps.Gt.GetData(),
+                     qX.GetData(), qW.GetData(), basis);
+}
+
+static void InitTensorBasis(const mfem::FiniteElementSpace &fes,
+                            const mfem::FiniteElement &fe,
+                            const mfem::IntegrationRule &ir,
+                            Ceed ceed, CeedBasis *basis)
+{
+   const mfem::DofToQuad &maps = fe.GetDofToQuad(ir, mfem::DofToQuad::TENSOR);
+   mfem::Mesh *mesh = fes.GetMesh();
+   const int ndofs = maps.ndof;
+   const int nqpts = maps.nqpt;
+   mfem::Vector qX(nqpts), qW(nqpts);
+   // The x-coordinates of the first `nqpts` points of the integration rule are
+   // the points of the corresponding 1D rule. We also scale the weights
+   // accordingly.
+   double w_sum = 0.0;
+   for (int i = 0; i < nqpts; i++)
+   {
+      const mfem::IntegrationPoint &ip = ir.IntPoint(i);
+      qX(i) = ip.x;
+      qW(i) = ip.weight;
+      w_sum += ip.weight;
+   }
+   qW *= 1.0/w_sum;
+   CeedBasisCreateTensorH1(ceed, mesh->Dimension(), fes.GetVDim(), ndofs,
+                           nqpts, maps.Bt.GetData(),
+                           maps.Gt.GetData(), qX.GetData(),
+                           qW.GetData(), basis);
+}
+
+static void InitBasisImpl(const FiniteElementSpace &fes,
+                          const FiniteElement &fe,
+                          const IntegrationRule &ir,
+                          Ceed ceed, CeedBasis *basis)
+{
+   // Check for FES -> basis, restriction in hash tables
+   const int P = fe.GetDof();
+   const int Q = ir.GetNPoints();
+   const int ncomp = fes.GetVDim();
+   BasisKey basis_key(&fes, &ir, ncomp, P, Q);
+   auto basis_itr = mfem::internal::ceed_basis_map.find(basis_key);
+   const bool tensor = dynamic_cast<const mfem::TensorBasisElement *>
+                       (&fe) != nullptr;
+
+   // Init or retreive key values
+   if (basis_itr == mfem::internal::ceed_basis_map.end())
+   {
+      if ( tensor )
+      {
+         InitTensorBasis(fes, fe, ir, ceed, basis);
+      }
+      else
+      {
+         InitNonTensorBasis(fes, fe, ir, ceed, basis);
+      }
+      mfem::internal::ceed_basis_map[basis_key] = *basis;
+   }
+   else
+   {
+      *basis = basis_itr->second;
+   }
+}
+
+void InitBasis(const FiniteElementSpace &fes,
+               const IntegrationRule &ir,
+               Ceed ceed, CeedBasis *basis)
+{
+   const mfem::FiniteElement &fe = *fes.GetFE(0);
+   InitBasisImpl(fes, fe, ir, ceed, basis);
+}
+
+void InitBasisWithIndices(const FiniteElementSpace &fes,
+                          const IntegrationRule &ir,
+                          int nelem,
+                          const int* indices,
+                          Ceed ceed, CeedBasis *basis)
+{
+   const mfem::FiniteElement &fe = *fes.GetFE(indices[0]);
+   InitBasisImpl(fes, fe, ir, ceed, basis);
+}
+
+#endif
+
+} // namespace ceed
+
+} // namespace mfem
@@ -0,0 +1,57 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIBCEED_BASIS
+#define MFEM_LIBCEED_BASIS
+
+#include "ceed.hpp"
+
+namespace mfem
+{
+
+namespace ceed
+{
+
+#ifdef MFEM_USE_CEED
+
+/** @brief Initialize a CeedBasis for non-mixed meshes.
+
+   @param[in] fes Input finite element space.
+   @param[in] ir Input integration rule.
+   @param[in] ceed Input Ceed object.
+   @param[out] basis The address of the initialized CeedBasis object.
+*/
+void InitBasis(const FiniteElementSpace &fes,
+               const IntegrationRule &ir,
+               Ceed ceed, CeedBasis *basis);
+
+/** @brief Initialize a CeedBasis for mixed meshes.
+
+    @param[in] fes The finite element space.
+    @param[in] ir is the integration rule for the operator.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[in] ceed The Ceed object.
+    @param[out] basis The `CeedBasis` to initialize. */
+void InitBasisWithIndices(const FiniteElementSpace &fes,
+                          const IntegrationRule &ir,
+                          int nelem,
+                          const int* indices,
+                          Ceed ceed, CeedBasis *basis);
+
+#endif
+
+} // namespace ceed
+
+} // namespace mfem
+
+#endif // MFEM_LIBCEED_BASIS
@@ -0,0 +1,35 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIBCEED_CEED
+#define MFEM_LIBCEED_CEED
+
+#ifdef MFEM_USE_CEED
+#include <ceed.h>
+#if !CEED_VERSION_GE(0,10,0)
+#error MFEM requires a libCEED version >= 0.10.0
+#endif
+
+namespace mfem
+{
+
+namespace internal
+{
+
+extern Ceed ceed;
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif // MFEM_USE_CEED
+
+#endif // MFEM_LIBCEED_CEED
@@ -0,0 +1,441 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIBCEED_COEFF
+#define MFEM_LIBCEED_COEFF
+
+#ifdef MFEM_USE_CEED
+
+#include "../../../general/forall.hpp"
+#include "../../../config/config.hpp"
+#include "../../../linalg/vector.hpp"
+#include "../../../linalg/dtensor.hpp"
+#include "../../../mesh/mesh.hpp"
+#include "../../gridfunc.hpp"
+#include "util.hpp"
+#include "ceed.hpp"
+
+namespace mfem
+{
+
+class Mesh;
+class IntegrationRule;
+class Coefficient;
+class VectorCoefficient;
+class GridFunction;
+
+namespace ceed
+{
+
+struct Coefficient
+{
+   const int ncomp;
+   Coefficient(int ncomp_) : ncomp(ncomp_) { }
+   virtual bool IsConstant() const { return true; }
+   virtual ~Coefficient() { }
+};
+
+struct VariableCoefficient : Coefficient
+{
+   CeedVector coeffVector = nullptr;
+   const CeedEvalMode emode;
+   VariableCoefficient(int ncomp_, CeedEvalMode emode_)
+      : Coefficient(ncomp_), emode(emode_) { }
+   virtual bool IsConstant() const override { return false; }
+   ~VariableCoefficient()
+   {
+      CeedVectorDestroy(&coeffVector);
+   }
+};
+
+struct GridCoefficient : VariableCoefficient
+{
+   const mfem::GridFunction &gf;
+   CeedBasis basis;
+   CeedElemRestriction restr;
+   GridCoefficient(const mfem::GridFunction &gf_)
+      : VariableCoefficient(gf_.VectorDim(), CEED_EVAL_INTERP),
+        gf(gf_)
+   {
+      InitVector(gf, coeffVector);
+   }
+};
+
+struct QuadCoefficient : VariableCoefficient
+{
+   mfem::Vector coeff;
+   CeedElemRestriction restr;
+   QuadCoefficient(int ncomp_) : VariableCoefficient(ncomp_, CEED_EVAL_NONE) { }
+};
+
+/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
+    mfem::Coefficient @a Q, an mfem::Mesh @a mesh, and an mfem::IntegrationRule
+    @a ir.
+
+    @param[in] Q is the coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
+template <typename Context>
+void InitCoefficient(mfem::Coefficient *Q, mfem::Mesh &mesh,
+                     const mfem::IntegrationRule &ir,
+                     Coefficient*& coeff_ptr, Context &ctx)
+{
+   if ( Q == nullptr )
+   {
+      Coefficient *ceedCoeff = new Coefficient(1);
+      ctx.coeff = 1.0;
+      coeff_ptr = ceedCoeff;
+   }
+   else if (ConstantCoefficient *const_coeff =
+               dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      Coefficient *ceedCoeff = new Coefficient(1);
+      ctx.coeff = const_coeff->constant;
+      coeff_ptr = ceedCoeff;
+   }
+   else if (GridFunctionCoefficient* gf_coeff =
+               dynamic_cast<GridFunctionCoefficient*>(Q))
+   {
+      GridCoefficient *ceedCoeff =
+         new GridCoefficient(*gf_coeff->GetGridFunction());
+      coeff_ptr = ceedCoeff;
+   }
+   else if (QuadratureFunctionCoefficient *cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      ceedCoeff->coeff.MakeRef(const_cast<mfem::QuadratureFunction &>(qFun),0);
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+   else
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      ceedCoeff->coeff.SetSize(nq * ne);
+      auto C = Reshape(ceedCoeff->coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir.IntPoint(q));
+         }
+      }
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+}
+
+
+/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
+    mfem::VectorCoefficient @a VQ, an mfem::Mesh @a mesh, and an
+    mfem::IntegrationRule @a ir.
+
+    @param[in] VQ is the vector coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
+template <typename Context>
+void InitCoefficient(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
+                     const mfem::IntegrationRule &ir,
+                     Coefficient *&coeff_ptr, Context &ctx)
+{
+   if (VectorConstantCoefficient *const_coeff =
+          dynamic_cast<VectorConstantCoefficient*>(VQ))
+   {
+      const int vdim = const_coeff->GetVDim();
+      const mfem::Vector &val = const_coeff->GetVec();
+      Coefficient *ceedCoeff = new Coefficient(vdim);
+      for (int i = 0; i < vdim; i++)
+      {
+         ctx.coeff[i] = val[i];
+      }
+      coeff_ptr = ceedCoeff;
+   }
+   else if (VectorGridFunctionCoefficient* vgf_coeff =
+               dynamic_cast<VectorGridFunctionCoefficient*>(VQ))
+   {
+      GridCoefficient *ceedCoeff =
+         new GridCoefficient(*vgf_coeff->GetGridFunction());
+      coeff_ptr = ceedCoeff;
+   }
+   else if (VectorQuadratureFunctionCoefficient *cQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(VQ))
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(cQ->GetVDim());
+      const int dim = mesh.Dimension();
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      ceedCoeff->coeff.MakeRef(const_cast<mfem::QuadratureFunction &>(qFun),0);
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+   else
+   {
+      const int dim = mesh.Dimension();
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(dim);
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      ceedCoeff->coeff.SetSize(dim * nq * ne);
+      auto C = Reshape(ceedCoeff->coeff.HostWrite(), dim, nq, ne);
+      mfem::DenseMatrix Q_ir;
+      for (int e = 0; e < ne; ++e)
+      {
+         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
+         VQ->Eval(Q_ir, T, ir);
+         for (int q = 0; q < nq; ++q)
+         {
+            for (int i = 0; i < dim; ++i)
+            {
+               C(i,q,e) = Q_ir(i,q);
+            }
+         }
+      }
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+}
+
+/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
+    mfem::Coefficient @a Q, an mfem::Mesh @a mesh, and an mfem::IntegrationRule
+    @a ir for the elements given by the indices @a indices.
+
+    @param[in] Q is the coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
+template <typename Context>
+void InitCoefficientWithIndices(mfem::Coefficient *Q, mfem::Mesh &mesh,
+                                const mfem::IntegrationRule &ir,
+                                int nelem,
+                                const int* indices,
+                                Coefficient*& coeff_ptr, Context &ctx)
+{
+   if ( Q == nullptr )
+   {
+      Coefficient *ceedCoeff = new Coefficient(1);
+      ctx.coeff = 1.0;
+      coeff_ptr = ceedCoeff;
+   }
+   else if (ConstantCoefficient *const_coeff =
+               dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      Coefficient *ceedCoeff = new Coefficient(1);
+      ctx.coeff = const_coeff->constant;
+      coeff_ptr = ceedCoeff;
+   }
+   else if (GridFunctionCoefficient* gf_coeff =
+               dynamic_cast<GridFunctionCoefficient*>(Q))
+   {
+      GridCoefficient *ceedCoeff =
+         new GridCoefficient(*gf_coeff->GetGridFunction());
+      coeff_ptr = ceedCoeff;
+   }
+   else if (QuadratureFunctionCoefficient *cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      ceedCoeff->coeff.SetSize(nq * nelem);
+      Memory<int> m_indices((int*)indices, nelem, false);
+      auto in = Reshape(qFun.Read(), nq, ne);
+      auto d_indices = Read(m_indices, nelem);
+      auto out = Reshape(ceedCoeff->coeff.Write(), nq, nelem);
+      MFEM_FORALL(i, nelem * nq,
+      {
+         const int q = i%nq;
+         const int sub_e = i/nq;
+         const int e = d_indices[sub_e];
+         out(q, sub_e) = in(q, e);
+      });
+      m_indices.DeleteDevice();
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+   else
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
+      const int nq = ir.GetNPoints();
+      ceedCoeff->coeff.SetSize(nq * nelem);
+      auto C = Reshape(ceedCoeff->coeff.HostWrite(), nq, nelem);
+      for (int i = 0; i < nelem; ++i)
+      {
+         const int e = indices[i];
+         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q, i) = Q->Eval(T, ir.IntPoint(q));
+         }
+      }
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+}
+
+
+/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
+    mfem::VectorCoefficient @a Q, an mfem::Mesh @a mesh, and an
+    mfem::IntegrationRule @a ir for the elements given by the indices @a indices.
+
+    @param[in] VQ is the vector coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
+template <typename Context>
+void InitCoefficientWithIndices(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
+                                const mfem::IntegrationRule &ir,
+                                int nelem,
+                                const int* indices,
+                                Coefficient *&coeff_ptr, Context &ctx)
+{
+   if (VectorConstantCoefficient *const_coeff =
+          dynamic_cast<VectorConstantCoefficient*>(VQ))
+   {
+      const int vdim = const_coeff->GetVDim();
+      const mfem::Vector &val = const_coeff->GetVec();
+      Coefficient *ceedCoeff = new Coefficient(vdim);
+      for (int i = 0; i < vdim; i++)
+      {
+         ctx.coeff[i] = val[i];
+      }
+      coeff_ptr = ceedCoeff;
+   }
+   else if (VectorGridFunctionCoefficient* vgf_coeff =
+               dynamic_cast<VectorGridFunctionCoefficient*>(VQ))
+   {
+      GridCoefficient *ceedCoeff =
+         new GridCoefficient(*vgf_coeff->GetGridFunction());
+      coeff_ptr = ceedCoeff;
+   }
+   else if (VectorQuadratureFunctionCoefficient *cQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(VQ))
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(cQ->GetVDim());
+      const int dim = mesh.Dimension();
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      ceedCoeff->coeff.SetSize(dim * nq * nelem);
+      Memory<int> m_indices((int*)indices, nelem, false);
+      auto in = Reshape(qFun.Read(), dim, nq, ne);
+      auto d_indices = Read(m_indices, nelem);
+      auto out = Reshape(ceedCoeff->coeff.Write(), dim, nq, nelem);
+      MFEM_FORALL(i, nelem * nq,
+      {
+         const int q = i%nq;
+         const int sub_e = i/nq;
+         const int e = d_indices[sub_e];
+         for (int d = 0; d < dim; d++)
+         {
+            out(d, q, sub_e) = in(d, q, e);
+         }
+      });
+      m_indices.DeleteDevice();
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+   else
+   {
+      const int dim = mesh.Dimension();
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(dim);
+      const int nq = ir.GetNPoints();
+      ceedCoeff->coeff.SetSize(dim * nq * nelem);
+      auto C = Reshape(ceedCoeff->coeff.HostWrite(), dim, nq, nelem);
+      mfem::DenseMatrix Q_ir;
+      for (int i = 0; i < nelem; ++i)
+      {
+         const int e = indices[i];
+         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
+         VQ->Eval(Q_ir, T, ir);
+         for (int q = 0; q < nq; ++q)
+         {
+            for (int d = 0; d < dim; ++d)
+            {
+               C(d, q, i) = Q_ir(d, q);
+            }
+         }
+      }
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+}
+
+template <typename Coeff, typename Context>
+void InitCoefficient(Coeff *Q, mfem::Mesh &mesh,
+                     const mfem::IntegrationRule &ir, int nelem,
+                     const int* indices, Coefficient *&coeff_ptr, Context &ctx)
+{
+   if (indices)
+   {
+      InitCoefficientWithIndices(Q, mesh, ir, nelem, indices, coeff_ptr, ctx);
+   }
+   else
+   {
+      InitCoefficient(Q, mesh, ir, coeff_ptr, ctx);
+   }
+}
+
+} // namespace ceed
+
+} // namespace mfem
+
+#endif
+
+#endif // MFEM_LIBCEED_COEFF
@@ -9,14 +9,17 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.

-#ifndef MFEM_LIBCEED_PAINTEG
-#define MFEM_LIBCEED_PAINTEG
+#ifndef MFEM_LIBCEED_INTEG
+#define MFEM_LIBCEED_INTEG

+#include "../../../config/config.hpp"
+#include "../../fespace.hpp"
+#include "../../gridfunc.hpp"
 #include "operator.hpp"
-#include "../../config/config.hpp"
 #include "coefficient.hpp"
-#include "../fespace.hpp"
-#include "../gridfunc.hpp"
+#include "restriction.hpp"
+#include "util.hpp"
+#include "ceed.hpp"

 namespace mfem
 {
@@ -84,6 +87,7 @@ protected:
   CeedQFunctionContext build_ctx;
   CeedOperator build_oper;

+public:
   PAIntegrator()
      : Operator(),
        trial_basis(nullptr), test_basis(nullptr), mesh_basis(nullptr),
@@ -93,23 +97,51 @@ protected:
        qdata(nullptr), coeff(nullptr), build_ctx(nullptr), build_oper(nullptr)
   { }

-public:
-   /** This method assembles the PAIntegrator.
+   /** @brief This method assembles the `PAIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.

-       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
-                       the `CeedOperatorInfo` type is expected to inherit from
-                       `OperatorInfo` and contain a `Context` type relevant to
-                       the qFunctions.
-       @param[in] fes the `FiniteElementSpace` for the form,
-       @param[in] ir the `IntegrationRule` for the numerical integration,
-       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] Q is the coefficient from the `Integrator`. */
   template <typename CeedOperatorInfo, typename CoeffType>
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
                 CoeffType *Q)
   {
-      Assemble(info, fes, fes, irm, Q);
+      Assemble(info, fes, ir, fes.GetNE(), nullptr, Q);
+   }
+
+   /** @brief This method assembles the `PAIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q for the elements given by the indices
+       @a indices.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.
+
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] nelem The number of elements.
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed.
+       @param[in] Q is the coefficient from the `Integrator`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
+                 CoeffType *Q)
+   {
+      Assemble(info, fes, fes, ir, nelem, indices, Q);
   }

   /** This method assembles the PAIntegrator for mixed forms.
@@ -126,12 +158,40 @@ public:
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &trial_fes,
                 const mfem::FiniteElementSpace &test_fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
+                 CoeffType *Q)
+   {
+      Assemble(info, trial_fes, test_fes, ir, trial_fes.GetNE(), nullptr, Q);
+   }
+
+   /** This method assembles the PAIntegrator for mixed forms on mixed meshes.
+
+       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
+                       the `CeedOperatorInfo` type is expected to inherit from
+                       `OperatorInfo` and contain a `Context` type relevant to
+                       the qFunctions.
+       @param[in] trial_fes the trial `FiniteElementSpace` for the form,
+       @param[in] test_fes the test `FiniteElementSpace` for the form,
+       @param[in] ir the `IntegrationRule` for the numerical integration,
+       @param[in] nelem The number of elements,
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed,
+       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &trial_fes,
+                 const mfem::FiniteElementSpace &test_fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
                 CoeffType *Q)
   {
      Ceed ceed(internal::ceed);
      mfem::Mesh &mesh = *trial_fes.GetMesh();
-      InitCoefficient(Q, mesh, irm, coeff, info.ctx);
+      MFEM_VERIFY(!(!indices && mesh.GetNumGeometries(mesh.Dimension()) > 1),
+                  "Use ceed::MixedIntegrator on mixed meshes.");
+      InitCoefficient(Q, mesh, ir, nelem, indices, coeff, info.ctx);
      bool const_coeff = coeff->IsConstant();
      std::string build_func = const_coeff ? info.build_func_const
                               : info.build_func_quad;
@@ -143,7 +203,6 @@ public:
                     info.trial_op,
                     info.test_op
                    };
-      CeedInt nqpts, nelem = mesh.GetNE();
      CeedInt dim = mesh.SpaceDimension();
      CeedInt trial_vdim = trial_fes.GetVDim();
      CeedInt test_vdim = test_fes.GetVDim();
@@ -151,23 +210,23 @@ public:
      mesh.EnsureNodes();
      if ( &trial_fes == &test_fes )
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
-                                 &trial_basis, &trial_restr);
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices,
+                                 ceed, &trial_basis, &trial_restr);
         test_basis = trial_basis;
         test_restr = trial_restr;
      }
      else
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
-                                 &trial_basis, &trial_restr);
-         InitBasisAndRestriction(test_fes, irm, ceed,
-                                 &test_basis, &test_restr);
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices,
+                                 ceed, &trial_basis, &trial_restr);
+         InitBasisAndRestriction(test_fes, ir, nelem, indices,
+                                 ceed, &test_basis, &test_restr);
      }

      const mfem::FiniteElementSpace *mesh_fes = mesh.GetNodalFESpace();
      MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
-      InitBasisAndRestriction(*mesh_fes, irm, ceed, &mesh_basis,
-                              &mesh_restr);
+      InitBasisAndRestriction(*mesh_fes, ir, nelem, indices,
+                              ceed, &mesh_basis, &mesh_restr);

      CeedInt trial_nqpts, test_nqpts;
      CeedBasisGetNumQuadraturePoints(trial_basis, &trial_nqpts);
@@ -175,7 +234,7 @@ public:
      MFEM_VERIFY(trial_nqpts == test_nqpts,
                  "Trial and test basis must have the same number of quadrature"
                  " points.");
-      nqpts = trial_nqpts;
+      CeedInt nqpts = trial_nqpts;

      const int qdatasize = op.qdatasize;
      InitStridedRestriction(*mesh_fes, nelem, nqpts, qdatasize,
@@ -219,8 +278,10 @@ public:
      CeedOperatorCreate(ceed, build_qfunc, NULL, NULL, &build_oper);
      if (GridCoefficient *gridCoeff = dynamic_cast<GridCoefficient*>(coeff))
      {
-         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), irm, ceed,
-                                 &gridCoeff->basis, &gridCoeff->restr);
+         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), ir,
+                                 nelem, indices, ceed,
+                                 &gridCoeff->basis,
+                                 &gridCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", gridCoeff->restr,
                              gridCoeff->basis, gridCoeff->coeffVector);
      }
@@ -229,7 +290,8 @@ public:
      {
         const int ncomp = quadCoeff->ncomp;
         CeedInt strides[3] = {ncomp, 1, ncomp*nqpts};
-         InitStridedRestriction(*mesh_fes, nelem, nqpts, ncomp, strides,
+         InitStridedRestriction(*mesh.GetNodalFESpace(),
+                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", quadCoeff->restr,
                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
@@ -252,22 +314,17 @@ public:
      switch (op.trial_op)
      {
         case EvalMode::None:
-            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim,
-                                  CEED_EVAL_NONE);
+            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim, CEED_EVAL_NONE);
            break;
         case EvalMode::Interp:
-            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim,
-                                  CEED_EVAL_INTERP);
+            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim, CEED_EVAL_INTERP);
            break;
         case EvalMode::Grad:
-            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim,
-                                  CEED_EVAL_GRAD);
+            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim, CEED_EVAL_GRAD);
            break;
         case EvalMode::InterpAndGrad:
-            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim,
-                                  CEED_EVAL_INTERP);
-            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim,
-                                  CEED_EVAL_GRAD);
+            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim, CEED_EVAL_INTERP);
+            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim, CEED_EVAL_GRAD);
            break;
      }
      // qdata
@@ -276,22 +333,17 @@ public:
      switch (op.test_op)
      {
         case EvalMode::None:
-            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim,
-                                   CEED_EVAL_NONE);
+            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim, CEED_EVAL_NONE);
            break;
         case EvalMode::Interp:
-            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim,
-                                   CEED_EVAL_INTERP);
+            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim, CEED_EVAL_INTERP);
            break;
         case EvalMode::Grad:
-            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim,
-                                   CEED_EVAL_GRAD);
+            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim, CEED_EVAL_GRAD);
            break;
         case EvalMode::InterpAndGrad:
-            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim,
-                                   CEED_EVAL_INTERP);
-            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim,
-                                   CEED_EVAL_GRAD);
+            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim, CEED_EVAL_INTERP);
+            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim, CEED_EVAL_GRAD);
            break;
      }
      CeedQFunctionSetContext(apply_qfunc, build_ctx);
@@ -306,18 +358,14 @@ public:
                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
-            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Grad:
-            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::InterpAndGrad:
-            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
-            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
            break;
      }
      // qdata
@@ -331,18 +379,14 @@ public:
                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
-            CeedOperatorSetField(oper, "v", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Grad:
-            CeedOperatorSetField(oper, "gv", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gv", test_restr, test_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::InterpAndGrad:
-            CeedOperatorSetField(oper, "v", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
-            CeedOperatorSetField(oper, "gv", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gv", test_restr, test_basis, CEED_VECTOR_ACTIVE);
            break;
      }

@@ -400,6 +444,7 @@ protected:
   Coefficient *coeff;
   CeedQFunctionContext build_ctx;

+public:
   MFIntegrator()
      : Operator(),
        trial_basis(nullptr), test_basis(nullptr), mesh_basis(nullptr),
@@ -408,23 +453,51 @@ protected:
        apply_qfunc(nullptr), node_coords(nullptr),
        qdata(nullptr), coeff(nullptr), build_ctx(nullptr) { }

-public:
-   /** This method assembles the MFIntegrator.
+   /** @brief This method assembles the `MFIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.

-       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
-                       the `CeedOperatorInfo` type is expected to inherit from
-                       `OperatorInfo` and contain a `Context` type relevant to
-                       the qFunctions.
-       @param[in] fes the `FiniteElementSpace` for the form,
-       @param[in] ir the `IntegrationRule` for the numerical integration,
-       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] Q is the coefficient from the `Integrator`. */
   template <typename CeedOperatorInfo, typename CoeffType>
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
                 CoeffType *Q)
   {
-      Assemble(info, fes, fes, irm, Q);
+      Assemble(info, fes, ir, fes.GetNE(), nullptr, Q);
+   }
+
+   /** @brief This method assembles the `MFIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q for the elements given by the indices
+       @a indices.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.
+
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] nelem The number of elements.
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed.
+       @param[in] Q is the coefficient from the `Integrator`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
+                 CoeffType *Q)
+   {
+      Assemble(info, fes, fes, ir, nelem, indices, Q);
   }

   /** This method assembles the MFIntegrator for mixed forms.
@@ -441,12 +514,40 @@ public:
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &trial_fes,
                 const mfem::FiniteElementSpace &test_fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
+                 CoeffType *Q)
+   {
+      Assemble(info, trial_fes, test_fes, ir, trial_fes.GetNE(), nullptr, Q);
+   }
+
+   /** This method assembles the MFIntegrator for mixed forms.
+
+       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
+                       the `CeedOperatorInfo` type is expected to inherit from
+                       `OperatorInfo` and contain a `Context` type relevant to
+                       the qFunctions.
+       @param[in] trial_fes the trial `FiniteElementSpace` for the form,
+       @param[in] test_fes the test `FiniteElementSpace` for the form,
+       @param[in] ir the `IntegrationRule` for the numerical integration,
+       @param[in] nelem The number of elements,
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed,
+       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &trial_fes,
+                 const mfem::FiniteElementSpace &test_fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
                 CoeffType *Q)
   {
      Ceed ceed(internal::ceed);
      Mesh &mesh = *trial_fes.GetMesh();
-      InitCoefficient(Q, mesh, irm, coeff, info.ctx);
+      MFEM_VERIFY(!(!indices && mesh.GetNumGeometries(mesh.Dimension()) > 1),
+                  "Use ceed::MixedIntegrator on mixed meshes.");
+      InitCoefficient(Q, mesh, ir, nelem, indices, coeff, info.ctx);
      bool const_coeff = coeff->IsConstant();
      std::string apply_func = const_coeff ? info.apply_func_mf_const
                               : info.apply_func_mf_quad;
@@ -457,7 +558,7 @@ public:
                     info.trial_op,
                     info.test_op
                    };
-      CeedInt nqpts, nelem = mesh.GetNE();
+
      CeedInt dim = mesh.SpaceDimension();
      CeedInt trial_vdim = trial_fes.GetVDim();
      CeedInt test_vdim = test_fes.GetVDim();
@@ -465,22 +566,22 @@ public:
      mesh.EnsureNodes();
      if ( &trial_fes == &test_fes )
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices, ceed,
                                 &trial_basis, &trial_restr);
         test_basis = trial_basis;
         test_restr = trial_restr;
      }
      else
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices, ceed,
                                 &trial_basis, &trial_restr);
-         InitBasisAndRestriction(test_fes, irm, ceed,
+         InitBasisAndRestriction(test_fes, ir, nelem, indices, ceed,
                                 &test_basis, &test_restr);
      }

      const mfem::FiniteElementSpace *mesh_fes = mesh.GetNodalFESpace();
      MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
-      InitBasisAndRestriction(*mesh_fes, irm, ceed, &mesh_basis,
+      InitBasisAndRestriction(*mesh_fes, ir, nelem, indices, ceed, &mesh_basis,
                              &mesh_restr);

      CeedInt trial_nqpts, test_nqpts;
@@ -489,7 +590,7 @@ public:
      MFEM_VERIFY(trial_nqpts == test_nqpts,
                  "Trial and test basis must have the same number of quadrature"
                  " points.");
-      nqpts = trial_nqpts;
+      CeedInt nqpts = trial_nqpts;

      InitVector(*mesh.GetNodes(), node_coords);

@@ -570,8 +671,8 @@ public:
      // coefficient
      if (GridCoefficient *gridCoeff = dynamic_cast<GridCoefficient*>(coeff))
      {
-         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), irm, ceed,
-                                 &gridCoeff->basis, &gridCoeff->restr);
+         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), ir, nelem, indices,
+                                 ceed, &gridCoeff->basis, &gridCoeff->restr);
         CeedOperatorSetField(oper, "coeff", gridCoeff->restr,
                              gridCoeff->basis, gridCoeff->coeffVector);
      }
@@ -673,4 +774,4 @@ private:

 } // namespace mfem

-#endif // MFEM_LIBCEED_PAINTEG
+#endif // MFEM_LIBCEED_INTEG
@@ -0,0 +1,32 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIBCEED_INTERFACE
+#define MFEM_LIBCEED_INTERFACE
+
+// Object wrapping a CeedOperator in a mfem::Operator.
+#include "operator.hpp"
+// Functions to initialize CeedBasis objects.
+#include "basis.hpp"
+// Functions to initialize CeedRestriction objects.
+#include "restriction.hpp"
+// Functions to initialize coefficients.
+#include "coefficient.hpp"
+// PA or MF Operator using libCEED.
+#include "integrator.hpp"
+// PA Operator supporting mixed finite element spaces.
+#include "mixed_integrator.hpp"
+// Utility functions
+#include "util.hpp"
+// Wrapper to include <ceed.h>
+#include "ceed.hpp"
+
+#endif // MFEM_LIBCEED_INTERFACE
@@ -0,0 +1,126 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIBCEED_MIXED_INTEGRATOR
+#define MFEM_LIBCEED_MIXED_INTEGRATOR
+
+#include "ceed.hpp"
+#include "integrator.hpp"
+#include <unordered_map>
+
+namespace mfem
+{
+
+namespace ceed
+{
+
+/** @brief This class wraps a `ceed::PAIntegrator` or `ceed::MFIntegrator` to
+    support mixed finite element spaces. */
+template <typename CeedInteg>
+class MixedIntegrator : public ceed::Operator
+{
+#ifdef MFEM_USE_CEED
+   using ElementKey = std::pair<int, int>; //< Element::Type, Order >
+   struct key_hash
+   {
+      std::size_t operator()(const ElementKey& k) const
+      {
+         return k.first + 2 * k.second;
+      }
+   };
+   using ElementsMap = std::unordered_map<const ElementKey, int*, key_hash>;
+   std::vector<CeedInteg*> sub_ops;
+
+public:
+   template <typename Integrator, typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(const Integrator &integ,
+                 CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &fes,
+                 CoeffType *Q)
+   {
+      ElementsMap count;
+      ElementsMap element_indices;
+      ElementsMap offsets;
+
+      // Count the number of elements of each type
+      for (int i = 0; i < fes.GetNE(); i++)
+      {
+         ElementKey key(fes.GetElementType(i), fes.GetElementOrder(i));
+         auto value = count.find(key);
+         if (value == count.end())
+         {
+            count[key] = new int(1);
+         }
+         else
+         {
+            (*value->second)++;
+         }
+      }
+
+      // Initialization of the arrays
+      for ( const auto& value : count )
+      {
+         element_indices[value.first] = new int[*value.second];
+         offsets[value.first] = new int(0);
+      }
+
+      // Populates the indices arrays for each element type
+      for (int i = 0; i < fes.GetNE(); i++)
+      {
+         ElementKey key(fes.GetElementType(i), fes.GetElementOrder(i));
+         int &offset = *(offsets[key]);
+         int* indices_array = element_indices[key];
+         indices_array[offset] = i;
+         offset++;
+      }
+
+      // Create composite CeedOperator
+      CeedCompositeOperatorCreate(internal::ceed, &oper);
+
+      // Create each sub-CeedOperator
+      sub_ops.reserve(element_indices.size());
+      for (const auto& value : element_indices)
+      {
+         const int* indices = value.second;
+         const int first_index = indices[0];
+         const mfem::FiniteElement &el = *fes.GetFE(first_index);
+         auto &T = *fes.GetMesh()->GetElementTransformation(first_index);
+         MFEM_ASSERT(!integ.GetIntegrationRule(),
+                     "Mixed mesh integrators should not have an"
+                     " IntegrationRule.");
+         const IntegrationRule &ir = GetRule(integ, el, el, T);
+         auto sub_op = new CeedInteg();
+         int nelem = *count[value.first];
+         sub_op->Assemble(info, fes, ir, nelem, indices, Q);
+         sub_ops.push_back(sub_op);
+         CeedCompositeOperatorAddSub(oper, sub_op->GetCeedOperator());
+      }
+
+      const int ndofs = fes.GetVDim() * fes.GetNDofs();
+      CeedVectorCreate(internal::ceed, ndofs, &u);
+      CeedVectorCreate(internal::ceed, ndofs, &v);
+   }
+
+   virtual ~MixedIntegrator()
+   {
+      for (auto sub_op : sub_ops)
+      {
+         delete sub_op;
+      }
+   }
+#endif
+};
+
+} // namespace ceed
+
+} // namespace mfem
+
+#endif // MFEM_LIBCEED_MIXED_INTEGRATOR
@@ -11,9 +11,11 @@

 #include "operator.hpp"

-#include "../../config/config.hpp"
-#include "../../linalg/vector.hpp"
-#include "../fespace.hpp"
+#include "../../../config/config.hpp"
+#include "../../../linalg/vector.hpp"
+#include "../../fespace.hpp"
+#include "util.hpp"
+#include "ceed.hpp"

 namespace mfem
 {
@@ -12,8 +12,8 @@
 #ifndef MFEM_LIBCEED_OPERATOR
 #define MFEM_LIBCEED_OPERATOR

-#include "util.hpp"
-#include "../../linalg/operator.hpp"
+#include "../../../linalg/operator.hpp"
+#include "ceed.hpp"

 namespace mfem
 {
@@ -41,7 +41,6 @@ public:
   void AddMult(const mfem::Vector &x, mfem::Vector &y) const;
   void GetDiagonal(mfem::Vector &diag) const;
   using mfem::Operator::SetupRAP;
-
   virtual ~Operator()
   {
 #ifdef MFEM_USE_CEED
@@ -0,0 +1,301 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "../../../fem/gridfunc.hpp"
+#include "ceed.hpp"
+
+namespace mfem
+{
+
+namespace ceed
+{
+
+#ifdef MFEM_USE_CEED
+
+static void InitNativeRestr(const mfem::FiniteElementSpace &fes,
+                            Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(0);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   const mfem::Table &el_dof = fes.GetElementToDofTable();
+   mfem::Array<int> tp_el_dof(el_dof.Size_of_connections());
+   const mfem::TensorBasisElement * tfe =
+      dynamic_cast<const mfem::TensorBasisElement *>(fe);
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
+   const mfem::Array<int>& dof_map = tfe->GetDofMap();
+
+   for (int i = 0; i < fes.GetNE(); i++)
+   {
+      const int el_offset = P * i;
+      for (int j = 0; j < P; j++)
+      {
+         tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[dof_map[j]+el_offset];
+      }
+   }
+
+   CeedElemRestrictionCreate(ceed, fes.GetNE(), P, fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+static void InitLexicoRestr(const mfem::FiniteElementSpace &fes,
+                            Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(0);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   const mfem::Table &el_dof = fes.GetElementToDofTable();
+   mfem::Array<int> tp_el_dof(el_dof.Size_of_connections());
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
+
+   for (int e = 0; e < fes.GetNE(); e++)
+   {
+      for (int i = 0; i < P; i++)
+      {
+         tp_el_dof[i + e*P] = stride*el_dof.GetJ()[i + e*P];
+      }
+   }
+
+   CeedElemRestrictionCreate(ceed, fes.GetNE(), P, fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+static void InitRestrictionImpl(const mfem::FiniteElementSpace &fes,
+                                Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(0);
+   const mfem::TensorBasisElement * tfe =
+      dynamic_cast<const mfem::TensorBasisElement *>(fe);
+   if ( tfe && tfe->GetDofMap().Size()>0 ) // Native ordering using dof_map
+   {
+      InitNativeRestr(fes, ceed, restr);
+   }
+   else  // Lexicographic ordering
+   {
+      InitLexicoRestr(fes, ceed, restr);
+   }
+}
+
+static void InitNativeRestrWithIndices(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   mfem::Array<int> tp_el_dof(nelem*P);
+   const mfem::TensorBasisElement * tfe =
+      dynamic_cast<const mfem::TensorBasisElement *>(fe);
+   Array<int> dofs;
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
+   const mfem::Array<int>& dof_map = tfe->GetDofMap();
+
+   for (int i = 0; i < nelem; i++)
+   {
+      const int elem_index = indices[i];
+      fes.GetElementDofs(elem_index, dofs);
+      const int el_offset = P * i;
+      for (int j = 0; j < P; j++)
+      {
+         tp_el_dof[j + el_offset] = stride*dofs[dof_map[j]];
+      }
+   }
+
+   CeedElemRestrictionCreate(ceed, nelem, P, fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+static void InitLexicoRestrWithIndices(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   mfem::Array<int> tp_el_dof(nelem*P);
+   Array<int> dofs;
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
+
+   for (int i = 0; i < nelem; i++)
+   {
+      const int elem_index = indices[i];
+      fes.GetElementDofs(elem_index, dofs);
+      const int el_offset = P * i;
+      for (int j = 0; j < P; j++)
+      {
+         tp_el_dof[j + el_offset] = stride*dofs[j];
+      }
+   }
+
+   CeedElemRestrictionCreate(ceed, nelem, P, fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+static void InitRestrictionWithIndicesImpl(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const mfem::TensorBasisElement * tfe =
+      dynamic_cast<const mfem::TensorBasisElement *>(fe);
+   if ( tfe && tfe->GetDofMap().Size()>0 ) // Native ordering using dof_map
+   {
+      InitNativeRestrWithIndices(fes, nelem, indices, ceed, restr);
+   }
+   else  // Lexicographic ordering
+   {
+      InitLexicoRestrWithIndices(fes, nelem, indices, ceed, restr);
+   }
+}
+
+static void InitCoeffRestrictionWithIndicesImpl(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   int nquads,
+   int ncomp,
+   Ceed ceed,
+   CeedElemRestriction *restr)
+{
+   mfem::Array<int> tp_el_dof(nelem*nquads);
+   const int stride_quad = ncomp;
+   const int stride_elem = ncomp*nquads;
+   // TODO generalize to support different #quads
+   for (int i = 0; i < nelem; i++)
+   {
+      const int elem_index = indices[i];
+      const int el_offset = elem_index * stride_elem;
+      for (int j = 0; j < nquads; j++)
+      {
+         tp_el_dof[j + nquads * i] = j * stride_quad + el_offset;
+      }
+   }
+   CeedElemRestrictionCreate(ceed, nelem, nquads, ncomp, 1,
+                             ncomp*fes.GetNE()*nquads,
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+void InitStridedRestriction(const mfem::FiniteElementSpace &fes,
+                            CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
+                            const CeedInt *strides,
+                            CeedElemRestriction *restr)
+{
+   RestrKey restr_key(&fes, nelem, nqpts, qdatasize, restr_type::Strided);
+   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
+   if (restr_itr == mfem::internal::ceed_restr_map.end())
+   {
+      CeedElemRestrictionCreateStrided(mfem::internal::ceed, nelem, nqpts, qdatasize,
+                                       nelem*nqpts*qdatasize,
+                                       strides,
+                                       restr);
+      // Will be automatically destroyed when @a fes gets destroyed.
+      mfem::internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
+   }
+}
+
+void InitRestriction(const FiniteElementSpace &fes,
+                     Ceed ceed,
+                     CeedElemRestriction *restr)
+{
+   // Check for FES -> basis, restriction in hash tables
+   const mfem::FiniteElement *fe = fes.GetFE(0);
+   const int P = fe->GetDof();
+   const int nelem = fes.GetNE();
+   const int ncomp = fes.GetVDim();
+   RestrKey restr_key(&fes, nelem, P, ncomp, restr_type::Standard);
+   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
+
+   // Init or retreive key values
+   if (restr_itr == mfem::internal::ceed_restr_map.end())
+   {
+      InitRestrictionImpl(fes, ceed, restr);
+      mfem::internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
+   }
+}
+
+void InitRestrictionWithIndices(const FiniteElementSpace &fes,
+                                int nelem,
+                                const int* indices,
+                                Ceed ceed,
+                                CeedElemRestriction *restr)
+{
+   // Check for FES -> basis, restriction in hash tables
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const int P = fe->GetDof();
+   const int ncomp = fes.GetVDim();
+   RestrKey restr_key(&fes, nelem, P, ncomp, restr_type::Standard);
+   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
+
+   // Init or retreive key values
+   if (restr_itr == mfem::internal::ceed_restr_map.end())
+   {
+      InitRestrictionWithIndicesImpl(fes, nelem, indices, ceed, restr);
+      mfem::internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
+   }
+}
+
+void InitCoeffRestrictionWithIndices(const FiniteElementSpace &fes,
+                                     int nelem,
+                                     const int* indices,
+                                     int nquads,
+                                     int ncomp,
+                                     Ceed ceed,
+                                     CeedElemRestriction *restr)
+{
+   // Check for FES -> basis, restriction in hash tables
+   RestrKey restr_key(&fes, nelem, nquads, ncomp, restr_type::Coeff);
+   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
+
+   // Init or retreive key values
+   if (restr_itr == mfem::internal::ceed_restr_map.end())
+   {
+      InitCoeffRestrictionWithIndicesImpl(fes, nelem, indices, nquads, ncomp,
+                                          ceed, restr);
+      mfem::internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
+   }
+}
+
+#endif
+
+} // namespace ceed
+
+} // namespace mfem
--- a/Show More
+++ b/Show More