L2-projection

update before changing branches
modifications to pobstacle_mixed.cpp to match paper
2023-06-15 12:27:24 -04:00 · 2023-01-05 16:36:56 -05:00 · 2023-01-05 16:31:40 -05:00 · 2022-12-30 16:23:51 -05:00 · 2022-12-29 17:21:36 -05:00 · 2022-12-27 11:28:03 -05:00
395 changed files with 290706 additions and 5602 deletions
@@ -23,9 +23,8 @@ install:
 - set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
 - set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include

-# Install METIS, use a mirror because the original source server is not always
-# up. Original url:
-#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
+# Install METIS, use MFEM's mirror because the original source server is often
+# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0
@@ -1,4 +1,4 @@
-name: Build Deploy Container
+name: "Docker"

 on:

@@ -20,6 +20,7 @@ on:

 jobs:
  build:
+    if: github.repository == 'mfem/mfem' # Don't run in forks
    permissions:
      packages: write
    strategy:
@@ -27,7 +28,8 @@ jobs:
      matrix:

        # Dockerfiles to build, a matrix supports future expanded builds
-        container: [["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu-base"]]
+        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
+                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]

    runs-on: ubuntu-latest
    name: Build
@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.

 # In this CI section, we build different variants of mfem and run test on them.
-name: builds-and-tests
+name: "Tests"

 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -47,17 +47,17 @@ jobs:
  builds-and-tests:
    strategy:
      matrix:
-        os: [ubuntu-20.04, macos-10.15, windows-2022]
+        os: [ubuntu-latest, macos-latest, windows-latest]
        target: [dbg, opt]
        mpi: [seq, par]
        build-system: [make, cmake]
        hypre-target: [int32]
        exclude:
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
            build-system: cmake
-          - os: macos-10.15
+          - os: macos-latest
            build-system: cmake
-          - os: windows-2022
+          - os: windows-latest
            build-system: make
        # 'include' allows us to:
        # - Add a variable to all jobs without creating a new matrix dimension.
@@ -72,15 +72,15 @@ jobs:
            codecov: NO
          - target: opt
            codecov: YES
-          - os: windows-2022
+          - os: windows-latest
            codecov: NO
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
            build-system: cmake
            hypre-target: int32
-          - os: ubuntu-20.04
+          - os: ubuntu-latest
            target: opt
            codecov: NO
            mpi: par
@@ -112,35 +112,35 @@ jobs:
    # TODO: It would be nice to have only one step, e.g. with a dedicated
    # action, but I (@adrienbernede) don't see how at the moment.
    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install mpich libmpich-dev
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
      run: |
        sudo apt-get install lcov

    - name: Set up Homebrew
-      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3

    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install openmpi
        export MAKE_CXX_FLAG="MPICXX=mpic++"

    - name: get MPI (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
      run: |
        export HOMEBREW_NO_INSTALL_CLEANUP=1
        brew install lcov

    - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
      uses: mpi4py/setup-mpi@v1.0.3

    # Get Hypre through cache, or build it.
@@ -154,7 +154,7 @@ jobs:
        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2

    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
@@ -163,7 +163,7 @@ jobs:
        build-system: make

    - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
      uses: mfem/github-actions/build-hypre@v2.2
      with:
        archive: ${{ env.HYPRE_ARCHIVE }}
@@ -175,14 +175,14 @@ jobs:
    # Install will only run on cache miss.
    - name: cache metis
      id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
      uses: actions/cache@v2
      with:
        path: ${{ env.METIS_TOP_DIR }}
        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

    - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
      uses: mfem/github-actions/build-metis@v2.2
      with:
        archive: ${{ env.METIS_ARCHIVE }}
@@ -196,16 +196,16 @@ jobs:
        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1

    - name: prepare binary cache location
-      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
      run: |
        mkdir -p vcpkg_cache

    - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
      env:
        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
      run: |
-        vcpkg install metis --triplet=x64-windows-static
+        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports

    # MFEM build and test
    - name: build
@@ -248,7 +248,7 @@ jobs:
      shell: bash

    - name: cmake unit tests (Ubuntu 20.04)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
@@ -256,7 +256,7 @@ jobs:
      shell: bash

    - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
      run: |
        CTEST_CONFIG="Release"
        cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "Static Analysis"
+
+on:
+  push:
+    branches: [ "master", "next"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'cpp' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: build-analysis
+name: "Build Analysis"

 permissions:
  actions: write
@@ -31,7 +31,7 @@ env:

 jobs:
  gitignore:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest

    steps:
    - name: Cancel Previous Runs
@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.

-name: repo-check
+name: "Checks"

 permissions:
  actions: write
@@ -28,7 +28,7 @@ on:

 jobs:
  file-headers-check:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -78,7 +78,7 @@ jobs:
        exit 1

  code-style:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -88,14 +88,14 @@ jobs:

    - name: get astyle
      run: |
-        sudo apt-get install astyle=3.1-1ubuntu2
+        sudo apt-get install astyle

    - name: style check
      run: |
        ./config/githooks/pre-push --style

  documentation:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: |
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
@@ -106,6 +106,8 @@ jobs:
    - name: get doxygen and graphviz
      run: |
        sudo apt-get install doxygen graphviz
+        cd doc
+        doxygen -u CodeDocumentation.conf.in

    - name: build documentation
      run: |
@@ -118,7 +120,7 @@ jobs:
      github.ref != 'refs/heads/master' &&
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    steps:
    - name: checkout mfem
      uses: actions/checkout@v2
@@ -307,6 +307,9 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh

+miniapps/multidomain/multidomain
+miniapps/hooke/hooke
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests
@@ -10,20 +10,27 @@

 Version 4.4.1 (development)
 ===========================
- Added example for body-fitted volumetric and shape integration using the 
-  Algoim library.

- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
-  spatial Gaussian white noise.
+Meshing improvements
+--------------------
+- Added support for mixed meshes and pyramids in GSLIB-FindPoints.

- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
-  See fem/estimators.hpp.
+- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
+  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
+  and work with all existing MFEM interfaces like finite element spaces etc.

- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
-  which provides parallel non-conforming, non-matching, variational, volumetric
-  mesh information transfer. With ParMortarAssember, fields can be exchanged
-  between arbitrarily distributed and unrelated finite element meshes in a
-  variationally consistent way.
+Discretization improvements
+---------------------------
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration with arbitrary user-supplied coefficients.
+
+- Added support for partial assembly and fully matrix-free operators on mixed
+  meshes (different element types and p-adaptivity) through libCEED, including
+  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
+  currently limited by MFEM capabilities, i.e. 2D serial meshes. All mixed
+  element topologies are supported in serial and parallel: segment, triangle,
+  square, tetrahedron, cube, prism, and pyramid.

 - Added full assembly and device support for several LinearForm integrators:
  * DomainLF: (f, v)
@@ -31,18 +38,79 @@ Version 4.4.1 (development)
  * DomainLFGrad: (f, grad(v))
  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))

+- Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
+  spatial Gaussian white noise.
+
+- Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
+  See fem/estimators.hpp.
+
+- Fixes and improvements in LinearFormExtension.
+
+- Added a new class FaceQuadratureSpace that allows for the construction of
+  QuadratureFunctions on the interior or boundary faces of a mesh.
+
+- Added a class CoefficientVector for efficient access of variable coefficient
+  values at quadrature points (in particular for GPU/device kernels).
+
+Linear and nonlinear solvers
+----------------------------
+- Added a new class DGMassInverse that performs a local elementwise CG
+  iteration to solve systems involving the discontinuous Galerkin mass matrix,
+  including support for device/GPU acceleration.
+
+- Added more flexibility to the constraint solver classes:
+  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
+    (necessary for penalty contact)
+  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
+  * All constraint solver classes can take a user-defined preconditioner
+
+ - Added functions to toggle additional options for the SuperLU_Dist and Hypre
+   preconditioners (ParaSails, Euclid, ILU).
+
+New and updated examples and miniapps
+-------------------------------------
+- Added a new elasticity miniapp, Hooke, that showcases a low-level approach of
+  using MFEM to solve a nonlinear elasticity problem based on the fundamental
+  finite element operator decomposition. The miniapp also integrates with
+  automatic differentiation tools like a native dual number implementation or a
+  third party library such as Enzyme. See miniapps/elasticity for more details.
+
 - Add a new example code, Example 33/33p, to demonstrate the solution of
  spectral fractional PDEs with MFEM.

+Integrations, testing and documentation
+---------------------------------------
 - Added a Dockerfile for a simple MFEM container, see config/docker/README.md.

- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration.
+- Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
+  which provides parallel non-conforming, non-matching, variational, volumetric
+  mesh information transfer. With ParMortarAssember, fields can be exchanged
+  between arbitrarily distributed and unrelated finite element meshes in a
+  variationally consistent way.
+
+- Added support for the LLVM-based automatic differentiation tool Enzyme, see
+  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
+  header are provided. The functionality and interaction are demonstrated in a
+  new miniapp in miniapps/elasticity.
+
+- Added example for body-fitted volumetric and shape integration using the
+  Algoim library.

 - Added Windows 2022 CI testing with GitHub actions.

- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
+Miscellaneous
+-------------
+- Various other simplifications, extensions, and bugfixes in the code.
+
+- Added boundary elimination with device support for `SparseMatrix` and
+  `HypreParMatrix`.
+
+- When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
+  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
+  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).
+
+- Added TMOP metrics for mesh untangling and worst-case quality improvement.
+

 Version 4.4, released on March 21, 2022
 =======================================
@@ -75,6 +143,11 @@ Meshing improvements
 - Added a simpler interface to access mesh face information, see FaceInformation
  and GetFaceInformation in the Mesh class.

+- Added the method ParMesh::GetSerialMesh() that reconstructs a partitioned
+  parallel mesh on a given single rank. Also, added the method
+  ParMesh::PrintAsSerial() that saves the reconstructed serial mesh to a C++
+  stream on rank 0.
+
 - Gmsh meshes where all elements have zero physical tag (the default Gmsh output
  format if no physical groups are defined) are now successfully loaded, and
  elements are reassigned attribute number 1.
@@ -177,6 +250,13 @@ Miscellaneous

 - Fixed several MinGW build issues on Windows.

+- In various places in the library, replace the use of 'long' with 'long long'
+  to better support Win64 builds where 'long' is 32-bit and 'long long' is
+  64-bit. On Linux and MacOS, both types are typically 64-bit.
+
+- Update various "MemoryUsage" methods to return 'std::size_t' instead of 'long'
+  since the latter is 32-bit in Win64 builds.
+
 - Added 'double' atomicAdd implementation for previous versions of CUDA.

 - HypreParVector and Vector now support C++ move semantics, and the copy
@@ -136,6 +136,8 @@ if (MFEM_USE_CUDA)
       "CUDA flags set for MFEM" FORCE)
   set(CUSPARSE_FOUND TRUE)
   set(CUSPARSE_LIBRARIES "cusparse")
+   set(CUBLAS_FOUND TRUE)
+   set(CUSBLAS_LIBRARIES "cublas")
 endif()

 if (XSDK_ENABLE_C)
@@ -452,6 +454,11 @@ if (MFEM_USE_PARELAG)
   find_package(PARELAG REQUIRED)
 endif()

+# Enzyme
+if (MFEM_USE_ENZYME)
+   find_package(ENZYME REQUIRED)
+endif()
+
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
  if (APPLE)
@@ -470,16 +477,25 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
  endif()
 endif()

+# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
+# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
+#   Target "mfem" links to target "Threads::Threads" but the target was not
+#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
+#   an ALIAS target is missing?
+# Call Stack (most recent call first):
+#   CMakeLists.txt:474 (mfem_add_library)
+find_package(Threads REQUIRED)
+
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist METIS SuiteSparse SUNDIALS
    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM)
+    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
+    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -102,7 +102,9 @@ The MFEM source code has the following structure:
  .
  ├── config
  │   ├── cmake
-  │   └── githooks
+  │   ├── docker
+  │   ├── githooks
+  │   └── vcpkg
  ├── data
  ├── doc
  ├── examples
@@ -111,6 +113,7 @@ The MFEM source code has the following structure:
  │   ├── ginkgo
  │   ├── hiop
  │   ├── jupyter
+  │   ├── moonolith
  │   ├── petsc
  │   ├── pumi
  │   ├── sundials
@@ -118,21 +121,25 @@ The MFEM source code has the following structure:
  ├── fem
  │   ├── ceed
  │   ├── fe
-  │   ├── qinterp
+  │   ├── lor
  │   ├── moonolith
+  │   ├── qinterp
  │   └── tmop
  ├── general
  ├── linalg
  │   └── simd
  ├── mesh
+  │   └── submesh
  ├── miniapps
  │   ├── adjoint
  │   ├── autodiff
  │   ├── common
  │   ├── electromagnetics
  │   ├── gslib
+  │   ├── hooke
  │   ├── meshing
  │   ├── mtop
+  │   ├── multidomain
  │   ├── navier
  │   ├── nurbs
  │   ├── parelag
@@ -16,7 +16,11 @@ requires an MPI C++ compiler, as well as the following external libraries:
  https://github.com/hypre-space/hypre

 - METIS (a family of multilevel partitioning algorithms)
-  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
+  https://github.com/mfem/tpls
+
+  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
+  webpage, https://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
+  and we don't support yet the new repo https://github.com/KarypisLab/METIS.

 The hypre dependency can be downloaded as a tarball from GitHub or from the
 project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
@@ -472,10 +476,10 @@ MFEM_USE_CODIPACK = YES/NO
  www.scicomp.uni-kl.de/codi/

 MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical 
-  methods and C++ algorithms for working with implicitly-defined geometry and 
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
-  provides interface to Algoim v1. Thus, to check out the specific state use: 
+  Enable the usage of Algoim - a collection of high-order accurate numerical
+  methods and C++ algorithms for working with implicitly-defined geometry and
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM
+  provides interface to Algoim v1. Thus, to check out the specific state use:
  git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
  https://algoim.github.io

@@ -550,7 +554,7 @@ MFEM_USE_FMS = YES/NO
   Enables support for the FMS library which consists of the DataCollection
   sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
   fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   convetion routines between FMS's FmsDataCollection structure and MFEM's
+   conversion routines between FMS's FmsDataCollection structure and MFEM's
   DataCollection class, see the header file fem/fmsconvert.hpp.

 MFEM_USE_PARELAG = YES/NO
@@ -558,6 +562,14 @@ MFEM_USE_PARELAG = YES/NO
   use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
   currently only concerns the miniapps.

+MFEM_USE_ENZYME = YES/NO
+   Enables automatic differentiation support through the LLVM plugin Enzyme.
+   This requires the compiler to be set to clang (>=14.0.0). We also advise to
+   use the link time optimization (LTO) plugin, to enable functions that you
+   define over multiple files (compilation units) and want to be differentiated
+   automatically, to work. This requires to also use LLVM/LLD for linking.
+   Recommended options are in config/defaults.mk.
+
 MFEM_BUILD_TAG = (any value)
   An optional tag to characterize the build. Exported to config/config.mk.
   Can be used to identify the MFEM build from other makefiles.
@@ -589,7 +601,7 @@ The specific libraries and their options are:

 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
  MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
+  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
  Options: METIS_OPT, METIS_LIB.
  Versions: METIS 4.0.3 or 5.1.0.

@@ -760,8 +772,6 @@ The specific libraries and their options are:
  Options: BLITZ_OPT, BLITZ_LIB
  Versions: BLITZ = 1.0.2

-
-
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
  URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
  Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
@@ -838,6 +848,12 @@ The specific libraries and their options are:
  URL: https://github.com/LLNL/parelag
  Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.

+- Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
+  URL: https://github.com/EnzymeAD/Enzyme
+  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
+  Versions: Enzyme >= v0.0.33.
+
+
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -976,6 +992,7 @@ MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
 MFEM_USE_PARELAG
+MFEM_USE_ENZYME

 The following options are CMake specific:

@@ -1035,6 +1052,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
 - FMS
 - BENCHMARK
 - ParELAG
+ - Enzyme

 The following built-in CMake packages are also used:

@@ -61,6 +61,7 @@ set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
+set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)

 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")
@@ -190,4 +190,7 @@
 // Enable MFEM functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK

+// Enable Enzyme for AD
+#cmakedefine MFEM_USE_ENZYME
+
 #endif // MFEM_CONFIG_HEADER
@@ -0,0 +1,27 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+message(STATUS "Looking for ENZYME ...")
+message(STATUS "   in ENZYME_DIR = ${ENZYME_DIR}")
+
+# Make sure the directory and version combination works. Do nothing otherwise.
+if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
+  message(STATUS "Found ENZYME: ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
+
+  # Set ENZYME_FOUND
+  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
+
+  # Set CXX flags to accommodate the Enzyme Clang plugin
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
+  set(MFEM_USE_ENZYME YES)
+else()
+
+endif()
@@ -0,0 +1,57 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - HDF5_FOUND - If HDF5 was found
+#   - HDF5_LIBRARIES - The HDF5 libraries
+#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
+
+# First Check for HDF5_DIR
+if(NOT HDF5_DIR)
+    MESSAGE(FATAL_ERROR "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
+endif()
+
+# Find includes
+find_path( HDF5_INCLUDE_DIRS hdf5.h
+           PATHS  ${HDF5_DIR}/include/
+           NO_DEFAULT_PATH
+           NO_CMAKE_ENVIRONMENT_PATH
+           NO_CMAKE_PATH
+           NO_SYSTEM_ENVIRONMENT_PATH
+           NO_CMAKE_SYSTEM_PATH)
+
+find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
+              PATHS ${HDF5_DIR}/lib
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
+              PATHS ${HDF5_DIR}/lib
+              NO_DEFAULT_PATH
+              NO_CMAKE_ENVIRONMENT_PATH
+              NO_CMAKE_PATH
+              NO_SYSTEM_ENVIRONMENT_PATH
+              NO_CMAKE_SYSTEM_PATH)
+
+set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+
+# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
+# listed variables are TRUE
+find_package_handle_standard_args(HDF5  DEFAULT_MSG
+                                  HDF5_INCLUDE_DIRS
+                                  __HDF5_LIBRARY
+                                  __HDF5_HL_LIBRARY
+                                  HDF5_LIBRARIES )
@@ -14,6 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS

-include(MfemCmakeUtilities)
-mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
-  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")
+find_package(umpire REQUIRED CONFIG)
+set(UMPIRE_FOUND ${umpire_FOUND})
+set(UMPIRE_LIBRARIES "umpire")
@@ -43,22 +43,14 @@ function(convert_filenames_to_full_paths NAMES)
  set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()

-# Wrapper for add_executable that calls the HIP wrapper if applicable
+# Wrapper for add_executable
 macro(mfem_add_executable NAME)
-  if (MFEM_USE_HIP)
-    add_executable(${NAME} ${ARGN})
-  else()
-    add_executable(${NAME} ${ARGN})
-  endif()
+  add_executable(${NAME} ${ARGN})
 endmacro()

-# Wrapper for add_library that calls the HIP wrapper if applicable
+# Wrapper for add_library
 macro(mfem_add_library NAME)
-  if (MFEM_USE_HIP)
-    add_library(${NAME} ${ARGN})
-  else()
-    add_library(${NAME} ${ARGN})
-  endif()
+  add_library(${NAME} ${ARGN})
 endmacro()

 # Simple shortcut to add_custom_target() with option to add the target to the
@@ -894,7 +886,7 @@ function(mfem_export_mk_files)
      MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
      MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
      MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM)
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
  foreach(var ${CONFIG_MK_BOOL_VARS})
    if (${var})
      set(${var} YES)
@@ -31,9 +31,11 @@

 // Windows specific options
 #ifdef _WIN32
+#ifndef _USE_MATH_DEFINES
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
+#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().
@@ -195,4 +195,7 @@
 // Enable functionality based on the Google Benchmark library.
 // #define MFEM_USE_BENCHMARK

+// Enable the Enzyme LLVM plugin
+// #define MFEM_USE_ENZYME
+
 #endif // MFEM_CONFIG_HEADER
@@ -63,6 +63,7 @@ MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
+MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@

 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@
@@ -64,6 +64,7 @@ option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
+option(MFEM_USE_ENZYME "Enable Enzyme" OFF)

 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -42,6 +42,9 @@ STATIC = YES
 SHARED = NO

 # CUDA configuration options
+#
+# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with
+# clang as its host compiler.
 CUDA_CXX = nvcc
 CUDA_ARCH = sm_60
 CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
@@ -163,6 +166,7 @@ MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
+MFEM_USE_ENZYME        = NO

 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -175,7 +179,7 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
 endif

 # ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
 # the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
 ifeq ($(MFEM_USE_HIP),YES)
   HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
@@ -203,7 +207,7 @@ HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 ifeq (YES,$(MFEM_USE_CUDA))
   # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand
+   HYPRE_LIB += -lcusparse -lcurand -lcublas
 endif
 ifeq (YES,$(MFEM_USE_HIP))
   # This is only necessary when hypre is built with hip:
@@ -305,7 +309,7 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortan library:
+# Additional Fortran library:
 # MPI_FORTRAN_LIB += -lgfortran

 # MUMPS library configuration
@@ -520,6 +524,22 @@ PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
 PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG

+# Enzyme configuration
+
+# If you want to enable automatic differentiation at compile time, use the
+# options below, adapted to your configuration. To be more flexible, we
+# recommend using the Enzyme plugin during link time optimization. One option is
+# to add your options to the global compiler/linker flags like
+#
+# BASE_FLAGS += -flto
+# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\
+#                -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl,
+#
+ENZYME_DIR ?= @MFEM_DIR@/../enzyme
+ENZYME_VERSION ?= 14
+ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so
+ENZYME_LIB = ""
+
 # If YES, enable some informational messages
 VERBOSE = NO

@@ -1,30 +1,27 @@
-FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
+FROM ghcr.io/mfem/mfem-ubuntu-base:latest as builder

-# docker build -t ghcr.io/mfem/mfem-ubuntu-base .
+# docker build -t ghcr.io/mfem/mfem-ubuntu .
+
+COPY ./config/docker/spack.yaml /opt/mfem-env/spack.yaml
+RUN apt-get install -y python3 && \
+    cd /opt/mfem-env && \
+    . /opt/spack/share/spack/setup-env.sh && \
+    spack env activate . && \
+    spack env view regenerate
+
+FROM ubuntu:22.04
+
+COPY --from=builder /opt/view /opt/view
+COPY --from=builder /opt/mfem-view /opt/mfem-view

 RUN apt-get update && \
    apt-get install -y unzip gfortran && \
-    spack compiler find && \
    apt-get install -y libcurl4-openssl-dev libssl-dev

-# /code is the working directory for code
-WORKDIR /code
-COPY . /code
-
-# This is for a spack environment/view to install from there
-WORKDIR /opt/mfem-env
-RUN . /opt/spack/share/spack/setup-env.sh && \
-    spack env create -d . && \
-    echo "  concretization: together" >> spack.yaml && \
-    spack env activate . && \
-    spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps && \
-    spack install
-
-# ensure mfem always on various paths
-RUN cd /opt/mfem-env && \
-    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+ENV PATH=$PATH:/opt/mfem-view/bin
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/mfem-view/lib:/opt/mfem-view/lib64
+ENV DEBIAN_FRONTEND=noninteractive

 # The user will see the view on shell into the container
-WORKDIR /opt/mfem-env/.spack-env/view/
-ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
+WORKDIR /opt/mfem-view
+ENTRYPOINT ["/bin/bash"]
@@ -0,0 +1,47 @@
+FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
+
+# docker build -f Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
+
+RUN apt-get update && \
+    apt-get install -y unzip gfortran && \
+    spack compiler find && \
+    apt-get install -y libcurl4-openssl-dev libssl-dev
+
+# /code is the working directory for code
+WORKDIR /code
+COPY . /code
+
+# This is for a spack environment/view to install from there
+RUN mkdir -p /opt/mfem-env \
+&&  (echo "spack:" \
+&&   echo "  view:" \
+&&   echo "    mfem:" \
+&&   echo "      root: /opt/mfem-view" \
+&&   echo "      link_type: copy" \
+&&   echo "  packages:" \
+&&   echo "    all:" \
+&&   echo "      target:" \
+&&   echo "      - x86_64_v3" \
+&&   echo "  config:" \
+&&   echo "    concretizer: clingo" \
+&&   echo "    compiler:" \
+&&   echo "      target:" \
+&&   echo "      - x86_64_v3" \
+&&   echo "    install_missing_compilers: true" \
+&&   echo "  concretization: together") > /opt/mfem-env/spack.yaml
+
+RUN cd /opt/mfem-env && \
+    . /opt/spack/share/spack/setup-env.sh && \
+    spack env activate . && \
+    spack develop --path /code mfem@master+examples+miniapps && \
+    spack add mfem@master+examples+miniapps # && \
+#    spack install
+
+# ensure mfem always on various paths
+#RUN cd /opt/mfem-env && \
+#    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+
+# Present the software install when we shell in
+# The view is at /opt/mfem-env/.spack-env/view
+#WORKDIR /opt/software
+#ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
@@ -1,7 +1,8 @@
 # mfem Docker

-We provide a [Dockerfile](Dockerfile) to build an ubuntu base image. You can use
-this image for a demo of using mfem! 🎉️
+We provide a [Dockerfile.base](Dockerfile.base) to build an ubuntu base image,
+and a [Dockerfile](Dockerfile) to build a smaller one with a multi-stage build.
+You can use this image for a demo of using mfem! 🎉️

 Updated containers are built and deployed on merges to the main branch and releases.
 If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
@@ -14,18 +15,33 @@ is the [GitHub packages](https://github.com/features/packages) registry that sup
 Docker images and other OCI artifacts. From the root of the repository:

 ```bash
-$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base .
+$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
+$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
 ```

-or this directory:
+### Shell Ubuntu
+
+To shell into the container:

 ```bash
-$ docker build -f Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base ../../
+$ docker run -it ghcr.io/mfem/mfem-ubuntu
 ```

-### Shell
+This smaller image has a view where everything is installed.

-To shell into a container (here is an example with ubuntu):
+```bash
+$ ls
+bin  etc  include  lib  libexec  sbin  share  var
+```
+
+ - Examples are in share/mfem/examples
+ - Examples are in share/mfem/miniapps
+
+You can read more about interaction with these examples and miniapps below.
+
+### Shell Ubuntu Base
+
+To shell into the container:

 ```bash
 $ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
@@ -128,3 +144,4 @@ $ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
 In the above, we can pretend your project is in the present working directory (PWD) and we are
 binding to source. You can then use the mfem in the container for development, and if you
 want to distribute your library or app in a container, you can use the mfem container as the base.
+
@@ -0,0 +1,11 @@
+spack:
+  specs: [mfem@master+examples+miniapps]
+  view: 
+    mfem:
+      root: /opt/mfem-view 
+      link_type: copy
+  concretization: together
+  develop:
+    mfem:
+      path: /code
+      spec: mfem@master+examples+miniapps
@@ -0,0 +1,8 @@
+--- a/CMakeLists.txt	Wed Dec 21 18:24:22 2016
+++ b/CMakeLists.txt	Wed Dec 21 18:24:26 2016
+@@ -20,4 +20,4 @@
+ # Recursively look for CMakeLists.txt in subdirs.
+ add_subdirectory("include")
+ add_subdirectory("libmetis")
+-add_subdirectory("programs")
+# add_subdirectory("programs")
@@ -0,0 +1,15 @@
+--- a/CMakeLists.txt	Sat Mar 30 17:24:45 2013
+++ b/CMakeLists.txt	Wed Dec 21 18:23:43 2016
+@@ -4,11 +4,7 @@
+ set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+ 
+-if(MSVC)
+-  set(METIS_INSTALL FALSE)
+-else()
+-  set(METIS_INSTALL TRUE)
+-endif()
+set(METIS_INSTALL TRUE)
+ 
+ # Configure libmetis library.
+ if(SHARED)
@@ -0,0 +1,34 @@
+diff --git a/include/metis.h b/include/metis.h
+index dc5406a..7732437 100644
+--- a/include/metis.h
+++ b/include/metis.h
+@@ -72,10 +72,14 @@ typedef __int64 int64_t;
+ #define PRId64       "I64d"
+ #define SCNd32       "ld"
+ #define SCNd64       "I64d"
+#ifdef _WIN32
+#include <stdint.h>
+#else
+ #define INT32_MIN    ((int32_t)_I32_MIN)
+ #define INT32_MAX    _I32_MAX
+ #define INT64_MIN    ((int64_t)_I64_MIN)
+ #define INT64_MAX    _I64_MAX
+#endif
+ #else
+ #include <inttypes.h>
+ #endif
+diff --git a/GKlib/gk_arch.h b/GKlib/gk_arch.h
+index 78b1431..7258763 100644
+--- a/GKlib/gk_arch.h
+++ b/GKlib/gk_arch.h
+@@ -32,8 +32,8 @@
+ 
+ 
+ #ifdef __MSC__ 
+-  #include "ms_stdint.h"
+-  #include "ms_inttypes.h"
+  #include <stdint.h>
+  #include <inttypes.h>
+   #include "ms_stat.h"
+ #else
+ #ifndef SUNOS
@@ -0,0 +1,11 @@
+--- a/GKlib/gk_arch.h	Wed Dec 21 18:34:18 2016
+++ b/GKlib/gk_arch.h	Wed Dec 21 18:30:49 2016
+@@ -58,7 +58,7 @@
+ #define PTRDIFF_MAX  INT64_MAX
+ #endif
+ 
+-#ifdef __MSC__
+#if defined(__MSC__) && (_MSC_VER < 1900)
+ /* MSC does not have rint() function */
+ #define rint(x) ((int)((x)+0.5))  
+ 
@@ -0,0 +1,14 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index e94f050..b9613a7 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -1,7 +1,8 @@
+ cmake_minimum_required(VERSION 2.8)
+ project(METIS)
+ 
+-set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
+set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
+
+ set(SHARED FALSE CACHE BOOL "build a shared library")
+ 
+ set(METIS_INSTALL TRUE)
@@ -0,0 +1,11 @@
+--- a/libmetis/metislib.h	Sat Mar 30 17:24:45 2013
+++ b/libmetis/metislib.h	Wed Dec 21 18:30:59 2016
+@@ -31,7 +31,7 @@
+ #include <proto.h>
+ 
+ 
+-#if defined(COMPILER_MSC)
+#if defined(COMPILER_MSC) && (_MSC_VER < 1900)
+ #if defined(rint)
+   #undef rint
+ #endif
@@ -0,0 +1,10 @@
+--- a/libmetis/CMakeLists.txt	Sat Mar 30 17:24:45 2013
+++ b/libmetis/CMakeLists.txt	Wed Dec 21 17:41:37 2016
+@@ -11,6 +11,6 @@
+ if(METIS_INSTALL)
+   install(TARGETS metis
+     LIBRARY DESTINATION lib
+-    RUNTIME DESTINATION lib
+    RUNTIME DESTINATION bin
+     ARCHIVE DESTINATION lib)
+ endif()
@@ -0,0 +1,44 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index b9613a7..e43ffee 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -22,3 +22,23 @@ include_directories(include)
+ add_subdirectory("include")
+ add_subdirectory("libmetis")
+ # add_subdirectory("programs")
+
+if(METIS_INSTALL)
+  set(PRJ_NAME metis)
+  set(PRJ_VER 5.1.0)
+  install(EXPORT metisTargets 
+    FILE ${PRJ_NAME}Targets.cmake
+    DESTINATION lib/cmake/${PRJ_NAME})
+  include(CMakePackageConfigHelpers)
+  write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
+    VERSION ${PRJ_VER}
+    COMPATIBILITY SameMajorVersion)
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
+    "include(\${CMAKE_CURRENT_LIST_DIR}/${PRJ_NAME}Targets.cmake)")
+  install(FILES 
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
+    DESTINATION lib/cmake/${PRJ_NAME})
+endif()
+
+diff --git a/libmetis/CMakeLists.txt b/libmetis/CMakeLists.txt
+index 7a5fc74..5a68cf0 100644
+--- a/libmetis/CMakeLists.txt
+++ b/libmetis/CMakeLists.txt
+@@ -9,8 +9,9 @@ if(UNIX)
+ endif()
+ 
+ if(METIS_INSTALL)
+-  install(TARGETS metis
+  install(TARGETS metis EXPORT metisTargets
+     LIBRARY DESTINATION lib
+     RUNTIME DESTINATION bin
+-    ARCHIVE DESTINATION lib)
+    ARCHIVE DESTINATION lib
+    INCLUDES DESTINATION include)
+ endif()
@@ -0,0 +1,41 @@
+vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
+set(OPTIONS -DSHARED=OFF)
+
+set(METIS_VERSION 5.1.0)
+
+vcpkg_download_distfile(ARCHIVE
+    URLS "https://github.com/mfem/tpls/raw/gh-pages/metis-${METIS_VERSION}.tar.gz"
+    FILENAME "metis-${METIS_VERSION}.tar.gz"
+    SHA512 deea47749d13bd06fbeaf98a53c6c0b61603ddc17a43dae81d72c8015576f6495fd83c11b0ef68d024879ed5415c14ebdbd87ce49c181bdac680573bea8bdb25
+)
+
+vcpkg_extract_source_archive_ex(
+    OUT_SOURCE_PATH SOURCE_PATH
+    ARCHIVE ${ARCHIVE}
+    REF ${METIS_VERSION}
+    PATCHES
+        enable-install.patch
+        disable-programs.patch
+        fix-runtime-install-destination.patch
+        fix-metis-vs14-math.patch
+        fix-gklib-vs14-math.patch
+        fix-linux-build-error.patch
+        install-metisConfig.patch
+        fix-INT_MIN_define.patch
+)
+
+vcpkg_configure_cmake(
+    SOURCE_PATH ${SOURCE_PATH}
+    PREFER_NINJA
+    OPTIONS ${OPTIONS}
+)
+
+vcpkg_install_cmake()
+vcpkg_copy_pdbs()
+vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/metis)
+
+file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
+
+# Handle copyright
+file(COPY ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/metis)
+file(INSTALL ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
@@ -0,0 +1,7 @@
+{
+  "name": "metis-mfem",
+  "version-string": "5.1.0",
+  "port-version": 0,
+  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
+  "homepage": "https://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
+}
@@ -1,7 +1,7 @@
 MFEM mesh v1.0

 #
-# MFEM Geomety Types (see mesh/geom.hpp):
+# MFEM Geometry Types (see mesh/geom.hpp):
 #
 # POINT       = 0
 # SEGMENT     = 1
@@ -2349,7 +2349,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.

-DOT_GRAPH_MAX_NODES    = 50
+DOT_GRAPH_MAX_NODES    = 100

 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the
@@ -30,6 +30,7 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
+//             * ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -37,9 +38,13 @@
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
 //               ex1 -pa -d ceed-cpu -o 4 -a
+//               ex1 -pa -d ceed-cpu -m ../data/square-mixed.mesh
+//               ex1 -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
 //             * ex1 -pa -d ceed-cuda
 //             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
+//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -73,6 +78,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
+   bool fa = false;
   const char *device_config = "cpu";
   bool visualization = true;
   bool algebraic_ceed = false;
@@ -87,6 +93,8 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
@@ -184,6 +192,7 @@ int main(int argc, char *argv[])
   //    domain integrator.
   BilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 10. Assemble the bilinear form and the corresponding linear system,
@@ -30,13 +30,18 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
+//             * mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
 //               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
 //             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
+//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -74,6 +79,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
+   bool fa = false;
   const char *device_config = "cpu";
   bool visualization = true;
   bool algebraic_ceed = false;
@@ -88,6 +94,8 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
+                  "--no-full-assembly", "Enable Full Assembly.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
@@ -211,6 +219,7 @@ int main(int argc, char *argv[])
   //     Diffusion domain integrator.
   ParBilinearForm a(&fespace);
   if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
+   if (fa) { a.SetAssemblyLevel(AssemblyLevel::FULL); }
   a.AddDomainIntegrator(new DiffusionIntegrator(one));

   // 12. Assemble the parallel bilinear form and the corresponding linear
@@ -182,7 +182,7 @@ int main(int argc, char *argv[])
   }
   for (int level = 0; level < order_refinements; ++level)
   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection((int)std::pow(2, level+1), dim));
      fespaces.AddOrderRefinedLevel(collections.Last());
   }

@@ -219,7 +219,7 @@ int main(int argc, char *argv[])
   }
   for (int level = 0; level < order_refinements; ++level)
   {
-      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection((int)std::pow(2, level+1), dim));
      fespaces->AddOrderRefinedLevel(collections.Last());
   }

@@ -195,7 +195,7 @@ int main(int argc, char *argv[])
   Array<int> ess_tdof_list(0);
   if (h1 && pmesh.bdr_attributes.Size())
   {
-      // For a continuous basis the linear system must be modifed to enforce an
+      // For a continuous basis the linear system must be modified to enforce an
      // essential (Dirichlet) boundary condition. In the DG case this is not
      // necessary as the boundary condition will only be enforced weakly.
      fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);
@@ -3,34 +3,63 @@
 // Compile with: make ex33
 //
 // Sample runs:  ex33 -m ../data/square-disc.mesh -alpha 0.33 -o 2
+//               ex33 -m ../data/square-disc.mesh -alpha 4.5 -o 3
+//               ex33 -m ../data/star.mesh -alpha 1.4 -o 3
 //               ex33 -m ../data/star.mesh -alpha 0.99 -o 3
 //               ex33 -m ../data/inline-quad.mesh -alpha 0.5 -o 3
+//               ex33 -m ../data/amr-quad.mesh -alpha 1.5 -o 3
 //               ex33 -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
+//               ex33 -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
 //               ex33 -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
+//               ex33 -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
+//
+// Verification runs:
+//    ex33 -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
+//    ex33 -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
+//    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
+//    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
+//
+//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
+//        for all alpha.
 //
 // Description:
 //
 //  In this example we solve the following fractional PDE with MFEM:
 //
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
 //
-//  To solve this FPDE, we rely on a rational approximation [2] of the normal
-//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
-//  boundary conditions). Namely, we first approximate the operator
+//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
+//  N is given by floor(α). By doing so, we obtain
 //
-//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
+//
+//  We first compute the right hand side by solving the integer order PDE
+//
+//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
+//
+//  The remaining FPDE is then given by
+//
+//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
+//
+//  We rely on a rational approximation [2] of the normal linear operator
+//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
+//  and (a-N) in (0,1). We approximate the operator
+//
+//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
 //
 //  where I is the L2-identity operator and the coefficients c_i and d_i
 //  are generated offline to a prescribed accuracy in a pre-processing step.
 //  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve N+1
+//  that this partial fractional expansion derives from. We then solve M+1
 //  independent integer-order PDEs,
 //
-//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
+//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
 //
 //  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
 //
-//    u ≈ Σ_{i=0}^N u_i.
+//    u ≈ Σ_{i=0}^M u_i.
+//
+//  (If alpha is an integer, we stop after the first PDE was solved.)
 //
 // References:
 //
@@ -47,6 +76,8 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <math.h>
+#include <string>

 #include "ex33.hpp"

@@ -59,8 +90,9 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int num_refs = 3;
-   bool visualization = true;
   double alpha = 0.5;
+   bool visualization = true;
+   bool verification = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -75,6 +107,9 @@ int main(int argc, char *argv[])
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
+                  "--no-verification",
+                  "Use sinusoidal function (f) for analytic comparison.");
   args.Parse();
   if (!args.Good())
   {
@@ -84,9 +119,31 @@ int main(int argc, char *argv[])
   args.PrintOptions(cout);

   Array<double> coeffs, poles;
+   int progress_steps = 1;

-   // 2. Compute the coefficients that define the integer-order PDEs.
-   ComputePartialFractionApproximation(alpha,coeffs,poles);
+   // 2. Compute the rational expansion coefficients that define the
+   //    integer-order PDEs.
+   const int power_of_laplace = floor(alpha);
+   double exponent_to_approximate = alpha - power_of_laplace;
+   bool integer_order = false;
+   // Check if alpha is an integer or not.
+   if (abs(exponent_to_approximate) > 1e-12)
+   {
+      mfem::out << "Approximating the fractional exponent "
+                << exponent_to_approximate
+                << endl;
+      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
+                                          poles);
+
+      // If the example is build without LAPACK, the exponent_to_approximate
+      // might be modified by the function call above.
+      alpha = exponent_to_approximate + power_of_laplace;
+   }
+   else
+   {
+      integer_order = true;
+      mfem::out << "Treating integer order PDE." << endl;
+   }

   // 3. Read the mesh from the given mesh file.
   Mesh mesh(mesh_file, 1, 1);
@@ -99,8 +156,8 @@ int main(int argc, char *argv[])
   }

   // 5. Define a finite element space on the mesh.
-   FiniteElementCollection *fec = new H1_FECollection(order, dim);
-   FiniteElementSpace fespace(&mesh, fec);
+   H1_FECollection fec(order, dim);
+   FiniteElementSpace fespace(&mesh, &fec);
   cout << "Number of finite element unknowns: "
        << fespace.GetTrueVSize() << endl;

@@ -114,79 +171,234 @@ int main(int argc, char *argv[])
   }

   // 7. Define diffusion coefficient, load, and solution GridFunction.
-   ConstantCoefficient f(1.0);
+   auto func = [&alpha](const Vector &x)
+   {
+      double val = 1.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         val *= sin(M_PI*x(i));
+      }
+      return pow(x.Size()*pow(M_PI,2), alpha) * val;
+   };
+   FunctionCoefficient f(func);
   ConstantCoefficient one(1.0);
   GridFunction u(&fespace);
-   u = 0.;
+   GridFunction x(&fespace);
+   GridFunction g(&fespace);
+   u = 0.0;
+   x = 0.0;
+   g = 0.0;

   // 8. Prepare for visualization.
   char vishost[] = "localhost";
   int  visport   = 19916;
-   socketstream xout, uout;
-   ostringstream oss_x, oss_u;
-   if (visualization)
+
+   // 9. Set up the linear form b(.) for integer-order PDE solves.
+   LinearForm b(&fespace);
+   if (verification)
   {
-      xout.open(vishost, visport);
-      xout.precision(8);
-      uout.open(vishost, visport);
-      uout.precision(8);
+      // This statement is only relevant for the verification of the code. It
+      // uses a different f such that an analytic solution is known and easy
+      // to compare with the numerical one. The FPDE becomes:
+      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
+      // -> u(x,y) = sin(\pi x) sin(\pi y)
+      b.AddDomainIntegrator(new DomainLFIntegrator(f));
   }
-
-   for (int i = 0; i < coeffs.Size(); i++)
+   else
   {
-      // 9. Set up the linear form b(.) for integer-order PDE solve.
-      LinearForm b(&fespace);
-      ProductCoefficient cf(coeffs[i], f);
-      b.AddDomainIntegrator(new DomainLFIntegrator(cf));
-      b.Assemble();
+      b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   }
+   b.Assemble();

-      // 10. Define GridFunction for integer-order PDE solve.
-      GridFunction x(&fespace);
-      x = 0.0;
+   // ------------------------------------------------------------------------
+   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
+   // ------------------------------------------------------------------------

-      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
-      BilinearForm a(&fespace);
-      a.AddDomainIntegrator(new DiffusionIntegrator(one));
-      ConstantCoefficient c2(-poles[i]);
-      a.AddDomainIntegrator(new MassIntegrator(c2));
-      a.Assemble();
+   if (power_of_laplace > 0)
+   {
+      // 10.1 Compute Stiffnes Matrix
+      BilinearForm k(&fespace);
+      k.AddDomainIntegrator(new DiffusionIntegrator(one));
+      k.Assemble();

-      // 12. Assemble the bilinear form and the corresponding linear system.
-      OperatorPtr A;
+      // 10.2 Compute Mass Matrix
+      BilinearForm m(&fespace);
+      m.AddDomainIntegrator(new MassIntegrator(one));
+      m.Assemble();
+      SparseMatrix mass;
+      Array<int> empty;
+      m.FormSystemMatrix(empty, mass);
+
+      // 10.3 Form the system of equations
      Vector B, X;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+      OperatorPtr Op;
+      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
+      GSSmoother M((SparseMatrix&)(*Op));

-      // 13. Solve the linear system A X = B.
-      GSSmoother M((SparseMatrix&)(*A));
-
-      mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
-                << " u = " << coeffs[i] << " f " << endl;
-      PCG(*A, M, B, X, 3, 200, 1e-12, 0.0);
-
-      // 14. Recover the solution as a finite element grid function.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 15. Accumulate integer-order PDE solutions.
-      u+=x;
-
-      // 16. Send the solutions by socket to a GLVis server.
-      if (visualization)
+      mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
+                << " ( f ) " << endl;
+      for (int i = 0; i < power_of_laplace; i++)
      {
-         oss_x.str(""); oss_x.clear();
-         oss_x << "Solution of PDE -Δ u + " << -poles[i]
-               << " u = " << coeffs[i] << " f";
-         xout << "solution\n" << mesh << x
-              << "window_title '" << oss_x.str() << "'" << flush;
+         // 10.4 Solve the linear system Op X = B (N times).
+         PCG(*Op, M, B, X, 3, 300, 1e-12, 0.0);

-         oss_u.str(""); oss_u.clear();
-         oss_u << "Solution of fractional PDE -Δ^" << alpha
-               << " u = f";
-         uout << "solution\n" << mesh << u
-              << "window_title '" << oss_u.str() << "'" << flush;
+         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
+         if (i == power_of_laplace - 1)
+         {
+            // Needed for visualization and solution verification.
+            k.RecoverFEMSolution(X, b, g);
+            if (integer_order && verification)
+            {
+               // For an integer order PDE, g is also our solution u.
+               u+=g;
+            }
+            if (visualization)
+            {
+               socketstream fout;
+               ostringstream oss_f;
+               fout.open(vishost, visport);
+               fout.precision(8);
+               oss_f.str(""); oss_f.clear();
+               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
+                     << power_of_laplace
+                     << " g = f";
+               fout << "solution\n" << mesh << g
+                    << "window_title '" << oss_f.str() << "'" << flush;
+            }
+         }
+
+         // 10.6 Prepare for next iteration (primal / dual space)
+         mass.Mult(X, B);
+         X.SetSubVectorComplement(ess_tdof_list,0.0);
+      }
+
+      // 10.7 Extract solution for the next step. The b now corresponds to the
+      //      function g in the PDE.
+      const SparseMatrix * R = fespace.GetRestrictionMatrix();
+      if (R)
+      {
+         R->MultTranspose(B,b);
+      }
+      else
+      {
+         b = B;
      }
   }

-   // 17. Free the used memory.
-   delete fec;
+   // ------------------------------------------------------------------------
+   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
+   //     up the solutions.
+   // ------------------------------------------------------------------------
+   if (!integer_order)
+   {
+      // Setup visualization.
+      socketstream xout, uout;
+      ostringstream oss_x, oss_u;
+      if (visualization)
+      {
+         xout.open(vishost, visport);
+         xout.precision(8);
+         uout.open(vishost, visport);
+         uout.precision(8);
+      }
+      // Iterate over all expansion coefficient that contribute to the
+      // solution.
+      for (int i = 0; i < coeffs.Size(); i++)
+      {
+         mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                   << " u = " << coeffs[i] << " g " << endl;
+
+
+         // 11.1 Reset GridFunction for integer-order PDE solve.
+         x = 0.0;
+
+         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
+         BilinearForm a(&fespace);
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         ConstantCoefficient d_i(-poles[i]);
+         a.AddDomainIntegrator(new MassIntegrator(d_i));
+         a.Assemble();
+
+         // 11.3 Assemble the bilinear form and the corresponding linear system.
+         OperatorPtr A;
+         Vector B, X;
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+         // 11.4 Solve the linear system A X = B.
+         GSSmoother M((SparseMatrix&)(*A));
+
+         PCG(*A, M, B, X, 3, 300, 1e-12, 0.0);
+
+         // 11.5 Recover the solution as a finite element grid function.
+         a.RecoverFEMSolution(X, b, x);
+
+         // 11.6 Accumulate integer-order PDE solutions.
+         x *= coeffs[i];
+         u += x;
+
+         // 11.7 Send fractional PDE solution to a GLVis server.
+         if (visualization)
+         {
+            oss_x.str(""); oss_x.clear();
+            oss_x << "Step " << progress_steps
+                  << ": Solution of PDE -Δ u + " << -poles[i]
+                  << " u = " << coeffs[i] << " g";
+            xout << "solution\n" << mesh << x
+                 << "window_title '" << oss_x.str() << "'" << flush;
+
+            oss_u.str(""); oss_u.clear();
+            oss_u << "Step " << progress_steps + 1
+                  << ": Solution of fractional PDE (-Δ)^" << alpha
+                  << " u = f";
+            uout << "solution\n" << mesh << u
+                 << "window_title '" << oss_u.str() << "'"
+                 << flush;
+         }
+      }
+   }
+
+   // ------------------------------------------------------------------------
+   // 12. (optional) Verify the solution.
+   // ------------------------------------------------------------------------
+   if (verification)
+   {
+      auto solution = [] (const Vector &x)
+      {
+         double val = 1.0;
+         for (int i=0; i<x.Size(); i++)
+         {
+            val *= sin(M_PI*x(i));
+         }
+         return val;
+      };
+      FunctionCoefficient sol(solution);
+      double l2_error = u.ComputeL2Error(sol);
+
+      string analytic_solution,expected_mesh;
+      switch (dim)
+      {
+         case 1:
+            analytic_solution = "sin(π x)";
+            expected_mesh = "inline_segment.mesh";
+            break;
+         case 2:
+            analytic_solution = "sin(π x) sin(π y)";
+            expected_mesh = "inline_quad.mesh";
+            break;
+         default:
+            analytic_solution = "sin(π x) sin(π y) sin(π z)";
+            expected_mesh = "inline_hex.mesh";
+            break;
+      }
+
+      mfem::out << "\n" << string(80,'=')
+                << "\n\nSolution Verification in "<< dim << "D \n\n"
+                << "Analytic solution : " << analytic_solution << "\n"
+                << "Expected mesh     : " << expected_mesh <<"\n"
+                << "Your mesh         : " << mesh_file << "\n"
+                << "L2 error          : " << l2_error << "\n\n"
+                << string(80,'=') << endl;
+   }
+
   return 0;
 }
@@ -32,6 +32,7 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <string>

 using namespace std;
 using namespace mfem;
@@ -249,6 +250,13 @@ void PartialFractionExpansion(double scale, Array<double> & poles,
   coeffs.SetSize(psize);
   coeffs = scale;

+   // Note: C p(z)/q(z) = Σ_i c_i / (z - p_i) results in an system of equations
+   // where the N unknowns are the coefficients c_i. After multiplying the
+   // system with q(z), the coefficients c_i can be computed analytically by
+   // choosing N values for z. Choosing z_j = = p_j diagonalizes the system and
+   // one can obtain an analytic form for the c_i coefficients. The result is
+   // implemented in the code block below.
+
   for (int i=0; i<psize; i++)
   {
      double tmp_numer=1.0;
@@ -305,9 +313,12 @@ void ComputePartialFractionApproximation(double & alpha,
   if (print_warning)
   {
      mfem::out
-            << "\nMFEM is compiled without LAPACK.\nUsing precomputed values for PartialFractionApproximation. \n"
-            << "Only alpha = 0.33, 0.5, and 0.99 are available.\nThe default is alpha = 0.5."
-            << std::endl;
+            << "\n" << string(80, '=')
+            << "\nMFEM is compiled without LAPACK."
+            << "\nUsing precomputed values for PartialFractionApproximation."
+            << "\nOnly alpha = 0.33, 0.5, and 0.99 are available."
+            << "\nThe default is alpha = 0.5.\n" << string(80, '=') << "\n"
+            << endl;
   }
   const double eps = std::numeric_limits<double>::epsilon();

@@ -351,7 +362,7 @@ void ComputePartialFractionApproximation(double & alpha,

   if (print_warning)
   {
-      mfem::out << "Using precomputed values for alpha = "
+      mfem::out << "=> Using precomputed values for alpha = "
                << alpha << "\n" << std::endl;
   }

@@ -3,34 +3,63 @@
 // Compile with: make ex33p
 //
 // Sample runs:  mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 0.33 -o 2
+//               mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 4.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 1.4 -o 3
 //               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 0.99 -o 3
 //               mpirun -np 4 ex33p -m ../data/inline-quad.mesh -alpha 0.5 -o 3
-//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
+//               mpirun -np 4 ex33p -m ../data/amr-quad.mesh -alpha 1.5 -o 3
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3 -r 2
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
 //               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
+//               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
+//
+// Verification runs:
+//    mpirun -np 4 ex33p -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
+//    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
+
+//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
+//        for all alpha.
 //
 // Description:
 //
 //  In this example we solve the following fractional PDE with MFEM:
 //
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
 //
-//  To solve this FPDE, we rely on a rational approximation [2] of the normal
-//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
-//  boundary conditions). Namely, we first approximate the operator
+//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
+//  N is given by floor(α). By doing so, we obtain
 //
-//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
+//
+//  We first compute the right hand side by solving the integer order PDE
+//
+//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
+//
+//  The remaining FPDE is then given by
+//
+//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
+//
+//  We rely on a rational approximation [2] of the normal linear operator
+//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
+//  and (a-N) in (0,1). We approximate the operator
+//
+//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
 //
 //  where I is the L2-identity operator and the coefficients c_i and d_i
 //  are generated offline to a prescribed accuracy in a pre-processing step.
 //  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve N+1
+//  that this partial fractional expansion derives from. We then solve M+1
 //  independent integer-order PDEs,
 //
-//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
+//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
 //
 //  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
 //
-//    u ≈ Σ_{i=0}^N u_i.
+//    u ≈ Σ_{i=0}^M u_i.
+//
+//  (If alpha is an integer, we stop after the first PDE was solved.)
 //
 // References:
 //
@@ -47,6 +76,8 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
+#include <math.h>
+#include <string>

 #include "ex33.hpp"

@@ -65,9 +96,9 @@ int main(int argc, char *argv[])
   const char *mesh_file = "../data/star.mesh";
   int order = 1;
   int num_refs = 3;
-   bool visualization = true;
-   bool visualize_x = false;
   double alpha = 0.5;
+   bool visualization = true;
+   bool verification = false;

   OptionsParser args(argc, argv);
   args.AddOption(&mesh_file, "-m", "--mesh",
@@ -79,12 +110,12 @@ int main(int argc, char *argv[])
                  "Number of uniform refinements");
   args.AddOption(&alpha, "-alpha", "--alpha",
                  "Fractional exponent");
-   args.AddOption(&visualize_x, "-vis_x", "--visualize_x", "-no-vis_x",
-                  "--no-visualization_x",
-                  "Enable or disable GLVis visualization of each integer-order PDE solution.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
-                  "Enable or disable GLVis visualization of the fractional PDE solution.");
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
+                  "--no-verification",
+                  "Use sinusoidal function (f) for analytic comparison.");
   args.Parse();
   if (!args.Good())
   {
@@ -97,61 +128,51 @@ int main(int argc, char *argv[])
   }

   Array<double> coeffs, poles;
+   int progress_steps = 1;

-   // 2. Compute the coefficients that define the integer-order PDEs.
-   ComputePartialFractionApproximation(alpha,coeffs,poles);
-
-   int num_par_solves;
-   int max_par_solves = max(1,num_procs/2);
-   for (num_par_solves=max_par_solves; num_par_solves>0; num_par_solves--)
+   // 2. Compute the rational expansion coefficients that define the
+   //    integer-order PDEs.
+   const int power_of_laplace = floor(alpha);
+   double exponent_to_approximate = alpha - power_of_laplace;
+   bool integer_order = false;
+   // Check if alpha is an integer or not.
+   if (abs(exponent_to_approximate) > 1e-12)
   {
-      if (num_procs%num_par_solves==0 && num_par_solves<coeffs.Size())
+      if (Mpi::Root())
      {
-         break;
+         mfem::out << "Approximating the fractional exponent "
+                   << exponent_to_approximate
+                   << endl;
+      }
+      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
+                                          poles);
+
+      // If the example is build without LAPACK, the exponent_to_approximate
+      // might be modified by the function call above.
+      alpha = exponent_to_approximate + power_of_laplace;
+   }
+   else
+   {
+      integer_order = true;
+      if (Mpi::Root())
+      {
+         mfem::out << "Treating integer order PDE." << endl;
      }
   }
-   if (num_par_solves == 1) {num_par_solves = num_procs;}

-   int solver_ranks = num_procs/num_par_solves;
-
-   // 3. Split the MPI communicator:
-   //    row_comm is used for parallel partition of the mesh
-   //    col_comm is used for independent integer-order solves
-   int row_color = myid / solver_ranks; // Determine color based on row
-   int col_color = myid % solver_ranks; // Determine color based on col
-
-   MPI_Comm row_comm, col_comm;
-   MPI_Comm_split(MPI_COMM_WORLD, row_color, myid, &row_comm);
-   MPI_Comm_split(MPI_COMM_WORLD, col_color, myid, &col_comm);
-
-   int row_rank, row_size, col_rank, col_size;
-   MPI_Comm_rank(row_comm, &row_rank);
-   MPI_Comm_size(row_comm, &row_size);
-   MPI_Comm_rank(col_comm, &col_rank);
-   MPI_Comm_size(col_comm, &col_size);
-
-   if (Mpi::Root())
-   {
-      mfem::out << "\nTotal number of MPI ranks = " << num_procs << endl;
-      mfem::out << "Number of independent parallel solves = " << col_size << endl;
-      mfem::out << "Number of MPI ranks within each solve = " << row_size
-                <<"\n" << endl;
-   }
-
-   // 4. Read the mesh from the given mesh file.
+   // 3. Read the mesh from the given mesh file.
   Mesh mesh(mesh_file, 1, 1);
   int dim = mesh.Dimension();

-   // 5. Refine the mesh to increase the resolution.
+   // 4. Refine the mesh to increase the resolution.
   for (int i = 0; i < num_refs; i++)
   {
      mesh.UniformRefinement();
   }
-
-   ParMesh pmesh(row_comm, mesh);
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
   mesh.Clear();

-   // 6. Define a finite element space on the mesh.
+   // 5. Define a finite element space on the mesh.
   H1_FECollection fec(order, dim);
   ParFiniteElementSpace fespace(&pmesh, &fec);
   if (Mpi::Root())
@@ -160,7 +181,7 @@ int main(int argc, char *argv[])
           << fespace.GetTrueVSize() << endl;
   }

-   // 7. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
   Array<int> ess_tdof_list;
   if (pmesh.bdr_attributes.Size())
   {
@@ -169,120 +190,250 @@ int main(int argc, char *argv[])
      fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
   }

-   // 8. Define diffusion coefficient, load, and solution GridFunction.
-   ConstantCoefficient f(1.0);
+   // 7. Define diffusion coefficient, load, and solution GridFunction.
+   auto func = [&alpha](const Vector &x)
+   {
+      double val = 1.0;
+      for (int i=0; i<x.Size(); i++)
+      {
+         val *= sin(M_PI*x(i));
+      }
+      return pow(x.Size()*pow(M_PI,2), alpha) * val;
+   };
+   FunctionCoefficient f(func);
   ConstantCoefficient one(1.0);
   ParGridFunction u(&fespace);
   ParGridFunction x(&fespace);
+   ParGridFunction g(&fespace);
   u = 0.0;
+   x = 0.0;
+   g = 0.0;
+
+   // 8. Prepare for visualization.
+   char vishost[] = "localhost";
+   int  visport   = 19916;

   // 9. Set up the linear form b(.) for integer-order PDE solves.
   ParLinearForm b(&fespace);
-   b.AddDomainIntegrator(new DomainLFIntegrator(f));
+   if (verification)
+   {
+      // This statement is only relevant for the verification of the code. It
+      // uses a different f such that an analytic solution is known and easy
+      // to compare with the numerical one. The FPDE becomes:
+      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
+      // -> u(x,y) = sin(\pi x) sin(\pi y)
+      b.AddDomainIntegrator(new DomainLFIntegrator(f));
+   }
+   else
+   {
+      b.AddDomainIntegrator(new DomainLFIntegrator(one));
+   }
   b.Assemble();

-   int my_coeff_size = max(coeffs.Size()/col_size,1);
-   int ibeg = col_rank*my_coeff_size;
-   if (ibeg + 2*my_coeff_size > coeffs.Size())
+   // ------------------------------------------------------------------------
+   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
+   // ------------------------------------------------------------------------
+
+   if (power_of_laplace > 0)
   {
-      my_coeff_size = coeffs.Size()-col_rank*my_coeff_size;
-   }
-   else if (ibeg > coeffs.Size() - 1)
-   {
-      my_coeff_size = 0;
-   }
+      // 10.1 Compute Stiffnes Matrix
+      ParBilinearForm k(&fespace);
+      k.AddDomainIntegrator(new DiffusionIntegrator(one));
+      k.Assemble();

-   int iend = ibeg+my_coeff_size;
+      // 10.2 Compute Mass Matrix
+      ParBilinearForm m(&fespace);
+      m.AddDomainIntegrator(new MassIntegrator(one));
+      m.Assemble();
+      HypreParMatrix mass;
+      Array<int> empty;
+      m.FormSystemMatrix(empty, mass);

-
-   for (int i = ibeg; i < iend; i++)
-   {
-      // 10. Reset GridFunction for integer-order PDE solve.
-      x = 0.0;
-
-      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
-      ParBilinearForm a(&fespace);
-      a.AddDomainIntegrator(new DiffusionIntegrator(one));
-      ConstantCoefficient d_i(-poles[i]);
-      a.AddDomainIntegrator(new MassIntegrator(d_i));
-      a.Assemble();
-
-      // 12. Assemble the bilinear form and the corresponding linear system.
-      OperatorPtr A;
+      // 10.3 Form the system of equations
      Vector B, X;
-      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-      // 13. Solve the linear system A X = B.
-      HypreBoomerAMG * prec = new HypreBoomerAMG;
-      prec->SetPrintLevel(-1);
-
-      int print_level = (col_rank==0) ? 3 : 0;
-      if (Mpi::Root())
-      {
-         mfem::out << "\nMPI rank " <<  myid
-                   << ": Solving PDE -Δ u + " << -poles[i]
-                   << " u = " << coeffs[i] << " f " << endl;
-      }
-      CGSolver cg(row_comm);
+      OperatorPtr Op;
+      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
+      HypreBoomerAMG prec;
+      prec.SetPrintLevel(-1);
+      CGSolver cg(MPI_COMM_WORLD);
      cg.SetRelTol(1e-12);
      cg.SetMaxIter(2000);
-      cg.SetPrintLevel(print_level);
-      cg.SetPreconditioner(*prec);
-      cg.SetOperator(*A);
-      cg.Mult(B, X);
-      delete prec;
+      cg.SetPrintLevel(3);
+      cg.SetPreconditioner(prec);
+      cg.SetOperator(*Op);

-      // 14. Recover the solution as a finite element grid function.
-      a.RecoverFEMSolution(X, b, x);
-
-      // 15. Accumulate integer-order PDE solutions.
-      x *= coeffs[i];
-      u += x;
-
-      // 16. Send integer-order PDE solutions to a GLVis server.
-      if (visualize_x)
+      if (Mpi::Root())
      {
-         if (col_rank > 0 && i < iend-1)
+         mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
+                   << " ( f ) " << endl;
+      }
+      for (int i = 0; i < power_of_laplace; i++)
+      {
+         // 10.4 Solve the linear system Op X = B (N times).
+         cg.Mult(B, X);
+         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
+         if (i == power_of_laplace - 1)
         {
-            MPI_Status status;
-            MPI_Recv(nullptr,0,MPI_INT, col_rank-1,0,col_comm,&status);
+            // Needed for visualization and solution verification.
+            k.RecoverFEMSolution(X, b, g);
+            if (integer_order && verification)
+            {
+               // For an integer order PDE, g is also our solution u.
+               u+=g;
+            }
+            if (visualization)
+            {
+               socketstream fout;
+               ostringstream oss_f;
+               fout.open(vishost, visport);
+               fout.precision(8);
+               oss_f.str(""); oss_f.clear();
+               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
+                     << power_of_laplace
+                     << " g = f";
+               fout << "parallel " << num_procs << " " << myid << "\n"
+                    << "solution\n" << pmesh << g
+                    << "window_title '" << oss_f.str() << "'" << flush;
+            }
         }
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream xout(vishost, visport);
+
+         // 10.6 Prepare for next iteration (primal / dual space)
+         mass.Mult(X, B);
+         X.SetSubVectorComplement(ess_tdof_list,0.0);
+      }
+
+      // 10.7 Extract solution for the next step. The b now corresponds to the
+      //      function g in the PDE.
+      const SparseMatrix* rm = fespace.GetRestrictionMatrix();
+      rm->MultTranspose(B, b);
+   }
+
+   // ------------------------------------------------------------------------
+   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
+   //     up the solutions.
+   // ------------------------------------------------------------------------
+   if (!integer_order)
+   {
+      // Setup visualization.
+      socketstream xout, uout;
+      ostringstream oss_x, oss_u;
+      if (visualization)
+      {
+         xout.open(vishost, visport);
         xout.precision(8);
-         ostringstream oss;
-         oss << "Solution of PDE -Δ u + " << -poles[i]
-             << " u = " << coeffs[i] << " f" ;
-         xout << "parallel " << row_size << " " << row_rank << "\n";
-         xout << "solution\n" << pmesh << x
-              << "window_title '" << oss.str() << "'" << flush;
-         if (col_rank < col_size-1)
+         uout.open(vishost, visport);
+         uout.precision(8);
+      }
+      // Iterate over all expansion coefficient that contribute to the
+      // solution.
+      for (int i = 0; i < coeffs.Size(); i++)
+      {
+         if (Mpi::Root())
         {
-            MPI_Send(nullptr,0,MPI_INT,col_rank+1,0,col_comm);
+            mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                      << " u = " << coeffs[i] << " g " << endl;
+         }
+
+         // 11.1 Reset GridFunction for integer-order PDE solve.
+         x = 0.0;
+
+         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
+         ParBilinearForm a(&fespace);
+         a.AddDomainIntegrator(new DiffusionIntegrator(one));
+         ConstantCoefficient d_i(-poles[i]);
+         a.AddDomainIntegrator(new MassIntegrator(d_i));
+         a.Assemble();
+
+         // 11.3 Assemble the bilinear form and the corresponding linear system.
+         OperatorPtr A;
+         Vector B, X;
+         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+         // 11.4 Solve the linear system A X = B.
+         HypreBoomerAMG prec;
+         prec.SetPrintLevel(-1);
+
+         CGSolver cg(MPI_COMM_WORLD);
+         cg.SetRelTol(1e-12);
+         cg.SetMaxIter(2000);
+         cg.SetPrintLevel(3);
+         cg.SetPreconditioner(prec);
+         cg.SetOperator(*A);
+         cg.Mult(B, X);
+
+         // 11.5 Recover the solution as a finite element grid function.
+         a.RecoverFEMSolution(X, b, x);
+
+         // 11.6 Accumulate integer-order PDE solutions.
+         x *= coeffs[i];
+         u += x;
+
+         // 11.7 Send fractional PDE solution to a GLVis server.
+         if (visualization)
+         {
+            oss_x.str(""); oss_x.clear();
+            oss_x << "Step " << progress_steps
+                  << ": Solution of PDE -Δ u + " << -poles[i]
+                  << " u = " << coeffs[i] << " g";
+            xout << "parallel " << num_procs << " " << myid << "\n"
+                 << "solution\n" << pmesh << x
+                 << "window_title '" << oss_x.str() << "'" << flush;
+
+            oss_u.str(""); oss_u.clear();
+            oss_u << "Step " << progress_steps + 1
+                  << ": Solution of fractional PDE (-Δ)^" << alpha
+                  << " u = f";
+            uout << "parallel " << num_procs << " " << myid << "\n"
+                 << "solution\n" << pmesh << u
+                 << "window_title '" << oss_u.str() << "'"
+                 << flush;
         }
      }
   }

-   // 17. Accumulate for the fractional PDE solution
-   MPI_Allreduce(MPI_IN_PLACE, u.GetData(), u.Size(),
-                 MPI_DOUBLE, MPI_SUM,col_comm);
-
-   // 18. Send fractional PDE solution to a GLVis server.
-   if (visualization)
+   // ------------------------------------------------------------------------
+   // 12. (optional) Verify the solution.
+   // ------------------------------------------------------------------------
+   if (verification)
   {
-      if (col_rank == 0)
+      auto solution = [] (const Vector &x)
      {
-         char vishost[] = "localhost";
-         int  visport   = 19916;
-         socketstream uout(vishost, visport);
-         uout.precision(8);
-         ostringstream oss;
-         oss << "Solution of fractional PDE -Δ^" << alpha
-             << " u = f" ;
-         uout << "parallel " << row_size << " " << row_rank << "\n";
-         uout << "solution\n" << pmesh << u
-              << "window_title '" << oss.str() << "'" << flush;
+         double val = 1.0;
+         for (int i=0; i<x.Size(); i++)
+         {
+            val *= sin(M_PI*x(i));
+         }
+         return val;
+      };
+      FunctionCoefficient sol(solution);
+      double l2_error = u.ComputeL2Error(sol);
+
+      if (Mpi::Root())
+      {
+         string analytic_solution,expected_mesh;
+         switch (dim)
+         {
+            case 1:
+               analytic_solution = "sin(π x)";
+               expected_mesh = "inline_segment.mesh";
+               break;
+            case 2:
+               analytic_solution = "sin(π x) sin(π y)";
+               expected_mesh = "inline_quad.mesh";
+               break;
+            default:
+               analytic_solution = "sin(π x) sin(π y) sin(π z)";
+               expected_mesh = "inline_hex.mesh";
+               break;
+         }
+
+         mfem::out << "\n" << string(80,'=')
+                   << "\n\nSolution Verification in "<< dim << "D \n\n"
+                   << "Analytic solution : " << analytic_solution << "\n"
+                   << "Expected mesh     : " << expected_mesh <<"\n"
+                   << "Your mesh         : " << mesh_file << "\n"
+                   << "L2 error          : " << l2_error << "\n\n"
+                   << string(80,'=') << endl;
      }
   }

@@ -450,7 +450,7 @@ int main(int argc, char *argv[])
      for (int ti = 0; !done; )
      {
         // We cannot match exactly the time history of the Run method
-         // since we are explictly telling PETSc to use a time step
+         // since we are explicitly telling PETSc to use a time step
         double dt_real = min(dt, t_final - t);
         ode_solver->Step(*U, t, dt_real);
         ti++;
@@ -39,6 +39,7 @@ set(SRCS
  complex_fem.cpp
  convergence.cpp
  datacollection.cpp
+  dgmassinv.cpp
  doftrans.cpp
  eltrans.cpp
  estimators.cpp
@@ -72,6 +73,7 @@ set(SRCS
  linearform.cpp
  linearform_ext.cpp
  lininteg.cpp
+  lininteg_boundary.cpp
  lininteg_domain.cpp
  lininteg_domain_grad.cpp
  lor/lor.cpp
@@ -88,6 +90,7 @@ set(SRCS
  fespacehierarchy.cpp
  nonlininteg_vectorconvection.cpp
  nonlininteg_vectorconvection_mf.cpp
+  qfunction.cpp
  qinterp/det.cpp
  qinterp/eval_by_nodes.cpp
  qinterp/eval_by_vdim.cpp
@@ -95,6 +98,7 @@ set(SRCS
  qinterp/grad_by_vdim.cpp
  qinterp/grad_phys_by_nodes.cpp
  qinterp/grad_phys_by_vdim.cpp
+  qspace.cpp
  quadinterpolator.cpp
  quadinterpolator_face.cpp
  restriction.cpp
@@ -136,10 +140,13 @@ set(HDRS
  bilinearform.hpp
  bilinearform_ext.hpp
  bilininteg.hpp
+  bilininteg_mass_pa.hpp
  coefficient.hpp
  complex_fem.hpp
  convergence.hpp
  datacollection.hpp
+  dgmassinv.hpp
+  dgmassinv_kernels.hpp
  doftrans.hpp
  eltrans.hpp
  estimators.hpp
@@ -189,9 +196,11 @@ set(HDRS
  nonlinearform.hpp
  nonlinearform_ext.hpp
  nonlininteg.hpp
+  qfunction.hpp
  qinterp/dispatch.hpp
  qinterp/eval.hpp
  qinterp/grad.hpp
+  qspace.hpp
  quadinterpolator.hpp
  quadinterpolator_face.hpp
  restriction.hpp
@@ -992,6 +992,7 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
      mat_e = new SparseMatrix(height);
   }

+   vdofs_.HostRead();
   for (int i = 0; i < vdofs_.Size(); i++)
   {
      int vdof = vdofs_[i];
@@ -333,7 +333,7 @@ public:


   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
+        to it.  Used for transferring ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
@@ -438,6 +438,14 @@ public:
   virtual const Operator *GetOutputRestriction() const
   { return GetRestriction(); }

+   /// @brief Compute serial RAP operator and store it in @a A as a SparseMatrix.
+   void SerialRAP(OperatorHandle &A)
+   {
+      MFEM_ASSERT(mat, "SerialRAP requires the SparseMatrix to be assembled.");
+      ConformingAssemble();
+      A.Reset(mat, false);
+   }
+
   /** @brief Form the linear system A X = B, corresponding to this bilinear
       form and the linear form @a b(.). */
   /** This method applies any necessary transformations to the linear system
@@ -766,7 +774,7 @@ public:
   SparseMatrix &SpMat() { return *mat; }

   /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transfering ownership. */
+        to it.  Used for transferring ownership. */
   SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }

   /// Adds a domain integrator. Assumes ownership of @a bfi.
@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultMF(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultMF(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposeMF(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposeMF(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -251,6 +251,7 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)

 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 {
+   if ( Device::Allows(Backend::CEED_MASK) ) { return; }
   ElementDofOrdering ordering = UsesTensorBasis(*a->FESpace())?
                                 ElementDofOrdering::LEXICOGRAPHIC:
                                 ElementDofOrdering::NATIVE;
@@ -417,7 +418,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -433,7 +434,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -474,7 +475,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            intFaceIntegrators[i]->AddMultTransposePA(int_face_X, int_face_Y);
         }
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -490,7 +491,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
         {
            bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
         }
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -667,7 +668,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -698,7 +699,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -795,7 +796,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, 0, f) += res;
         });
         // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
+         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
      }
   }

@@ -826,7 +827,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
            Y(j, f) += res;
         });
         // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }
@@ -956,6 +957,60 @@ void FABilinearFormExtension::Assemble()
   }
 }

+
+void FABilinearFormExtension::RAP(OperatorHandle &A)
+{
+#ifdef MFEM_USE_MPI
+   if ( auto pa = dynamic_cast<ParBilinearForm*>(a) )
+   {
+      pa->ParallelRAP(*pa->mat, A);
+   }
+   else
+#endif
+   {
+      a->SerialRAP(A);
+   }
+}
+
+void FABilinearFormExtension::EliminateBC(const Array<int> &ess_dofs,
+                                          OperatorHandle &A)
+{
+   MFEM_VERIFY(a->diag_policy == DiagonalPolicy::DIAG_ONE,
+               "Only DiagonalPolicy::DIAG_ONE supported with"
+               " FABilinearFormExtension.");
+#ifdef MFEM_USE_MPI
+   if ( dynamic_cast<ParBilinearForm*>(a) )
+   {
+      A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
+                                          DiagonalPolicy::DIAG_ONE);
+   }
+   else
+#endif
+   {
+      A.As<SparseMatrix>()->EliminateBC(ess_dofs,
+                                        DiagonalPolicy::DIAG_ONE);
+   }
+}
+
+void FABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_dofs,
+                                               OperatorHandle &A)
+{
+   RAP(A);
+   EliminateBC(ess_dofs, A);
+}
+
+void FABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
+                                               Vector &x, Vector &b,
+                                               OperatorHandle &A,
+                                               Vector &X, Vector &B,
+                                               int copy_interior)
+{
+   Operator *A_out;
+   Operator::FormLinearSystem(ess_tdof_list, x, b, A_out, X, B, copy_interior);
+   delete A_out;
+   FormSystemMatrix(ess_tdof_list, A);
+}
+
 void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const
 {
 #ifdef MFEM_USE_MPI
@@ -125,6 +125,15 @@ public:
   FABilinearFormExtension(BilinearForm *form);

   void Assemble();
+   void RAP(OperatorHandle &A);
+   /** @note Always does `DIAG_ONE` policy to be consistent with
+       `Operator::FormConstrainedSystemOperator`. */
+   void EliminateBC(const Array<int> &ess_dofs, OperatorHandle &A);
+   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
+   void FormLinearSystem(const Array<int> &ess_tdof_list,
+                         Vector &x, Vector &b,
+                         OperatorHandle &A, Vector &X, Vector &B,
+                         int copy_interior = 0);
   void Mult(const Vector &x, Vector &y) const;
   void MultTranspose(const Vector &x, Vector &y) const;

@@ -2003,6 +2003,83 @@ void CurlCurlIntegrator::AssembleElementMatrix
   }
 }

+void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                                const FiniteElement &test_fe,
+                                                ElementTransformation &Trans,
+                                                DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   dim = trial_fe.GetDim();
+   int dimc = trial_fe.GetCurlDim();
+   double w;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector D;
+   DenseMatrix curlshape(tr_nd,dimc), curlshape_dFt(tr_nd,dimc), M;
+   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc);
+#else
+   curlshape.SetSize(tr_nd,dimc);
+   curlshape_dFt.SetSize(tr_nd,dimc);
+   te_curlshape.SetSize(te_nd,dimc);
+   te_curlshape_dFt.SetSize(te_nd,dimc);
+#endif
+   elmat.SetSize(te_nd, tr_nd);
+
+   if (MQ) { M.SetSize(dimc); }
+   if (DQ) { D.SetSize(dimc); }
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order;
+      if (trial_fe.Space() == FunctionSpace::Pk)
+      {
+         order = test_fe.GetOrder() + trial_fe.GetOrder() - 2;
+      }
+      else
+      {
+         order = test_fe.GetOrder() + trial_fe.GetOrder() + trial_fe.GetDim() - 1;
+      }
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      Trans.SetIntPoint(&ip);
+
+      w = ip.weight * Trans.Weight();
+      trial_fe.CalcPhysCurlShape(Trans, curlshape_dFt);
+      test_fe.CalcPhysCurlShape(Trans, te_curlshape_dFt);
+
+      if (MQ)
+      {
+         MQ->Eval(M, Trans, ip);
+         M *= w;
+         Mult(te_curlshape_dFt, M, te_curlshape);
+         AddMultABt(te_curlshape, curlshape_dFt, elmat);
+      }
+      else if (DQ)
+      {
+         DQ->Eval(D, Trans, ip);
+         D *= w;
+         AddMultADBt(te_curlshape_dFt,D,curlshape_dFt,elmat);
+      }
+      else
+      {
+         if (Q)
+         {
+            w *= Q->Eval(Trans, ip);
+         }
+         curlshape_dFt *= w;
+         AddMultABt(te_curlshape_dFt, curlshape_dFt, elmat);
+      }
+   }
+}
+
 void CurlCurlIntegrator
 ::ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans,
                     Vector &u, const FiniteElement &fluxelem, Vector &flux,
@@ -2240,6 +2317,84 @@ double VectorCurlCurlIntegrator::GetElementEnergy(
   return 0.5 * energy;
 }

+void MixedCurlIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe, const FiniteElement &test_fe,
+   ElementTransformation &Trans, DenseMatrix &elmat)
+{
+   int dim = trial_fe.GetDim();
+   int trial_dof = trial_fe.GetDof();
+   int test_dof = test_fe.GetDof();
+   int dimc = (dim == 3) ? 3 : 1;
+
+   MFEM_VERIFY(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
+               (dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE),
+               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
+   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
+               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
+               "Test finite element must be in H1/L2");
+
+   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
+
+   if (spaceH1)
+   {
+      dshape.SetSize(trial_dof,dim);
+      curlshape.SetSize(dim*trial_dof,1);
+      dimc = dim;
+   }
+   else
+   {
+      curlshape.SetSize(trial_dof,dimc);
+      elmat_comp.SetSize(test_dof, trial_dof);
+   }
+   elmat.SetSize(dimc * test_dof, trial_dof);
+   shape.SetSize(test_dof);
+   elmat = 0.0;
+
+   double c;
+   Vector d_col;
+   const IntegrationRule *ir = IntRule;
+
+   if (ir == NULL)
+   {
+      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
+      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
+   }
+
+   for (int i = 0; i < ir->GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+      Trans.SetIntPoint(&ip);
+      if (spaceH1)
+      {
+         trial_fe.CalcPhysDShape(Trans, dshape);
+         dshape.GradToCurl(curlshape);
+      }
+      else
+      {
+         trial_fe.CalcPhysCurlShape(Trans, curlshape);
+      }
+      test_fe.CalcPhysShape(Trans, shape);
+      c = ip.weight*Trans.Weight();
+      if (Q)
+      {
+         c *= Q->Eval(Trans, ip);
+      }
+      shape *= c;
+
+      for (int d = 0; d < dimc; ++d)
+      {
+         double * curldata = &(curlshape.GetData())[d*trial_dof];
+         for (int jj = 0; jj < trial_dof; ++jj)
+         {
+            for (int ii = 0; ii < test_dof; ++ii)
+            {
+               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
+            }
+         }
+      }
+   }
+}
+

 void VectorFEMassIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -2586,6 +2741,54 @@ void DivDivIntegrator::AssembleElementMatrix(
   }
 }

+void DivDivIntegrator::AssembleElementMatrix2(
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans,
+   DenseMatrix &elmat)
+{
+   int tr_nd = trial_fe.GetDof();
+   int te_nd = test_fe.GetDof();
+   double c;
+
+#ifdef MFEM_THREAD_SAFE
+   Vector divshape(tr_nd);
+   Vector te_divshape(te_nd);
+#else
+   divshape.SetSize(tr_nd);
+   te_divshape.SetSize(te_nd);
+#endif
+   elmat.SetSize(te_nd,tr_nd);
+
+   const IntegrationRule *ir = IntRule;
+   if (ir == NULL)
+   {
+      int order = 2 * max(test_fe.GetOrder(),
+                          trial_fe.GetOrder()) - 2; // <--- OK for RTk
+      ir = &IntRules.Get(test_fe.GetGeomType(), order);
+   }
+
+   elmat = 0.0;
+
+   for (int i = 0; i < ir -> GetNPoints(); i++)
+   {
+      const IntegrationPoint &ip = ir->IntPoint(i);
+
+      trial_fe.CalcDivShape(ip,divshape);
+      test_fe.CalcDivShape(ip,te_divshape);
+
+      Trans.SetIntPoint (&ip);
+      c = ip.weight / Trans.Weight();
+
+      if (Q)
+      {
+         c *= Q -> Eval (Trans, ip);
+      }
+
+      te_divshape *= c;
+      AddMultVWt(te_divshape, divshape, elmat);
+   }
+}

 void VectorDiffusionIntegrator::AssembleElementMatrix(
   const FiniteElement &el,
@@ -3780,7 +3983,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
      for (i = 0; i < ndof1; i++)
         for (j = 0; j < face_ndof; j++)
         {
-            elmat(i, j) -= shape1_n(i) * face_shape(j);
+            elmat(i, j) += shape1_n(i) * face_shape(j);
         }
      if (ndof2)
      {
@@ -3788,7 +3991,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
         for (i = 0; i < ndof2; i++)
            for (j = 0; j < face_ndof; j++)
            {
-               elmat(ndof1+i, j) += shape2_n(i) * face_shape(j);
+               elmat(ndof1+i, j) -= shape2_n(i) * face_shape(j);
            }
      }
   }
@@ -2174,6 +2174,7 @@ public:
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
 class MassIntegrator: public BilinearFormIntegrator
 {
+   friend class DGMassInverse;
 protected:
 #ifndef MFEM_THREAD_SAFE
   Vector shape, te_shape;
@@ -2524,6 +2525,7 @@ private:
 #ifndef MFEM_THREAD_SAFE
   Vector D;
   DenseMatrix curlshape, curlshape_dFt, M;
+   DenseMatrix te_curlshape, te_curlshape_dFt;
   DenseMatrix vshape, projcurl;
 #endif

@@ -2557,6 +2559,11 @@ public:
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);

+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
   virtual void ComputeElementFlux(const FiniteElement &el,
                                   ElementTransformation &Trans,
                                   Vector &u, const FiniteElement &fluxelem,
@@ -2602,6 +2609,35 @@ public:
                                   const Vector &elfun);
 };

+/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
+    an optional scalar coefficient, and v is a vector with components v_i in
+    the L2 or H1 space. This integrator handles 3 cases:
+    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
+    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
+    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
+        2D vector field with components v_i in L^2 or H^1 space.
+    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
+class MixedCurlIntegrator : public BilinearFormIntegrator
+{
+protected:
+   Coefficient *Q;
+
+private:
+   Vector shape;
+   DenseMatrix dshape;
+   DenseMatrix curlshape;
+   DenseMatrix elmat_comp;
+public:
+   MixedCurlIntegrator() : Q{NULL} { }
+   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
+   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+};
+
 /** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
    vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
    H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
@@ -2725,7 +2761,7 @@ protected:

 private:
 #ifndef MFEM_THREAD_SAFE
-   Vector divshape;
+   Vector divshape, te_divshape;
 #endif

   // PA extension
@@ -2737,11 +2773,18 @@ private:

 public:
   DivDivIntegrator() { Q = NULL; }
-   DivDivIntegrator(Coefficient &q) : Q(&q) { }
+   DivDivIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q) { }

   virtual void AssembleElementMatrix(const FiniteElement &el,
                                      ElementTransformation &Trans,
                                      DenseMatrix &elmat);
+
+   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
+                                       const FiniteElement &test_fe,
+                                       ElementTransformation &Trans,
+                                       DenseMatrix &elmat);
+
   const Coefficient *GetCoefficient() const { return Q; }
 };

@@ -30,7 +30,16 @@ void ConvectionIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFConvectionIntegrator(fes, *ir, Q, alpha);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFConvectionIntegrator(*this, fes, Q, alpha);
+      }
+      else
+      {
+         ceedOp = new ceed::MFConvectionIntegrator(fes, *ir, Q, alpha);
+      }
      return;
   }
   MFEM_ABORT("Error: ConvectionIntegrator::AssembleMF only implemented with"
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"

@@ -1386,7 +1387,16 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAConvectionIntegrator(*this, fes, Q, alpha);
+      }
+      else
+      {
+         ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
+      }
      return;
   }
   const int dims = el.GetDim();
@@ -1399,66 +1409,10 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * ne, mt);
-   Vector vel;
-   if (VectorConstantCoefficient *cQ =
-          dynamic_cast<VectorConstantCoefficient*>(Q))
-   {
-      vel = cQ->GetVec();
-   }
-   else if (VectorGridFunctionCoefficient *vgfQ =
-               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
-   {
-      vel.SetSize(dim * nq * ne, mt);

-      const GridFunction *gf = vgfQ->GetGridFunction();
-      const FiniteElementSpace &gf_fes = *gf->FESpace();
-      const QuadratureInterpolator *qi(gf_fes.GetQuadratureInterpolator(*ir));
-      const bool use_tensor_products = UsesTensorBasis(gf_fes);
-      const ElementDofOrdering ordering = use_tensor_products ?
-                                          ElementDofOrdering::LEXICOGRAPHIC :
-                                          ElementDofOrdering::NATIVE;
-      const Operator *R = gf_fes.GetElementRestriction(ordering);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);

-      Vector xe(R->Height(), mt);
-      xe.UseDevice(true);
-
-      R->Mult(*gf, xe);
-      qi->SetOutputLayout(QVectorLayout::byVDIM);
-      qi->DisableTensorProducts(!use_tensor_products);
-      qi->Values(xe,vel);
-   }
-   else if (VectorQuadratureFunctionCoefficient* vqfQ =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      vel.SetSize(dim * nq * ne);
-      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
-      DenseMatrix MQ_ir;
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         Q->Eval(MQ_ir, T, *ir);
-         for (int q = 0; q < nq; ++q)
-         {
-            for (int i = 0; i < dim; ++i)
-            {
-               C(i,q,e) = MQ_ir(i,q);
-            }
-         }
-      }
-   }
   PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
                     vel, alpha, pa_data);
 }
@@ -1497,6 +1451,7 @@ static void PAConvectionApply(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPAConvectionApply3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
         case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
         case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
         case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -1548,6 +1503,7 @@ static void PAConvectionApplyT(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPAConvectionApplyT3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
         case 0x23: return SmemPAConvectionApplyT3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
         case 0x24: return SmemPAConvectionApplyT3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
         case 0x26: return SmemPAConvectionApplyT3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "restriction.hpp"

 using namespace std;
@@ -136,6 +137,9 @@ static void PADGTraceSetup(const int dim,

 void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
 {
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
+
   nf = fes.GetNFbyType(type);
   if (nf==0) { return; }
   // Assumes tensor-product elements
@@ -153,93 +157,29 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
   geom = mesh->GetFaceGeometricFactors(
             *ir,
             FaceGeometricFactors::DETERMINANTS |
-             FaceGeometricFactors::NORMALS, type);
+             FaceGeometricFactors::NORMALS, type, mt);
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
-   Vector vel;
-   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
-                                        (u))
-   {
-      vel = c_u->GetVec();
-   }
-   else if (VectorQuadratureFunctionCoefficient* qf_u =
-               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
-   {
-      // Assumed to be in lexicographical ordering
-      const QuadratureFunction &qFun = qf_u->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
+   FaceQuadratureSpace qs(*mesh, *ir, type);
+   CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);
+
+   CoefficientVector r(qs, CoefficientStorage::COMPRESSED);
+   if (rho == nullptr)
   {
-      vel.SetSize(dim * nq * nf);
-      auto C = Reshape(vel.HostWrite(), dim, nq, nf);
-      Vector Vq(dim);
-      int f_ind = 0;
-      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
-      {
-         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse())
-         {
-            // We skip nonconforming coarse faces as they are treated
-            // by the corresponding nonconforming fine faces.
-            continue;
-         }
-         else if ( face.IsOfFaceType(type) )
-         {
-            const int mask = FaceElementTransformations::HAVE_ELEM1 |
-                             FaceElementTransformations::HAVE_LOC1;
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
-            for (int q = 0; q < nq; ++q)
-            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                      quad1D, q);
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               u->Eval(Vq, *T.Elem1, eip1);
-               for (int i = 0; i < dim; ++i)
-               {
-                  C(i,iq,f_ind) = Vq(i);
-               }
-            }
-            f_ind++;
-         }
-      }
-      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+      r.SetConstant(1.0);
   }
-   Vector r;
-   if (rho==nullptr)
+   else if (ConstantCoefficient *const_rho = dynamic_cast<ConstantCoefficient*>
+                                             (rho))
   {
-      r.SetSize(1);
-      r(0) = 1.0;
-   }
-   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
-   {
-      r.SetSize(1);
-      r(0) = c_rho->constant;
+      r.SetConstant(const_rho->constant);
   }
   else if (QuadratureFunctionCoefficient* qf_rho =
               dynamic_cast<QuadratureFunctionCoefficient*>(rho))
   {
-      const QuadratureFunction &qFun = qf_rho->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * nf,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+      r.MakeRef(qf_rho->GetQuadFunction());
   }
   else
   {
@@ -251,45 +191,42 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
      {
         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse())
+         if (face.IsNonconformingCoarse() || !face.IsOfFaceType(type))
         {
            // We skip nonconforming coarse faces as they are treated
            // by the corresponding nonconforming fine faces.
            continue;
         }
-         else if ( face.IsOfFaceType(type) )
+         FaceElementTransformations &T =
+            *fes.GetMesh()->GetFaceElementTransformations(f);
+         for (int q = 0; q < nq; ++q)
         {
-            FaceElementTransformations &T =
-               *fes.GetMesh()->GetFaceElementTransformations(f);
-            for (int q = 0; q < nq; ++q)
+            // Convert to lexicographic ordering
+            int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                   quad1D, q);
+
+            T.SetAllIntPoints(&ir->IntPoint(q));
+            const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+            const IntegrationPoint &eip2 = T.GetElement2IntPoint();
+            double rq;
+
+            if (face.IsBoundary())
            {
-               // Convert to lexicographic ordering
-               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                      quad1D, q);
-
-               T.SetAllIntPoints(&ir->IntPoint(q));
-               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-               double rq;
-
-               if ( face.IsBoundary() )
-               {
-                  rq = rho->Eval(*T.Elem1, eip1);
-               }
-               else
-               {
-                  double udotn = 0.0;
-                  for (int d=0; d<dim; ++d)
-                  {
-                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
-                  }
-                  if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
-                  else { rq = rho->Eval(*T.Elem1, eip1); }
-               }
-               C(iq,f_ind) = rq;
+               rq = rho->Eval(*T.Elem1, eip1);
            }
-            f_ind++;
+            else
+            {
+               double udotn = 0.0;
+               for (int d=0; d<dim; ++d)
+               {
+                  udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+               }
+               if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
+               else { rq = rho->Eval(*T.Elem1, eip1); }
+            }
+            C(iq,f_ind) = rq;
         }
+         f_ind++;
      }
      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
   }
@@ -695,6 +632,7 @@ static void PADGTraceApply(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPADGTraceApply3D<2,2,1>(NF,B,Bt,op,x,y);
         case 0x23: return SmemPADGTraceApply3D<2,3,1>(NF,B,Bt,op,x,y);
         case 0x34: return SmemPADGTraceApply3D<3,4,2>(NF,B,Bt,op,x,y);
         case 0x45: return SmemPADGTraceApply3D<4,5,2>(NF,B,Bt,op,x,y);
@@ -1124,6 +1062,7 @@ static void PADGTraceApplyTranspose(const int dim,
   {
      switch ((D1D << 4 ) | Q1D)
      {
+         case 0x22: return SmemPADGTraceApplyTranspose3D<2,2>(NF,B,Bt,op,x,y);
         case 0x23: return SmemPADGTraceApplyTranspose3D<2,3>(NF,B,Bt,op,x,y);
         case 0x34: return SmemPADGTraceApplyTranspose3D<3,4>(NF,B,Bt,op,x,y);
         case 0x45: return SmemPADGTraceApplyTranspose3D<4,5>(NF,B,Bt,op,x,y);
@@ -33,7 +33,16 @@ void DiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
      MFEM_VERIFY(!VQ && !MQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
-      ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFDiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: DiffusionIntegrator::AssembleMF only implemented with"
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;
@@ -271,18 +272,21 @@ void PADiffusionSetup3D(const int Q1D,
                  D(qx,qy,qz,1,e) = D12; // 1,2
                  D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3

-                  const double D21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
                  const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
                  const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);

                  const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);

-                  D(qx,qy,qz,3,e) = symmetric ? D22 : D21; // 2,2 or 2,1
                  D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
                  D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3

-                  if (!symmetric)
+                  if (symmetric)
                  {
+                     D(qx,qy,qz,3,e) = D22; // 2,2
+                  }
+                  else
+                  {
+                     D(qx,qy,qz,3,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
                     D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
                     D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
                     D(qx,qy,qz,8,e) = D33; // 3,3
@@ -365,7 +369,16 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
      MFEM_VERIFY(!VQ && !MQ,
                  "Only scalar coefficient supported for DiffusionIntegrator"
                  " with libCEED");
-      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   const int dims = el.GetDim();
@@ -378,120 +391,21 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
-   int coeffDim = 1;
-   Vector coeff;
-   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
-   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
-   {
-      MFEM_VERIFY(SMQ->GetSize() == dim, "");
-      coeffDim = symmDims;
-      coeff.SetSize(symmDims * nq * ne);

-      DenseSymmetricMatrix sym_mat;
-      sym_mat.SetSize(dim);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);

-      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
+   if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (VQ) { coeff.Project(*VQ); }
+   else if (Q) { coeff.Project(*Q); }
+   else { coeff.SetConstant(1.0); }

-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
-            int cnt = 0;
-            for (int i=0; i<dim; ++i)
-               for (int j=i; j<dim; ++j, ++cnt)
-               {
-                  C(cnt, p, e) = sym_mat(i,j);
-               }
-         }
-      }
-   }
-   else if (MQ)
-   {
-      symmetric = false;
-      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dims*dims);
+   const int pa_size = symmetric ? symmDims : dims*dims;

-      coeffDim = MQfullDim;
-
-      coeff.SetSize(MQfullDim * nq * ne);
-
-      DenseMatrix mat;
-      mat.SetSize(dim);
-
-      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            MQ->Eval(mat, *tr, ir->IntPoint(p));
-            for (int i=0; i<dim; ++i)
-               for (int j=0; j<dim; ++j)
-               {
-                  C(j+(i*dim), p, e) = mat(i,j);
-               }
-         }
-      }
-   }
-   else if (VQ)
-   {
-      MFEM_VERIFY(VQ->GetVDim() == dim, "");
-      coeffDim = VQ->GetVDim();
-      coeff.SetSize(coeffDim * nq * ne);
-      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-      Vector DM(coeffDim);
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            VQ->Eval(DM, *tr, ir->IntPoint(p));
-            for (int i=0; i<coeffDim; ++i)
-            {
-               C(i, p, e) = DM[i];
-            }
-         }
-      }
-   }
-   else if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
-   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
+   pa_data.SetSize(pa_size * nq * ne, mt);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne, ir->GetWeights(),
                    geom->J, coeff, pa_data);
 }

@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"

 using namespace std;

@@ -209,44 +210,8 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
               "PA requires test and trial space to have same number of quadrature points!");
   pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());

-   Vector coeff;
-
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");
-
-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *trial_fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);

   PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
                   ne, ir->GetWeights(), geom->J, coeff, pa_data);
@@ -865,4 +830,3 @@ void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 }

 } // namespace mfem
-
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qspace.hpp"

 using namespace std;

@@ -967,8 +968,6 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dim = mesh->Dimension();
   MFEM_VERIFY(dim == 2 || dim == 3, "");

-   const int dimc = (dim == 3) ? 3 : 1;
-
   ne = fes.GetNE();
   geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
   mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -978,88 +977,19 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");

-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }

-   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
-
-   symmetric = (SMQ || MQ == NULL);
-
-   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+   const int sym_dims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? sym_dims : dim*dim);
   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   Vector coeff(coeffDim * ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
-   {
-      Vector DM(DQ ? coeffDim : 0);
-      DenseMatrix GM;
-      DenseSymmetricMatrix SM;
-
-      if (DQ)
-      {
-         MFEM_VERIFY(coeffDim == dimc, "");
-      }
-      if (SMQ)
-      {
-         SM.SetSize(dimc);
-         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
-      }
-      else if (MQ)
-      {
-         GM.SetSize(dimc);
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            if (SMQ)
-            {
-               SMQ->Eval(SM, *tr, ir->IntPoint(p));
-
-               int cnt = 0;
-               for (int i=0; i<dimc; ++i)
-                  for (int j=i; j<dimc; ++j, ++cnt)
-                  {
-                     coeffh(cnt, p, e) = SM(i,j);
-                  }
-
-            }
-            else if (MQ)
-            {
-               MQ->Eval(GM, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dimc; ++i)
-                  for (int j=0; j<dimc; ++j)
-                  {
-                     coeffh(j+(i*dimc), p, e) = GM(i,j);
-                  }
-
-            }
-            else if (DQ)
-            {
-               DQ->Eval(DM, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = DM[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
-
   if (el->GetDerivType() != mfem::FiniteElement::CURL)
   {
      MFEM_ABORT("Unknown kernel.");
@@ -1067,7 +997,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)

   if (dim == 3)
   {
-      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
+      PACurlCurlSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J, coeff,
                        pa_data);
   }
   else
@@ -3489,20 +3419,8 @@ void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), nq, ne);
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeffh(p, e) = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   if (dim == 2)
   {
@@ -3593,38 +3511,11 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   const int ndata = curlSpaces ? (coeffDim == 1 ? 1 : 9) : symmDims;
   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   Vector coeff(coeffDim * nq * ne);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ)
-   {
-      Vector V(coeffDim);
-      if (DQ)
-      {
-         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-
-         for (int p=0; p<nq; ++p)
-         {
-            if (DQ)
-            {
-               DQ->Eval(V, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = V[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }

   if (testType == mfem::FiniteElement::CURL &&
       trialType == mfem::FiniteElement::CURL && dim == 3)
@@ -5146,38 +5037,11 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace

   pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());

-   Vector coeff(coeffDim * nq * ne);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ)
-   {
-      Vector V(coeffDim);
-      if (DQ)
-      {
-         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
-      }
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-
-         for (int p=0; p<nq; ++p)
-         {
-            if (DQ)
-            {
-               DQ->Eval(V, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = V[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::FULL);
+   if (Q) { coeff.Project(*Q); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }

   if (trialType == mfem::FiniteElement::CURL && dim == 3)
   {
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qspace.hpp"

 using namespace std;

@@ -24,18 +25,20 @@ namespace mfem

 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
                   Vector &coeff_,
                   Vector &op)
 {
+   const bool symmetric = (coeffDim != 4);
   const int NQ = Q1D*Q1D;
   auto W = w.Read();

   auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto coeff = Reshape(coeff_.Read(), NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 3, NE);
+   auto C = Reshape(coeff_.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 3 : 4, NE);

   MFEM_FORALL(e, NE,
   {
@@ -45,28 +48,60 @@ void PAHdivSetup2D(const int Q1D,
         const double J21 = J(q,1,0,e);
         const double J12 = J(q,0,1,e);
         const double J22 = J(q,1,1,e);
-         const double c_detJ = W[q] * coeff(q, e) / ((J11*J22)-(J21*J12));
-         // (c/detJ) J^T J
-         y(q,0,e) = c_detJ * (J11*J11 + J21*J21); // 1,1
-         y(q,1,e) = c_detJ * (J11*J12 + J21*J22); // 1,2
-         y(q,2,e) = c_detJ * (J12*J12 + J22*J22); // 2,2
+         const double c_detJ = W[q] / ((J11*J22)-(J21*J12));
+
+         // (1/detJ) J^T C J
+         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
+         {
+            const double C11 = C(0,q,e);
+            const double C12 = C(1,q,e);
+            const double C21 = symmetric ? C12 : C(2,q,e);
+            const double C22 = symmetric ? C(2,q,e) : C(3,q,e);
+            const double R11 = C11*J11 + C12*J21;
+            const double R21 = C21*J11 + C22*J21;
+            const double R12 = C11*J12 + C12*J22;
+            const double R22 = C21*J12 + C22*J22;
+
+            y(q,0,e) = c_detJ * (J11*R11 + J21*R21); // 1,1
+            y(q,1,e) = c_detJ * (J11*R12 + J21*R22); // 1,2
+
+            if (symmetric)
+            {
+               y(q,2,e) = c_detJ * (J12*R12 + J22*R22); // 2,2
+            }
+            else
+            {
+               y(q,2,e) = c_detJ * (J12*R11 + J22*R21); // 2,1
+               y(q,3,e) = c_detJ * (J12*R12 + J22*R22); // 2,2
+            }
+         }
+         else // Vector or scalar coefficient
+         {
+            const double C1 = C(0,q,e);
+            const double C2 = (coeffDim == 2 ? C(1,q,e) : C1);
+            y(q,0,e) = c_detJ * (J11*C1*J11 + J21*C2*J21); // 1,1
+            y(q,1,e) = c_detJ * (J11*C1*J12 + J21*C2*J22); // 1,2
+            y(q,2,e) = c_detJ * (J12*C1*J12 + J22*C2*J22); // 2,2
+         }
      }
   });
 }

 // PA H(div) Mass Assemble 3D kernel
 void PAHdivSetup3D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
                   Vector &coeff_,
                   Vector &op)
 {
+   const bool symmetric = (coeffDim != 9);
   const int NQ = Q1D*Q1D*Q1D;
   auto W = w.Read();
   auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto coeff = Reshape(coeff_.Read(), NQ, NE);
-   auto y = Reshape(op.Write(), NQ, 6, NE);
+   auto C = Reshape(coeff_.Read(), coeffDim, NQ, NE);
+   auto y = Reshape(op.Write(), NQ, symmetric ? 6 : 9, NE);

   MFEM_FORALL(e, NE,
   {
@@ -84,14 +119,58 @@ void PAHdivSetup3D(const int Q1D,
         const double detJ = J11 * (J22 * J33 - J32 * J23) -
         /* */               J21 * (J12 * J33 - J32 * J13) +
         /* */               J31 * (J12 * J23 - J22 * J13);
-         const double c_detJ = W[q] * coeff(q, e) / detJ;
-         // (c/detJ) J^T J
-         y(q,0,e) = c_detJ * (J11*J11 + J21*J21 + J31*J31); // 1,1
-         y(q,1,e) = c_detJ * (J12*J11 + J22*J21 + J32*J31); // 2,1
-         y(q,2,e) = c_detJ * (J13*J11 + J23*J21 + J33*J31); // 3,1
-         y(q,3,e) = c_detJ * (J12*J12 + J22*J22 + J32*J32); // 2,2
-         y(q,4,e) = c_detJ * (J13*J12 + J23*J22 + J33*J32); // 3,2
-         y(q,5,e) = c_detJ * (J13*J13 + J23*J23 + J33*J33); // 3,3
+         const double c_detJ = W[q] / detJ;
+
+         // (1/detJ) J^T C J
+         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
+         {
+            double M[3][3];
+            M[0][0] = C(0, q, e);
+            M[0][1] = C(1, q, e);
+            M[0][2] = C(2, q, e);
+            M[1][0] = (!symmetric) ? C(3, q, e) : M[0][1];
+            M[1][1] = (!symmetric) ? C(4, q, e) : C(3, q, e);
+            M[1][2] = (!symmetric) ? C(5, q, e) : C(4, q, e);
+            M[2][0] = (!symmetric) ? C(6, q, e) : M[0][2];
+            M[2][1] = (!symmetric) ? C(7, q, e) : M[1][2];
+            M[2][2] = (!symmetric) ? C(8, q, e) : C(5, q, e);
+
+            int idx = 0;
+            for (int i=0; i<3; ++i)
+               for (int j = (symmetric ? i : 0); j<3; ++j)
+               {
+                  y(q,idx,e) = 0.0;
+                  for (int k=0; k<3; ++k)
+                  {
+                     double MJ_kj = 0.0;
+                     for (int l=0; l<3; ++l)
+                     {
+                        MJ_kj += M[k][l] * J(q,l,j,e);
+                     }
+
+                     y(q,idx,e) += J(q,k,i,e) * MJ_kj;
+                  }
+
+                  y(q,idx,e) *= c_detJ;
+                  idx++;
+               }
+         }
+         else  // Vector or scalar coefficient version
+         {
+            int idx = 0;
+            for (int i=0; i<3; ++i)
+               for (int j=i; j<3; ++j)
+               {
+                  y(q,idx,e) = 0.0;
+                  for (int k=0; k<3; ++k)
+                  {
+                     y(q,idx,e) += J(q,k,i,e) * C(coeffDim == 3 ? k : 0, q, e) * J(q,k,j,e);
+                  }
+
+                  y(q,idx,e) *= c_detJ;
+                  idx++;
+               }
+         }
      }
   });
 }
@@ -99,6 +178,7 @@ void PAHdivSetup3D(const int Q1D,
 void PAHdivMassApply2D(const int D1D,
                       const int Q1D,
                       const int NE,
+                       const bool symmetric,
                       const Array<double> &Bo_,
                       const Array<double> &Bc_,
                       const Array<double> &Bot_,
@@ -115,7 +195,7 @@ void PAHdivMassApply2D(const int D1D,
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
   auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
   auto Bct = Reshape(Bct_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 3, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
   auto x = Reshape(x_.Read(), 2*(D1D-1)*D1D, NE);
   auto y = Reshape(y_.ReadWrite(), 2*(D1D-1)*D1D, NE);

@@ -178,11 +258,12 @@ void PAHdivMassApply2D(const int D1D,
         {
            const double O11 = op(qx,qy,0,e);
            const double O12 = op(qx,qy,1,e);
-            const double O22 = op(qx,qy,2,e);
+            const double O21 = symmetric ? O12 : op(qx,qy,2,e);
+            const double O22 = symmetric ? op(qx,qy,2,e) : op(qx,qy,3,e);
            const double massX = mass[qy][qx][0];
            const double massY = mass[qy][qx][1];
            mass[qy][qx][0] = (O11*massX)+(O12*massY);
-            mass[qy][qx][1] = (O12*massX)+(O22*massY);
+            mass[qy][qx][1] = (O21*massX)+(O22*massY);
         }
      }

@@ -225,9 +306,179 @@ void PAHdivMassApply2D(const int D1D,
   }); // end of element loop
 }

+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAHdivMassApply2D(const int NE,
+                           const bool symmetric,
+                           const Array<double> &Bo_,
+                           const Array<double> &Bc_,
+                           const Array<double> &Bot_,
+                           const Array<double> &Bct_,
+                           const Vector &op_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(Bot_);
+   MFEM_CONTRACT_VAR(Bct_);
+
+   static constexpr int VDIM = 2;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
+   const auto x = Reshape(x_.Read(), D1D*(D1D-1), VDIM, NE);
+   auto y = y_.ReadWrite();
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, VDIM,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED double smo[MQ1*(MD1-1)];
+      DeviceMatrix Bo(smo, D1D-1, Q1D);
+
+      MFEM_SHARED double smc[MQ1*MD1];
+      DeviceMatrix Bc(smc, D1D, Q1D);
+
+      MFEM_SHARED double sm0[VDIM*MDQ*MDQ];
+      MFEM_SHARED double sm1[VDIM*MDQ*MDQ];
+      DeviceMatrix X(sm0, D1D*(D1D-1), VDIM);
+      DeviceCube QD(sm1, Q1D, D1D, VDIM);
+      DeviceCube QQ(sm0, Q1D, Q1D, VDIM);
+
+      // Load X, Bo and Bc into shared memory
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               if (qx < D1D && dy < (D1D-1)) { X(qx + dy*D1D,vd) = x(qx+dy*D1D,vd,e); }
+               if (tidz == 0)
+               {
+                  if (dy < (D1D-1)) { Bo(dy,qx) = bo(qx,dy); }
+                  Bc(dy,qx) = bc(qx,dy);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply B operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceCube Xxy(X, nx, ny, VDIM);
+         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double dq = 0.0;
+               for (int dx = 0; dx < nx; ++dx)
+               {
+                  dq += Xxy(dx,dy,vd) * Bx(dx,qx);
+               }
+               QD(qx,dy,vd) = dq;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix By = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double qq = 0.0;
+               for (int dy = 0; dy < ny; ++dy)
+               {
+                  qq += QD(qx,dy,vd) * By(dy,qy);
+               }
+               QQ(qx,qy,vd) = qq;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply D operator
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               const double Qx = QQ(qx,qy,0);
+               const double Qy = QQ(qx,qy,1);
+
+               const double D11 = D(qx,qy,0,e);
+               const double D12 = D(qx,qy,1,e);
+               const double D21 = symmetric ? D12 : D(qx,qy,2,e);
+               const double D22 = symmetric ? D(qx,qy,2,e) : D(qx,qy,3,e);
+
+               QQ(qx,qy,0) = D11*Qx + D12*Qy;
+               QQ(qx,qy,1) = D21*Qx + D22*Qy;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double qd = 0.0;
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
+               }
+               QD(dx,qy,vd) = qd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
+         DeviceTensor<4> Yxy(y, nx, ny, VDIM, NE);
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double dd = 0.0;
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  dd += QD(dx,qy,vd) * Bty(dy,qy);
+               }
+               Yxy(dx,dy,vd,e) += dd;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
@@ -238,7 +489,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,

   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 3, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
   auto diag = Reshape(diag_.ReadWrite(), 2*(D1D-1)*D1D, NE);

   MFEM_FORALL(e, NE,
@@ -259,7 +510,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
               for (int qy = 0; qy < Q1D; ++qy)
               {
                  const double wy = (c == 1) ? Bc(qy,dy) : Bo(qy,dy);
-                  mass[qx] += wy*wy*((c == 0) ? op(qx,qy,0,e) : op(qx,qy,2,e));
+                  mass[qx] += wy*wy*((c == 0) ? op(qx,qy,0,e) : op(qx,qy,symmetric ? 2 : 3,e));
               }
            }

@@ -283,6 +534,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 void PAHdivMassAssembleDiagonal3D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
@@ -294,7 +546,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,

   auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto diag = Reshape(diag_.ReadWrite(), 3*(D1D-1)*(D1D-1)*D1D, NE);

   MFEM_FORALL(e, NE,
@@ -307,7 +559,8 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
         const int D1Dy = (c == 1) ? D1D : D1D - 1;
         const int D1Dx = (c == 0) ? D1D : D1D - 1;

-         const int opc = (c == 0) ? 0 : ((c == 1) ? 3 : 5);
+         const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
+                                         (symmetric ? 5 : 8));

         double mass[HDIV_MAX_Q1D];

@@ -350,6 +603,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
 void PAHdivMassApply3D(const int D1D,
                       const int Q1D,
                       const int NE,
+                       const bool symmetric,
                       const Array<double> &Bo_,
                       const Array<double> &Bc_,
                       const Array<double> &Bot_,
@@ -366,7 +620,7 @@ void PAHdivMassApply3D(const int D1D,
   auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
   auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
   auto Bct = Reshape(Bct_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 6, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
   auto x = Reshape(x_.Read(), 3*(D1D-1)*(D1D-1)*D1D, NE);
   auto y = Reshape(y_.ReadWrite(), 3*(D1D-1)*(D1D-1)*D1D, NE);

@@ -461,15 +715,19 @@ void PAHdivMassApply3D(const int D1D,
               const double O11 = op(qx,qy,qz,0,e);
               const double O12 = op(qx,qy,qz,1,e);
               const double O13 = op(qx,qy,qz,2,e);
-               const double O22 = op(qx,qy,qz,3,e);
-               const double O23 = op(qx,qy,qz,4,e);
-               const double O33 = op(qx,qy,qz,5,e);
+               const double O21 = symmetric ? O12 : op(qx,qy,qz,3,e);
+               const double O22 = symmetric ? op(qx,qy,qz,3,e) : op(qx,qy,qz,4,e);
+               const double O23 = symmetric ? op(qx,qy,qz,4,e) : op(qx,qy,qz,5,e);
+               const double O31 = symmetric ? O13 : op(qx,qy,qz,6,e);
+               const double O32 = symmetric ? O23 : op(qx,qy,qz,7,e);
+               const double O33 = symmetric ? op(qx,qy,qz,5,e) : op(qx,qy,qz,8,e);
+
               const double massX = mass[qz][qy][qx][0];
               const double massY = mass[qz][qy][qx][1];
               const double massZ = mass[qz][qy][qx][2];
               mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
-               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
-               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
+               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
            }
         }
      }
@@ -537,6 +795,337 @@ void PAHdivMassApply3D(const int D1D,
   }); // end of element loop
 }

+template<int T_D1D = 0, int T_Q1D = 0>
+void SmemPAHdivMassApply3D(const int NE,
+                           const bool symmetric,
+                           const Array<double> &Bo_,
+                           const Array<double> &Bc_,
+                           const Array<double> &Bot_,
+                           const Array<double> &Bct_,
+                           const Vector &op_,
+                           const Vector &x_,
+                           Vector &y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   MFEM_CONTRACT_VAR(Bot_);
+   MFEM_CONTRACT_VAR(Bct_);
+
+   static constexpr int VDIM = 3;
+
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
+   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
+   const auto D = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   const auto x = Reshape(x_.Read(), D1D*(D1D-1)*(D1D-1), VDIM, NE);
+   auto y = y_.ReadWrite();
+
+   MFEM_FORALL_3D(e, NE, Q1D, Q1D, VDIM,
+   {
+      const int tidz = MFEM_THREAD_ID(z);
+
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+      MFEM_SHARED double smo[MQ1*(MD1-1)];
+      DeviceMatrix Bo(smo, D1D-1, Q1D);
+
+      MFEM_SHARED double smc[MQ1*MD1];
+      DeviceMatrix Bc(smc, D1D, Q1D);
+
+      MFEM_SHARED double sm0[VDIM*MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[VDIM*MDQ*MDQ*MDQ];
+      DeviceMatrix X(sm0, D1D*(D1D-1)*(D1D-1), VDIM);
+      DeviceTensor<4> QDD(sm1, Q1D, D1D, D1D, VDIM);
+      DeviceTensor<4> QQD(sm0, Q1D, Q1D, D1D, VDIM);
+      DeviceTensor<4> QQQ(sm1, Q1D, Q1D, Q1D, VDIM);
+      DeviceTensor<4> DQQ(sm0, D1D, Q1D, Q1D, VDIM);
+      DeviceTensor<4> DDQ(sm1, D1D, D1D, Q1D, VDIM);
+
+      // Load X into shared memory
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         MFEM_FOREACH_THREAD(dz,y,D1D-1)
+         {
+            MFEM_FOREACH_THREAD(dy,x,D1D-1)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  X(dx+(dy+dz*(D1D-1))*D1D,vd) = x(dx+(dy+dz*(D1D-1))*D1D,vd,e);
+               }
+            }
+         }
+      }
+      // Load Bo and Bc into shared memory
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(d,y,D1D-1)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bo(d,q) = bo(q,d);
+            }
+         }
+         MFEM_FOREACH_THREAD(d,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bc(d,q) = bc(q,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply B operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceTensor<4> Xxyz(X, nx, ny, nz, VDIM);
+         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dx = 0; dx < nx; ++dx)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += Xxyz(dx,dy,dz,vd) * Bx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { QDD(qx,dy,dz,vd) = u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceMatrix By = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dy = 0; dy < ny; ++dy)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += QDD(qx,dy,dz,vd) * By(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { QQD(qx,qy,dz,vd) = u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceMatrix Bz = (vd == 2) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += QQD(qx,qy,dz,vd) * Bz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { QQQ(qx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply D operator
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(qx,x,Q1D)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  const double Qx = QQQ(qx,qy,qz,0);
+                  const double Qy = QQQ(qx,qy,qz,1);
+                  const double Qz = QQQ(qx,qy,qz,2);
+
+                  const double D11 = D(qx,qy,qz,0,e);
+                  const double D12 = D(qx,qy,qz,1,e);
+                  const double D13 = D(qx,qy,qz,2,e);
+                  const double D21 = symmetric ? D12 : D(qx,qy,qz,3,e);
+                  const double D22 = symmetric ? D(qx,qy,qz,3,e) : D(qx,qy,qz,4,e);
+                  const double D23 = symmetric ? D(qx,qy,qz,4,e) : D(qx,qy,qz,5,e);
+                  const double D31 = symmetric ? D13 : D(qx,qy,qz,6,e);
+                  const double D32 = symmetric ? D23 : D(qx,qy,qz,7,e);
+                  const double D33 = symmetric ? D(qx,qy,qz,5,e) : D(qx,qy,qz,8,e);
+
+                  QQQ(qx,qy,qz,0) = D11*Qx + D12*Qy + D13*Qz;
+                  QQQ(qx,qy,qz,1) = D21*Qx + D22*Qy + D23*Qz;
+                  QQQ(qx,qy,qz,2) = D31*Qx + D32*Qy + D33*Qz;
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      // Apply Bt operator
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(qy,y,Q1D)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[Q1D];
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  MFEM_UNROLL(MQ1)
+                  for (int qz = 0; qz < Q1D; ++qz)
+                  {
+                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
+                  }
+               }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(vd,z,VDIM)
+      {
+         const int nx = (vd == 0) ? D1D : D1D-1;
+         const int ny = (vd == 1) ? D1D : D1D-1;
+         const int nz = (vd == 2) ? D1D : D1D-1;
+         DeviceTensor<5> Yxyz(y, nx, ny, nz, VDIM, NE);
+         DeviceMatrix Btz = (vd == 2) ? Bc : Bo;
+         MFEM_FOREACH_THREAD(dy,y,ny)
+         {
+            MFEM_FOREACH_THREAD(dx,x,nx)
+            {
+               double u[D1D];
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  MFEM_UNROLL(MD1)
+                  for (int dz = 0; dz < nz; ++dz)
+                  {
+                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
+                  }
+               }
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+   });
+}
+
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<double> &Bo,
+                     const Array<double> &Bc,
+                     const Array<double> &Bot,
+                     const Array<double> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y)
+{
+   const int id = (D1D << 4) | Q1D;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+         default: // fallback
+            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
+      }
+   }
+}
+
 // PA H(div) div-div assemble 2D kernel
 // NOTE: this is identical to PACurlCurlSetup3D
 static void PADivDivSetup2D(const int Q1D,
@@ -626,7 +1215,7 @@ static void PADivDivApply2D(const int D1D,
   {
      double div[MAX_Q1D][MAX_Q1D];

-      // div[qy][qx] will be computed as du_x/dx + duy_/dy
+      // div[qy][qx] will be computed as du_x/dx + du_y/dy

      for (int qy = 0; qy < Q1D; ++qy)
      {
@@ -925,19 +1514,8 @@ void DivDivIntegrator::AssemblePA(const FiniteElementSpace &fes)

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
   {
@@ -1195,18 +1773,14 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,

   pa_data.SetSize(nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+
+   if (test_el->GetMapType() == FiniteElement::INTEGRAL)
   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
+      const GeometricFactors *geom =
+         mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS);
+      coeff /= geom->detJ;
   }

   if (trial_el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
@@ -31,7 +31,16 @@ void MassIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: MassIntegrator::AssembleMF only implemented with"
@@ -12,7 +12,9 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/mass/mass.hpp"
+#include "bilininteg_mass_pa.hpp"

 using namespace std;

@@ -38,7 +40,16 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   int map_type = el.GetMapType();
@@ -51,43 +62,10 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   dofs1D = maps->ndof;
   quad1D = maps->nqpt;
   pa_data.SetSize(ne*nq, mt);
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == nq * ne,
-                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto C = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            C(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+
   if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
   if (dim==2)
   {
@@ -581,85 +559,18 @@ static void PAMassApply2D(const int NE,
                          const int d1d = 0,
                          const int q1d = 0)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b_.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto X = Reshape(x_.Read(), D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
   MFEM_FORALL(e, NE,
   {
-      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            sol_x[qy] = 0.0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = X(dx,dy,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx)* s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double d2q = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += d2q * sol_x[qx];
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] *= D(qx,qy,e);
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0.0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xy[qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double q2d = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx,dy,e) += q2d * sol_x[dx];
-            }
-         }
-      }
+      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
   });
 }

@@ -681,108 +592,13 @@ static void SmemPAMassApply2D(const int NE,
   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
   MFEM_VERIFY(D1D <= MD1, "");
   MFEM_VERIFY(Q1D <= MQ1, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
+   const auto b = b_.Read();
+   const auto D = d_.Read();
+   const auto x = x_.Read();
+   auto Y = y_.ReadWrite();
   MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double BBt[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) BBt;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            X[dy][dx] = x(dx,dy,e);
-         }
-      }
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               B[q][dy] = b(q,dy);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double dq = 0.0;
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               dq += X[dy][dx] * B[qx][dx];
-            }
-            DQ[dy][qx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double qq = 0.0;
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               qq += DQ[dy][qx] * B[qy][dy];
-            }
-            QQ[qy][qx] = qq * D(qx, qy, e);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bt[dy][q] = b(q,dy);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dq = 0.0;
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               dq += QQ[qy][qx] * Bt[dx][qx];
-            }
-            QD[qy][dx] = dq;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double dd = 0.0;
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               dd += (QD[qy][dx] * Bt[dy][qy]);
-            }
-            Y(dx, dy, e) += dd;
-         }
-      }
+      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
   });
 }

@@ -796,134 +612,18 @@ static void PAMassApply3D(const int NE,
                          const int d1d = 0,
                          const int q1d = 0)
 {
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
-   auto B = Reshape(b_.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
+
+   const auto B = b_.Read();
+   const auto Bt = bt_.Read();
+   const auto D = d_.Read();
+   const auto X = x_.Read();
+   auto Y = y_.ReadWrite();
+
   MFEM_FORALL(e, NE,
   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
-      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] = 0.0;
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         double sol_xy[max_Q1D][max_Q1D];
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            double sol_x[max_Q1D];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] = 0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const double s = X(dx,dy,dz,e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_x[qx] += B(qx,dx) * s;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const double wy = B(qy,dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xy[qy][qx] += wy * sol_x[qx];
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            const double wz = B(qz,dz);
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-               }
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         double sol_xy[max_D1D][max_D1D];
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] = 0;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            double sol_x[max_D1D];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] = 0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const double s = sol_xyz[qz][qy][qx];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_x[dx] += Bt(dx,qx) * s;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const double wy = Bt(dy,qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  sol_xy[dy][dx] += wy * sol_x[dx];
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            const double wz = Bt(dz,qz);
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-               }
-            }
-         }
-      }
+      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
   });
 }

@@ -944,213 +644,13 @@ static void SmemPAMassApply3D(const int NE,
   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
   MFEM_VERIFY(D1D <= M1D, "");
   MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = Reshape(b_.Read(), Q1D, D1D);
-   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
+   auto b = b_.Read();
+   auto d = d_.Read();
+   auto x = x_.Read();
+   auto y = y_.ReadWrite();
   MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-      MFEM_SHARED double sDQ[MQ1*MD1];
-      double (*B)[MD1] = (double (*)[MD1]) sDQ;
-      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               X[dz][dy][dx] = x(dx,dy,dz,e);
-            }
-         }
-         MFEM_FOREACH_THREAD(dx,x,Q1D)
-         {
-            B[dx][dy] = b(dx,dy);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += X[dz][dy][dx] * B[qx][dx];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               DDQ[dz][dy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; dz++)
-               {
-                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               DQQ[dz][qy][qx] = u[dz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(qx,x,Q1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; qz++)
-               {
-                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[d][q] = b(q,d);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(qy,y,Q1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QQD[qz][qy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[Q1D];
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-               }
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               QDD[qz][dy][dx] = u[qz];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(dx,x,D1D)
-         {
-            double u[D1D];
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] = 0;
-            }
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < D1D; ++dz)
-               {
-                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-               }
-            }
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               y(dx,dy,dz,e) += u[dz];
-            }
-         }
-      }
+      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
   });
 }

@@ -0,0 +1,632 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_BILININTEG_MASS_PA_HPP
+#define MFEM_BILININTEG_MASS_PA_HPP
+
+#include "../config/config.hpp"
+#include "../general/forall.hpp"
+#include "../linalg/dtensor.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+template <bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void PAMassApply2D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d = 0,
+                           const int q1d = 0)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
+   auto X = ConstDeviceCube(x_, D1D, D1D, NE);
+   auto Y = DeviceCube(y_, D1D, D1D, NE);
+
+   if (!ACCUMULATE)
+   {
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y(dx, dy, e) = 0.0;
+         }
+      }
+   }
+
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
+   double sol_xy[max_Q1D][max_Q1D];
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         sol_xy[qy][qx] = 0.0;
+      }
+   }
+   for (int dy = 0; dy < D1D; ++dy)
+   {
+      double sol_x[max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         sol_x[qy] = 0.0;
+      }
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         const double s = X(dx,dy,e);
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_x[qx] += B(qx,dx)* s;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         const double d2q = B(qy,dy);
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] += d2q * sol_x[qx];
+         }
+      }
+   }
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         sol_xy[qy][qx] *= D(qx,qy,e);
+      }
+   }
+   for (int qy = 0; qy < Q1D; ++qy)
+   {
+      double sol_x[max_D1D];
+      for (int dx = 0; dx < D1D; ++dx)
+      {
+         sol_x[dx] = 0.0;
+      }
+      for (int qx = 0; qx < Q1D; ++qx)
+      {
+         const double s = sol_xy[qy][qx];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] += Bt(dx,qx) * s;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         const double q2d = Bt(dy,qy);
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            Y(dx,dy,e) += q2d * sol_x[dx];
+         }
+      }
+   }
+}
+
+template<int T_D1D, int T_Q1D, int T_NBZ, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply2D_Element(const int e,
+                               const int NE,
+                               const double *b_,
+                               const double *d_,
+                               const double *x_,
+                               double *y_,
+                               int d1d = 0,
+                               int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
+   auto x = ConstDeviceCube(x_, D1D, D1D, NE);
+   auto Y = DeviceCube(y_, D1D, D1D, NE);
+
+   const int tidz = MFEM_THREAD_ID(z);
+
+   MFEM_SHARED double BBt[MQ1*MD1];
+   double (*B)[MD1] = (double (*)[MD1]) BBt;
+   double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
+   MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
+   MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
+   double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
+   double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
+   double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
+   double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
+
+
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         X[dy][dx] = x(dx,dy,e);
+      }
+   }
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            B[q][dy] = b(q,dy);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double dq = 0.0;
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            dq += X[dy][dx] * B[qx][dx];
+         }
+         DQ[dy][qx] = dq;
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double qq = 0.0;
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            qq += DQ[dy][qx] * B[qy][dy];
+         }
+         QQ[qy][qx] = qq * D(qx, qy, e);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   if (tidz == 0)
+   {
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[dy][q] = b(q,dy);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double dq = 0.0;
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            dq += QQ[qy][qx] * Bt[dx][qx];
+         }
+         QD[qy][dx] = dq;
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double dd = 0.0;
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            dd += (QD[qy][dx] * Bt[dy][qy]);
+         }
+         if (ACCUMULATE)
+         {
+            Y(dx, dy, e) += dd;
+         }
+         else
+         {
+            Y(dx, dy, e) = dd;
+         }
+      }
+   }
+}
+
+template <bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void PAMassApply3D_Element(const int e,
+                           const int NE,
+                           const double *b_,
+                           const double *bt_,
+                           const double *d_,
+                           const double *x_,
+                           double *y_,
+                           const int d1d,
+                           const int q1d)
+{
+   const int D1D = d1d;
+   const int Q1D = q1d;
+   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
+   auto D = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
+   auto X = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
+   auto Y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
+
+   if (!ACCUMULATE)
+   {
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx, dy, dz, e) = 0.0;
+            }
+         }
+      }
+   }
+
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
+   double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xyz[qz][qy][qx] = 0.0;
+         }
+      }
+   }
+   for (int dz = 0; dz < D1D; ++dz)
+   {
+      double sol_xy[max_Q1D][max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double sol_x[max_Q1D];
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_x[qx] = 0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s = X(dx,dy,dz,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] += B(qx,dx) * s;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double wy = B(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] += wy * sol_x[qx];
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         const double wz = B(qz,dz);
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+            }
+         }
+      }
+   }
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
+         }
+      }
+   }
+   for (int qz = 0; qz < Q1D; ++qz)
+   {
+      double sol_xy[max_D1D][max_D1D];
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_xy[dy][dx] = 0;
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double sol_x[max_D1D];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] = 0;
+         }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double s = sol_xyz[qz][qy][qx];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] += Bt(dx,qx) * s;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            const double wy = Bt(dy,qy);
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_xy[dy][dx] += wy * sol_x[dx];
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         const double wz = Bt(dz,qz);
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
+            }
+         }
+      }
+   }
+}
+
+template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
+MFEM_HOST_DEVICE inline
+void SmemPAMassApply3D_Element(const int e,
+                               const int NE,
+                               const double *b_,
+                               const double *d_,
+                               const double *x_,
+                               double *y_,
+                               const int d1d = 0,
+                               const int q1d = 0)
+{
+   constexpr int D1D = T_D1D ? T_D1D : d1d;
+   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+
+   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
+   auto d = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
+   auto x = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
+   auto y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
+
+   MFEM_SHARED double sDQ[MQ1*MD1];
+   double (*B)[MD1] = (double (*)[MD1]) sDQ;
+   double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
+   MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
+   MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
+   double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
+   double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
+   double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
+   double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
+   double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
+   double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            X[dz][dy][dx] = x(dx,dy,dz,e);
+         }
+      }
+      MFEM_FOREACH_THREAD(dx,x,Q1D)
+      {
+         B[dx][dy] = b(dx,dy);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] += X[dz][dy][dx] * B[qx][dx];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            DDQ[dz][dy][qx] = u[dz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; dz++)
+         {
+            DQQ[dz][qy][qx] = u[dz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,Q1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; qz++)
+         {
+            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(di,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(q,x,Q1D)
+      {
+         Bt[di][q] = b(q,di);
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(qy,y,Q1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            QQD[qz][qy][dx] = u[qz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[Q1D];
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            u[qz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+            }
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            QDD[qz][dy][dx] = u[qz];
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+   MFEM_FOREACH_THREAD(dy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(dx,x,D1D)
+      {
+         double u[D1D];
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            u[dz] = 0;
+         }
+         MFEM_UNROLL(MQ1)
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+            }
+         }
+         MFEM_UNROLL(MD1)
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            if (ACCUMULATE)
+            {
+               y(dx,dy,dz,e) += u[dz];
+            }
+            else
+            {
+               y(dx,dy,dz,e) = u[dz];
+            }
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif
@@ -12,6 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"

 using namespace std;
@@ -149,7 +150,16 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   const int dims = el.GetDim();
@@ -166,43 +176,9 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)

   MFEM_VERIFY(!VQ && !MQ,
               "Only scalar coefficient supported for partial assembly for VectorDiffusionIntegrator");
-   Vector coeff;
-   if (Q == nullptr)
-   {
-      coeff.SetSize(1);
-      coeff(0) = 1.0;
-   }
-   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
-   {
-      coeff.SetSize(1);
-      coeff(0) = cQ->constant;
-   }
-   else if (QuadratureFunctionCoefficient* qfQ =
-               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
-   {
-      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
-      MFEM_VERIFY(qFun.Size() == ne*nq,
-                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
-                  "IntegrationRule used within integrator and in"
-                  " QuadratureFunction appear to be different");
-      qFun.Read();
-      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
-   }
-   else
-   {
-      coeff.SetSize(nq * ne);
-      auto Co = Reshape(coeff.HostWrite(), nq, ne);
-      for (int e = 0; e < ne; ++e)
-      {
-         ElementTransformation& T = *fes.GetElementTransformation(e);
-         for (int q = 0; q < nq; ++q)
-         {
-            Co(q,e) = Q->Eval(T, ir->IntPoint(q));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);

   const Array<double> &w = ir->GetWeights();
   const Vector &j = geom->J;
@@ -30,7 +30,19 @@ void VectorDiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      MFEM_VERIFY(!VQ && !MQ,
+                  "Only scalar coefficient supported for DiffusionIntegrator"
+                  " with libCEED");
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFDiffusionIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: VectorDiffusionIntegrator::AssembleMF only implemented"
@@ -34,7 +34,16 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   dim = mesh->Dimension();
@@ -34,7 +34,16 @@ void VectorMassIntegrator::AssembleMF(const FiniteElementSpace &fes)
   if (DeviceCanUseCeed())
   {
      delete ceedOp;
-      ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+                         fes.IsVariableOrder();
+      if (mixed)
+      {
+         ceedOp = new ceed::MixedMFMassIntegrator(*this, fes, Q);
+      }
+      else
+      {
+         ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
+      }
      return;
   }
   MFEM_ABORT("Error: VectorMassIntegrator::AssembleMF only implemented with"
@@ -11,6 +11,8 @@

 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
+#include "qspace.hpp"
+#include "gridfunc.hpp"

 namespace mfem
 {
@@ -89,6 +91,7 @@ void SmemPAHcurlMassApply3D(const int D1D,
                            Vector &y);

 void PAHdivSetup2D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
@@ -96,6 +99,7 @@ void PAHdivSetup2D(const int Q1D,
                   Vector &op);

 void PAHdivSetup3D(const int Q1D,
+                   const int coeffDim,
                   const int NE,
                   const Array<double> &w,
                   const Vector &j,
@@ -149,6 +153,7 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
@@ -157,32 +162,24 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 void PAHdivMassAssembleDiagonal3D(const int D1D,
                                  const int Q1D,
                                  const int NE,
+                                  const bool symmetric,
                                  const Array<double> &Bo_,
                                  const Array<double> &Bc_,
                                  const Vector &op_,
                                  Vector &diag_);

-void PAHdivMassApply2D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const Array<double> &Bo_,
-                       const Array<double> &Bc_,
-                       const Array<double> &Bot_,
-                       const Array<double> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
-
-void PAHdivMassApply3D(const int D1D,
-                       const int Q1D,
-                       const int NE,
-                       const Array<double> &Bo_,
-                       const Array<double> &Bc_,
-                       const Array<double> &Bot_,
-                       const Array<double> &Bct_,
-                       const Vector &op_,
-                       const Vector &x_,
-                       Vector &y_);
+void PAHdivMassApply(const int dim,
+                     const int D1D,
+                     const int Q1D,
+                     const int NE,
+                     const bool symmetric,
+                     const Array<double> &Bo,
+                     const Array<double> &Bc,
+                     const Array<double> &Bot,
+                     const Array<double> &Bct,
+                     const Vector &op,
+                     const Vector &x,
+                     Vector &y);

 void PAHcurlL2Setup(const int NQ,
                    const int coeffDim,
@@ -797,129 +794,63 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
   trial_fetype = trial_el->GetDerivType();
   test_fetype = test_el->GetDerivType();

-   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
-
-   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
-   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
-   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
-   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
-
-   symmetric = (SMQ || MQ == NULL);
-
   const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
   const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
   const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
   const bool test_div = (test_fetype == mfem::FiniteElement::DIV);

+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
+   if (Q) { coeff.Project(*Q); }
+   else if (MQ) { coeff.ProjectTranspose(*MQ); }
+   else if (DQ) { coeff.Project(*DQ); }
+   else { coeff.SetConstant(1.0); }
+
+   const int coeff_dim = coeff.GetVDim();
+   symmetric = (coeff_dim != dim*dim);
+
   if ((trial_curl && test_div) || (trial_div && test_curl))
-      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
+      pa_data.SetSize((coeff_dim == 1 ? 1 : dim*dim) * nq * ne,
                      Device::GetMemoryType());
   else
-      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
+      pa_data.SetSize((symmetric ? symmDims : dims*dims) * nq * ne,
                      Device::GetMemoryType());

-   Vector coeff(coeffDim * ne * nq);
-   coeff = 1.0;
-   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
-   if (Q || DQ || MQ)
-   {
-      Vector DM(DQ ? coeffDim : 0);
-      DenseMatrix M;
-      DenseSymmetricMatrix SM;
-
-      if (DQ)
-      {
-         MFEM_VERIFY(coeffDim == dim, "");
-      }
-      if (SMQ)
-      {
-         MFEM_VERIFY(SMQ->GetSize() == dim, "");
-         SM.SetSize(dim);
-      }
-      else if (MQ)
-      {
-         MFEM_VERIFY(coeffDim == MQdim, "");
-         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
-         M.SetSize(dim);
-      }
-
-
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            if (SMQ)
-            {
-               SMQ->Eval(SM, *tr, ir->IntPoint(p));
-               int cnt = 0;
-               for (int i=0; i<dim; ++i)
-                  for (int j=i; j<dim; ++j, ++cnt)
-                  {
-                     coeffh(cnt, p, e) = SM(i,j);
-                  }
-            }
-            else if (MQ)
-            {
-               MQ->Eval(M, *tr, ir->IntPoint(p));
-
-               for (int i=0; i<dim; ++i)
-                  for (int j=0; j<dim; ++j)
-                  {
-                     coeffh(j+(i*dim), p, e) = M(i,j);
-                  }
-            }
-            else if (DQ)
-            {
-               DQ->Eval(DM, *tr, ir->IntPoint(p));
-               for (int i=0; i<coeffDim; ++i)
-               {
-                  coeffh(i, p, e) = DM[i];
-               }
-            }
-            else
-            {
-               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
-            }
-         }
-      }
-   }
-
   if (trial_curl && test_curl && dim == 3)
   {
-      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                         coeff, pa_data);
   }
   else if (trial_curl && test_curl && dim == 2)
   {
-      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup2D<2>(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                            coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 3)
   {
-      PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (trial_div && test_div && dim == 2)
   {
-      PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup2D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
                    coeff, pa_data);
   }
   else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
            test_fel->GetOrder() == trial_fel->GetOrder())
   {
-      if (coeffDim == 1)
+      if (coeff_dim == 1)
      {
-         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
+         PAHcurlL2Setup(nq, coeff_dim, ne, ir->GetWeights(), coeff, pa_data);
      }
      else
      {
         const bool tr = (trial_div && test_curl);
         if (dim == 3)
-            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup3D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
                               geom->J, coeff, pa_data);
         else
-            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup2D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
                               geom->J, coeff, pa_data);
      }
   }
@@ -963,7 +894,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
      else if (trial_fetype == mfem::FiniteElement::DIV &&
               test_fetype == trial_fetype)
      {
-         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne,
+         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
                                      mapsO->B, mapsC->B, pa_data, diag);
      }
      else
@@ -971,7 +902,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
         MFEM_ABORT("Unknown kernel.");
      }
   }
-   else
+   else // 2D
   {
      if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
      {
@@ -981,7 +912,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
      else if (trial_fetype == mfem::FiniteElement::DIV &&
               test_fetype == trial_fetype)
      {
-         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne,
+         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne, symmetric,
                                      mapsO->B, mapsC->B, pa_data, diag);
      }
      else
@@ -1034,8 +965,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_div && test_div)
      {
-         PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                           mapsC->Bt, pa_data, x, y);
+         PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
+                         mapsC->Bt, pa_data, x, y);
      }
      else if (trial_curl && test_div)
      {
@@ -1056,7 +987,7 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
         MFEM_ABORT("Unknown kernel.");
      }
   }
-   else
+   else // 2D
   {
      if (trial_curl && test_curl)
      {
@@ -1065,8 +996,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
      }
      else if (trial_div && test_div)
      {
-         PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
-                           mapsC->Bt, pa_data, x, y);
+         PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
+                         mapsC->Bt, pa_data, x, y);
      }
      else if ((trial_curl && test_div) || (trial_div && test_curl))
      {
@@ -1111,6 +1042,11 @@ void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,

   if (symmetricSpaces)
   {
+      if (MQ && dynamic_cast<SymmetricMatrixCoefficient*>(MQ) == NULL)
+      {
+         MFEM_ABORT("VectorFEMassIntegrator transpose not implemented for asymmetric MatrixCoefficient");
+      }
+
      this->AddMultPA(x, y);
   }
 }
@@ -1156,19 +1092,8 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace

   pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());

-   Vector coeff(ne * nq);
-   coeff = 1.0;
-   if (Q)
-   {
-      for (int e=0; e<ne; ++e)
-      {
-         ElementTransformation *tr = mesh->GetElementTransformation(e);
-         for (int p=0; p<nq; ++p)
-         {
-            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
-         }
-      }
-   }
+   QuadratureSpace qs(*mesh, *ir);
+   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);

   // Use the same setup functions as VectorFEMassIntegrator.
   if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)
@@ -62,6 +62,20 @@ PAConvectionIntegrator::PAConvectionIntegrator(
 #endif
 }

+MixedPAConvectionIntegrator::MixedPAConvectionIntegrator(
+   const ConvectionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::VectorCoefficient *Q,
+   const double alpha)
+{
+#ifdef MFEM_USE_CEED
+   ConvectionOperatorInfo info(fes.GetMesh()->Dimension(), alpha);
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFConvectionIntegrator::MFConvectionIntegrator(
   const mfem::FiniteElementSpace &fes,
   const mfem::IntegrationRule &irm,
@@ -77,6 +91,20 @@ MFConvectionIntegrator::MFConvectionIntegrator(
 #endif
 }

+MixedMFConvectionIntegrator::MixedMFConvectionIntegrator(
+   const ConvectionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::VectorCoefficient *Q,
+   const double alpha)
+{
+#ifdef MFEM_USE_CEED
+   ConvectionOperatorInfo info(fes.GetMesh()->Dimension(), alpha);
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -13,6 +13,7 @@
 #define MFEM_LIBCEED_CONV_HPP

 #include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"

 namespace mfem
@@ -26,21 +27,39 @@ class PAConvectionIntegrator : public PAIntegrator
 {
 public:
   PAConvectionIntegrator(const mfem::FiniteElementSpace &fes,
-                          const mfem::IntegrationRule &irm,
+                          const mfem::IntegrationRule &ir,
                          mfem::VectorCoefficient *Q,
                          const double alpha);
 };

+class MixedPAConvectionIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPAConvectionIntegrator(const ConvectionIntegrator &integ,
+                               const mfem::FiniteElementSpace &fes,
+                               mfem::VectorCoefficient *Q,
+                               const double alpha);
+};
+
 /// Represent a ConvectionIntegrator with AssemblyLevel::None using libCEED.
 class MFConvectionIntegrator : public MFIntegrator
 {
 public:
   MFConvectionIntegrator(const mfem::FiniteElementSpace &fes,
-                          const mfem::IntegrationRule &irm,
+                          const mfem::IntegrationRule &ir,
                          mfem::VectorCoefficient *Q,
                          const double alpha);
 };

+class MixedMFConvectionIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFConvectionIntegrator(const ConvectionIntegrator &integ,
+                               const mfem::FiniteElementSpace &fes,
+                               mfem::VectorCoefficient *Q,
+                               const double alpha);
+};
+
 }

 }
@@ -60,6 +60,32 @@ PADiffusionIntegrator::PADiffusionIntegrator(
 #endif
 }

+MixedPADiffusionIntegrator::MixedPADiffusionIntegrator(
+   const DiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedPADiffusionIntegrator::MixedPADiffusionIntegrator(
+   const VectorDiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFDiffusionIntegrator::MFDiffusionIntegrator(
   const mfem::FiniteElementSpace &fes,
   const mfem::IntegrationRule &irm,
@@ -74,6 +100,32 @@ MFDiffusionIntegrator::MFDiffusionIntegrator(
 #endif
 }

+MixedMFDiffusionIntegrator::MixedMFDiffusionIntegrator(
+   const DiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedMFDiffusionIntegrator::MixedMFDiffusionIntegrator(
+   const VectorDiffusionIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -13,6 +13,7 @@
 #define MFEM_LIBCEED_DIFF_HPP

 #include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"

 namespace mfem
@@ -26,19 +27,43 @@ class PADiffusionIntegrator : public PAIntegrator
 {
 public:
   PADiffusionIntegrator(const mfem::FiniteElementSpace &fes,
-                         const mfem::IntegrationRule &irm,
+                         const mfem::IntegrationRule &ir,
                         mfem::Coefficient *Q);
 };

+class MixedPADiffusionIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPADiffusionIntegrator(const DiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+
+   MixedPADiffusionIntegrator(const VectorDiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+};
+
 /// Represent a DiffusionIntegrator with AssemblyLevel::None using libCEED.
 class MFDiffusionIntegrator : public MFIntegrator
 {
 public:
   MFDiffusionIntegrator(const mfem::FiniteElementSpace &fes,
-                         const mfem::IntegrationRule &irm,
+                         const mfem::IntegrationRule &ir,
                         mfem::Coefficient *Q);
 };

+class MixedMFDiffusionIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFDiffusionIntegrator(const DiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+
+   MixedMFDiffusionIntegrator(const VectorDiffusionIntegrator &integ,
+                              const mfem::FiniteElementSpace &fes,
+                              mfem::Coefficient *Q);
+};
+
 }

 }
@@ -59,6 +59,30 @@ PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
 #endif
 }

+MixedPAMassIntegrator::MixedPAMassIntegrator(const MassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedPAMassIntegrator::MixedPAMassIntegrator(const VectorMassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
                                   const mfem::IntegrationRule &irm,
                                   mfem::Coefficient *Q)
@@ -72,6 +96,30 @@ MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
 #endif
 }

+MixedMFMassIntegrator::MixedMFMassIntegrator(const MassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
+MixedMFMassIntegrator::MixedMFMassIntegrator(const VectorMassIntegrator &integ,
+                                             const mfem::FiniteElementSpace &fes,
+                                             mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   MassOperatorInfo info;
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -13,6 +13,7 @@
 #define MFEM_LIBCEED_MASS_HPP

 #include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"

 namespace mfem
@@ -26,19 +27,43 @@ class PAMassIntegrator : public PAIntegrator
 {
 public:
   PAMassIntegrator(const mfem::FiniteElementSpace &fes,
-                    const mfem::IntegrationRule &irm,
+                    const mfem::IntegrationRule &ir,
                    mfem::Coefficient *Q);
 };

+class MixedPAMassIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPAMassIntegrator(const MassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+
+   MixedPAMassIntegrator(const VectorMassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+};
+
 /// Represent a MassIntegrator with AssemblyLevel::None using libCEED.
 class MFMassIntegrator : public MFIntegrator
 {
 public:
   MFMassIntegrator(const mfem::FiniteElementSpace &fes,
-                    const mfem::IntegrationRule &irm,
+                    const mfem::IntegrationRule &ir,
                    mfem::Coefficient *Q);
 };

+class MixedMFMassIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFMassIntegrator(const MassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+
+   MixedMFMassIntegrator(const VectorMassIntegrator &integ,
+                         const mfem::FiniteElementSpace &fes,
+                         mfem::Coefficient *Q);
+};
+
 }

 }
@@ -60,6 +60,19 @@ PAVectorConvectionNLFIntegrator::PAVectorConvectionNLFIntegrator(
 #endif
 }

+MixedPAVectorConvectionNLIntegrator::MixedPAVectorConvectionNLIntegrator(
+   const VectorConvectionNLFIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   NLConvectionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 MFVectorConvectionNLFIntegrator::MFVectorConvectionNLFIntegrator(
   const mfem::FiniteElementSpace &fes,
   const mfem::IntegrationRule &irm,
@@ -74,6 +87,19 @@ MFVectorConvectionNLFIntegrator::MFVectorConvectionNLFIntegrator(
 #endif
 }

+MixedMFVectorConvectionNLIntegrator::MixedMFVectorConvectionNLIntegrator(
+   const VectorConvectionNLFIntegrator &integ,
+   const mfem::FiniteElementSpace &fes,
+   mfem::Coefficient *Q)
+{
+#ifdef MFEM_USE_CEED
+   NLConvectionOperatorInfo info(fes.GetMesh()->Dimension());
+   Assemble(integ, info, fes, Q);
+#else
+   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
+#endif
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -13,6 +13,7 @@
 #define MFEM_LIBCEED_NLCONV_HPP

 #include "../../interface/integrator.hpp"
+#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"

 namespace mfem
@@ -31,6 +32,15 @@ public:
                                   mfem::Coefficient *coeff);
 };

+class MixedPAVectorConvectionNLIntegrator : public MixedIntegrator<PAIntegrator>
+{
+public:
+   MixedPAVectorConvectionNLIntegrator(
+      const VectorConvectionNLFIntegrator &integ,
+      const mfem::FiniteElementSpace &fes,
+      mfem::Coefficient *Q);
+};
+
 /** Represent a VectorConvectionNLFIntegrator with AssemblyLevel::None
    using libCEED. */
 class MFVectorConvectionNLFIntegrator : public MFIntegrator
@@ -41,6 +51,15 @@ public:
                                   mfem::Coefficient *coeff);
 };

+class MixedMFVectorConvectionNLIntegrator : public MixedIntegrator<MFIntegrator>
+{
+public:
+   MixedMFVectorConvectionNLIntegrator(
+      const VectorConvectionNLFIntegrator &integ,
+      const mfem::FiniteElementSpace &fes,
+      mfem::Coefficient *Q);
+};
+
 }

 }
@@ -327,13 +327,13 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
            const CeedScalar A33 = J11 * J22 - J12 * J21;
            const CeedScalar w = qw[i] * coeff;
            const CeedScalar qd00 = w * A11;
-            const CeedScalar qd01 = w * A21;
-            const CeedScalar qd02 = w * A31;
-            const CeedScalar qd10 = w * A12;
+            const CeedScalar qd10 = w * A21;
+            const CeedScalar qd20 = w * A31;
+            const CeedScalar qd01 = w * A12;
            const CeedScalar qd11 = w * A22;
-            const CeedScalar qd12 = w * A32;
-            const CeedScalar qd20 = w * A13;
-            const CeedScalar qd21 = w * A23;
+            const CeedScalar qd21 = w * A32;
+            const CeedScalar qd02 = w * A13;
+            const CeedScalar qd12 = w * A23;
            const CeedScalar qd22 = w * A33;
            const CeedScalar u0   = u[i + Q * 0];
            const CeedScalar u1   = u[i + Q * 1];
@@ -440,13 +440,13 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
            const CeedScalar A33 = J11 * J22 - J12 * J21;
            const CeedScalar w = qw[i] * c[i];
            const CeedScalar qd00 = w * A11;
-            const CeedScalar qd01 = w * A21;
-            const CeedScalar qd02 = w * A31;
-            const CeedScalar qd10 = w * A12;
+            const CeedScalar qd10 = w * A21;
+            const CeedScalar qd20 = w * A31;
+            const CeedScalar qd01 = w * A12;
            const CeedScalar qd11 = w * A22;
-            const CeedScalar qd12 = w * A32;
-            const CeedScalar qd20 = w * A13;
-            const CeedScalar qd21 = w * A23;
+            const CeedScalar qd21 = w * A32;
+            const CeedScalar qd02 = w * A13;
+            const CeedScalar qd12 = w * A23;
            const CeedScalar qd22 = w * A33;
            const CeedScalar u0   = u[i + Q * 0];
            const CeedScalar u1   = u[i + Q * 1];
@@ -36,6 +36,8 @@ static CeedElemTopology GetCeedTopology(Geometry::Type geom)
         return CEED_TOPOLOGY_HEX;
      case Geometry::PRISM:
         return CEED_TOPOLOGY_PRISM;
+      case Geometry::PYRAMID:
+         return CEED_TOPOLOGY_PYRAMID;
      default:
         MFEM_ABORT("This type of element is not supported");
         return CEED_TOPOLOGY_PRISM; // Silence warning
@@ -43,11 +45,11 @@ static CeedElemTopology GetCeedTopology(Geometry::Type geom)
 }

 static void InitNonTensorBasis(const mfem::FiniteElementSpace &fes,
+                               const mfem::FiniteElement &fe,
                               const mfem::IntegrationRule &ir,
                               Ceed ceed, CeedBasis *basis)
 {
-   const mfem::DofToQuad &maps = fes.GetFE(0)->
-                                 GetDofToQuad(ir,mfem::DofToQuad::FULL);
+   const mfem::DofToQuad &maps = fe.GetDofToQuad(ir, mfem::DofToQuad::FULL);
   mfem::Mesh *mesh = fes.GetMesh();
   const int dim = mesh->Dimension();
   const int ndofs = maps.ndof;
@@ -62,18 +64,18 @@ static void InitNonTensorBasis(const mfem::FiniteElementSpace &fes,
      if (dim>2) { qX(2,i) = ip.z; }
      qW(i) = ip.weight;
   }
-   CeedBasisCreateH1(ceed, GetCeedTopology(fes.GetFE(0)->GetGeomType()),
+   CeedBasisCreateH1(ceed, GetCeedTopology(fe.GetGeomType()),
                     fes.GetVDim(), ndofs, nqpts,
                     maps.Bt.GetData(), maps.Gt.GetData(),
                     qX.GetData(), qW.GetData(), basis);
 }

 static void InitTensorBasis(const mfem::FiniteElementSpace &fes,
+                            const mfem::FiniteElement &fe,
                            const mfem::IntegrationRule &ir,
                            Ceed ceed, CeedBasis *basis)
 {
-   const mfem::DofToQuad &maps =
-      fes.GetFE(0)->GetDofToQuad(ir, mfem::DofToQuad::TENSOR);
+   const mfem::DofToQuad &maps = fe.GetDofToQuad(ir, mfem::DofToQuad::TENSOR);
   mfem::Mesh *mesh = fes.GetMesh();
   const int ndofs = maps.ndof;
   const int nqpts = maps.nqpt;
@@ -96,28 +98,30 @@ static void InitTensorBasis(const mfem::FiniteElementSpace &fes,
                           qW.GetData(), basis);
 }

-void InitBasis(const FiniteElementSpace &fes,
-               const IntegrationRule &irm,
-               Ceed ceed, CeedBasis *basis)
+static void InitBasisImpl(const FiniteElementSpace &fes,
+                          const FiniteElement &fe,
+                          const IntegrationRule &ir,
+                          Ceed ceed, CeedBasis *basis)
 {
   // Check for FES -> basis, restriction in hash tables
-   const mfem::FiniteElement *fe = fes.GetFE(0);
-   const int P = fe->GetDof();
-   const int Q = irm.GetNPoints();
+   const int P = fe.GetDof();
+   const int Q = ir.GetNPoints();
   const int ncomp = fes.GetVDim();
-   BasisKey basis_key(&fes, &irm, ncomp, P, Q);
+   BasisKey basis_key(&fes, &ir, ncomp, P, Q);
   auto basis_itr = mfem::internal::ceed_basis_map.find(basis_key);
+   const bool tensor = dynamic_cast<const mfem::TensorBasisElement *>
+                       (&fe) != nullptr;

-   // Init or retreive key values
+   // Init or retrieve key values
   if (basis_itr == mfem::internal::ceed_basis_map.end())
   {
-      if (UsesTensorBasis(fes))
+      if ( tensor )
      {
-         InitTensorBasis(fes, irm, ceed, basis);
+         InitTensorBasis(fes, fe, ir, ceed, basis);
      }
      else
      {
-         InitNonTensorBasis(fes, irm, ceed, basis);
+         InitNonTensorBasis(fes, fe, ir, ceed, basis);
      }
      mfem::internal::ceed_basis_map[basis_key] = *basis;
   }
@@ -127,6 +131,24 @@ void InitBasis(const FiniteElementSpace &fes,
   }
 }

+void InitBasis(const FiniteElementSpace &fes,
+               const IntegrationRule &ir,
+               Ceed ceed, CeedBasis *basis)
+{
+   const mfem::FiniteElement &fe = *fes.GetFE(0);
+   InitBasisImpl(fes, fe, ir, ceed, basis);
+}
+
+void InitBasisWithIndices(const FiniteElementSpace &fes,
+                          const IntegrationRule &ir,
+                          int nelem,
+                          const int* indices,
+                          Ceed ceed, CeedBasis *basis)
+{
+   const mfem::FiniteElement &fe = *fes.GetFE(indices[0]);
+   InitBasisImpl(fes, fe, ir, ceed, basis);
+}
+
 #endif

 } // namespace ceed
@@ -22,17 +22,32 @@ namespace ceed

 #ifdef MFEM_USE_CEED

-/** @brief Initialize a CeedBasis.
+/** @brief Initialize a CeedBasis for non-mixed meshes.

   @param[in] fes Input finite element space.
-   @param[in] irm Input integration rule.
+   @param[in] ir Input integration rule.
   @param[in] ceed Input Ceed object.
   @param[out] basis The address of the initialized CeedBasis object.
 */
 void InitBasis(const FiniteElementSpace &fes,
-               const IntegrationRule &irm,
+               const IntegrationRule &ir,
               Ceed ceed, CeedBasis *basis);

+/** @brief Initialize a CeedBasis for mixed meshes.
+
+    @param[in] fes The finite element space.
+    @param[in] ir is the integration rule for the operator.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[in] ceed The Ceed object.
+    @param[out] basis The `CeedBasis` to initialize. */
+void InitBasisWithIndices(const FiniteElementSpace &fes,
+                          const IntegrationRule &ir,
+                          int nelem,
+                          const int* indices,
+                          Ceed ceed, CeedBasis *basis);
+
 #endif

 } // namespace ceed
@@ -14,11 +14,13 @@

 #ifdef MFEM_USE_CEED

+#include "../../../general/forall.hpp"
 #include "../../../config/config.hpp"
 #include "../../../linalg/vector.hpp"
 #include "../../../linalg/dtensor.hpp"
 #include "../../../mesh/mesh.hpp"
 #include "../../gridfunc.hpp"
+#include "../../qfunction.hpp"
 #include "util.hpp"
 #include "ceed.hpp"

@@ -77,7 +79,14 @@ struct QuadCoefficient : VariableCoefficient

 /** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
    mfem::Coefficient @a Q, an mfem::Mesh @a mesh, and an mfem::IntegrationRule
-    @a ir. */
+    @a ir.
+
+    @param[in] Q is the coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
 template <typename Context>
 void InitCoefficient(mfem::Coefficient *Q, mfem::Mesh &mesh,
                     const mfem::IntegrationRule &ir,
@@ -113,7 +122,7 @@ void InitCoefficient(mfem::Coefficient *Q, mfem::Mesh &mesh,
      MFEM_VERIFY(qFun.Size() == nq * ne,
                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
                  "IntegrationRule used within integrator and in"
                  " QuadratureFunction appear to be different");
      qFun.Read();
@@ -143,8 +152,15 @@ void InitCoefficient(mfem::Coefficient *Q, mfem::Mesh &mesh,


 /** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
-    mfem::VectorCoefficient @a Q, an mfem::Mesh @a mesh, and an
-    mfem::IntegrationRule @a ir. */
+    mfem::VectorCoefficient @a VQ, an mfem::Mesh @a mesh, and an
+    mfem::IntegrationRule @a ir.
+
+    @param[in] VQ is the vector coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
 template <typename Context>
 void InitCoefficient(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
                     const mfem::IntegrationRule &ir,
@@ -180,7 +196,7 @@ void InitCoefficient(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
                  "Incompatible QuadratureFunction dimension \n");

-      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetElementIntRule(0),
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
                  "IntegrationRule used within integrator and in"
                  " QuadratureFunction appear to be different");
      qFun.Read();
@@ -214,6 +230,209 @@ void InitCoefficient(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
   }
 }

+/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
+    mfem::Coefficient @a Q, an mfem::Mesh @a mesh, and an mfem::IntegrationRule
+    @a ir for the elements given by the indices @a indices.
+
+    @param[in] Q is the coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
+template <typename Context>
+void InitCoefficientWithIndices(mfem::Coefficient *Q, mfem::Mesh &mesh,
+                                const mfem::IntegrationRule &ir,
+                                int nelem,
+                                const int* indices,
+                                Coefficient*& coeff_ptr, Context &ctx)
+{
+   if ( Q == nullptr )
+   {
+      Coefficient *ceedCoeff = new Coefficient(1);
+      ctx.coeff = 1.0;
+      coeff_ptr = ceedCoeff;
+   }
+   else if (ConstantCoefficient *const_coeff =
+               dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      Coefficient *ceedCoeff = new Coefficient(1);
+      ctx.coeff = const_coeff->constant;
+      coeff_ptr = ceedCoeff;
+   }
+   else if (GridFunctionCoefficient* gf_coeff =
+               dynamic_cast<GridFunctionCoefficient*>(Q))
+   {
+      GridCoefficient *ceedCoeff =
+         new GridCoefficient(*gf_coeff->GetGridFunction());
+      coeff_ptr = ceedCoeff;
+   }
+   else if (QuadratureFunctionCoefficient *cQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      ceedCoeff->coeff.SetSize(nq * nelem);
+      Memory<int> m_indices((int*)indices, nelem, false);
+      auto in = Reshape(qFun.Read(), nq, ne);
+      auto d_indices = Read(m_indices, nelem);
+      auto out = Reshape(ceedCoeff->coeff.Write(), nq, nelem);
+      MFEM_FORALL(i, nelem * nq,
+      {
+         const int q = i%nq;
+         const int sub_e = i/nq;
+         const int e = d_indices[sub_e];
+         out(q, sub_e) = in(q, e);
+      });
+      m_indices.DeleteDevice();
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+   else
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(1);
+      const int nq = ir.GetNPoints();
+      ceedCoeff->coeff.SetSize(nq * nelem);
+      auto C = Reshape(ceedCoeff->coeff.HostWrite(), nq, nelem);
+      for (int i = 0; i < nelem; ++i)
+      {
+         const int e = indices[i];
+         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q, i) = Q->Eval(T, ir.IntPoint(q));
+         }
+      }
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+}
+
+
+/** @brief Initializes an mfem::ceed::Coefficient @a coeff_ptr from an
+    mfem::VectorCoefficient @a Q, an mfem::Mesh @a mesh, and an
+    mfem::IntegrationRule @a ir for the elements given by the indices @a indices.
+
+    @param[in] VQ is the vector coefficient from the `Integrator`.
+    @param[in] mesh is the mesh.
+    @param[in] ir is the integration rule.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[out] coeff_ptr is the structure to store the coefficient for the
+                          `CeedOperator`.
+    @param[out] ctx is the Context associated to the QFunction. */
+template <typename Context>
+void InitCoefficientWithIndices(mfem::VectorCoefficient *VQ, mfem::Mesh &mesh,
+                                const mfem::IntegrationRule &ir,
+                                int nelem,
+                                const int* indices,
+                                Coefficient *&coeff_ptr, Context &ctx)
+{
+   if (VectorConstantCoefficient *const_coeff =
+          dynamic_cast<VectorConstantCoefficient*>(VQ))
+   {
+      const int vdim = const_coeff->GetVDim();
+      const mfem::Vector &val = const_coeff->GetVec();
+      Coefficient *ceedCoeff = new Coefficient(vdim);
+      for (int i = 0; i < vdim; i++)
+      {
+         ctx.coeff[i] = val[i];
+      }
+      coeff_ptr = ceedCoeff;
+   }
+   else if (VectorGridFunctionCoefficient* vgf_coeff =
+               dynamic_cast<VectorGridFunctionCoefficient*>(VQ))
+   {
+      GridCoefficient *ceedCoeff =
+         new GridCoefficient(*vgf_coeff->GetGridFunction());
+      coeff_ptr = ceedCoeff;
+   }
+   else if (VectorQuadratureFunctionCoefficient *cQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(VQ))
+   {
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(cQ->GetVDim());
+      const int dim = mesh.Dimension();
+      const int ne = mesh.GetNE();
+      const int nq = ir.GetNPoints();
+      const mfem::QuadratureFunction &qFun = cQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(&ir == &qFun.GetSpace()->GetIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      ceedCoeff->coeff.SetSize(dim * nq * nelem);
+      Memory<int> m_indices((int*)indices, nelem, false);
+      auto in = Reshape(qFun.Read(), dim, nq, ne);
+      auto d_indices = Read(m_indices, nelem);
+      auto out = Reshape(ceedCoeff->coeff.Write(), dim, nq, nelem);
+      MFEM_FORALL(i, nelem * nq,
+      {
+         const int q = i%nq;
+         const int sub_e = i/nq;
+         const int e = d_indices[sub_e];
+         for (int d = 0; d < dim; d++)
+         {
+            out(d, q, sub_e) = in(d, q, e);
+         }
+      });
+      m_indices.DeleteDevice();
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+   else
+   {
+      const int dim = mesh.Dimension();
+      QuadCoefficient *ceedCoeff = new QuadCoefficient(dim);
+      const int nq = ir.GetNPoints();
+      ceedCoeff->coeff.SetSize(dim * nq * nelem);
+      auto C = Reshape(ceedCoeff->coeff.HostWrite(), dim, nq, nelem);
+      mfem::DenseMatrix Q_ir;
+      for (int i = 0; i < nelem; ++i)
+      {
+         const int e = indices[i];
+         mfem::ElementTransformation &T = *mesh.GetElementTransformation(e);
+         VQ->Eval(Q_ir, T, ir);
+         for (int q = 0; q < nq; ++q)
+         {
+            for (int d = 0; d < dim; ++d)
+            {
+               C(d, q, i) = Q_ir(d, q);
+            }
+         }
+      }
+      InitVector(ceedCoeff->coeff, ceedCoeff->coeffVector);
+      coeff_ptr = ceedCoeff;
+   }
+}
+
+template <typename Coeff, typename Context>
+void InitCoefficient(Coeff *Q, mfem::Mesh &mesh,
+                     const mfem::IntegrationRule &ir, int nelem,
+                     const int* indices, Coefficient *&coeff_ptr, Context &ctx)
+{
+   if (indices)
+   {
+      InitCoefficientWithIndices(Q, mesh, ir, nelem, indices, coeff_ptr, ctx);
+   }
+   else
+   {
+      InitCoefficient(Q, mesh, ir, coeff_ptr, ctx);
+   }
+}
+
 } // namespace ceed

 } // namespace mfem
@@ -18,6 +18,7 @@
 #include "operator.hpp"
 #include "coefficient.hpp"
 #include "restriction.hpp"
+#include "util.hpp"
 #include "ceed.hpp"

 namespace mfem
@@ -86,6 +87,7 @@ protected:
   CeedQFunctionContext build_ctx;
   CeedOperator build_oper;

+public:
   PAIntegrator()
      : Operator(),
        trial_basis(nullptr), test_basis(nullptr), mesh_basis(nullptr),
@@ -95,23 +97,51 @@ protected:
        qdata(nullptr), coeff(nullptr), build_ctx(nullptr), build_oper(nullptr)
   { }

-public:
-   /** This method assembles the PAIntegrator.
+   /** @brief This method assembles the `PAIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.

-       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
-                       the `CeedOperatorInfo` type is expected to inherit from
-                       `OperatorInfo` and contain a `Context` type relevant to
-                       the qFunctions.
-       @param[in] fes the `FiniteElementSpace` for the form,
-       @param[in] ir the `IntegrationRule` for the numerical integration,
-       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] Q is the coefficient from the `Integrator`. */
   template <typename CeedOperatorInfo, typename CoeffType>
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
                 CoeffType *Q)
   {
-      Assemble(info, fes, fes, irm, Q);
+      Assemble(info, fes, ir, fes.GetNE(), nullptr, Q);
+   }
+
+   /** @brief This method assembles the `PAIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q for the elements given by the indices
+       @a indices.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.
+
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] nelem The number of elements.
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed.
+       @param[in] Q is the coefficient from the `Integrator`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
+                 CoeffType *Q)
+   {
+      Assemble(info, fes, fes, ir, nelem, indices, Q);
   }

   /** This method assembles the PAIntegrator for mixed forms.
@@ -128,12 +158,40 @@ public:
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &trial_fes,
                 const mfem::FiniteElementSpace &test_fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
+                 CoeffType *Q)
+   {
+      Assemble(info, trial_fes, test_fes, ir, trial_fes.GetNE(), nullptr, Q);
+   }
+
+   /** This method assembles the PAIntegrator for mixed forms on mixed meshes.
+
+       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
+                       the `CeedOperatorInfo` type is expected to inherit from
+                       `OperatorInfo` and contain a `Context` type relevant to
+                       the qFunctions.
+       @param[in] trial_fes the trial `FiniteElementSpace` for the form,
+       @param[in] test_fes the test `FiniteElementSpace` for the form,
+       @param[in] ir the `IntegrationRule` for the numerical integration,
+       @param[in] nelem The number of elements,
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed,
+       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &trial_fes,
+                 const mfem::FiniteElementSpace &test_fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
                 CoeffType *Q)
   {
      Ceed ceed(internal::ceed);
      mfem::Mesh &mesh = *trial_fes.GetMesh();
-      InitCoefficient(Q, mesh, irm, coeff, info.ctx);
+      MFEM_VERIFY(!(!indices && mesh.GetNumGeometries(mesh.Dimension()) > 1),
+                  "Use ceed::MixedIntegrator on mixed meshes.");
+      InitCoefficient(Q, mesh, ir, nelem, indices, coeff, info.ctx);
      bool const_coeff = coeff->IsConstant();
      std::string build_func = const_coeff ? info.build_func_const
                               : info.build_func_quad;
@@ -145,7 +203,6 @@ public:
                     info.trial_op,
                     info.test_op
                    };
-      CeedInt nqpts, nelem = mesh.GetNE();
      CeedInt dim = mesh.SpaceDimension();
      CeedInt trial_vdim = trial_fes.GetVDim();
      CeedInt test_vdim = test_fes.GetVDim();
@@ -153,23 +210,23 @@ public:
      mesh.EnsureNodes();
      if ( &trial_fes == &test_fes )
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
-                                 &trial_basis, &trial_restr);
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices,
+                                 ceed, &trial_basis, &trial_restr);
         test_basis = trial_basis;
         test_restr = trial_restr;
      }
      else
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
-                                 &trial_basis, &trial_restr);
-         InitBasisAndRestriction(test_fes, irm, ceed,
-                                 &test_basis, &test_restr);
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices,
+                                 ceed, &trial_basis, &trial_restr);
+         InitBasisAndRestriction(test_fes, ir, nelem, indices,
+                                 ceed, &test_basis, &test_restr);
      }

      const mfem::FiniteElementSpace *mesh_fes = mesh.GetNodalFESpace();
      MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
-      InitBasisAndRestriction(*mesh_fes, irm, ceed, &mesh_basis,
-                              &mesh_restr);
+      InitBasisAndRestriction(*mesh_fes, ir, nelem, indices,
+                              ceed, &mesh_basis, &mesh_restr);

      CeedInt trial_nqpts, test_nqpts;
      CeedBasisGetNumQuadraturePoints(trial_basis, &trial_nqpts);
@@ -177,7 +234,7 @@ public:
      MFEM_VERIFY(trial_nqpts == test_nqpts,
                  "Trial and test basis must have the same number of quadrature"
                  " points.");
-      nqpts = trial_nqpts;
+      CeedInt nqpts = trial_nqpts;

      const int qdatasize = op.qdatasize;
      InitStridedRestriction(*mesh_fes, nelem, nqpts, qdatasize,
@@ -221,8 +278,10 @@ public:
      CeedOperatorCreate(ceed, build_qfunc, NULL, NULL, &build_oper);
      if (GridCoefficient *gridCoeff = dynamic_cast<GridCoefficient*>(coeff))
      {
-         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), irm, ceed,
-                                 &gridCoeff->basis, &gridCoeff->restr);
+         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), ir,
+                                 nelem, indices, ceed,
+                                 &gridCoeff->basis,
+                                 &gridCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", gridCoeff->restr,
                              gridCoeff->basis, gridCoeff->coeffVector);
      }
@@ -231,7 +290,8 @@ public:
      {
         const int ncomp = quadCoeff->ncomp;
         CeedInt strides[3] = {ncomp, 1, ncomp*nqpts};
-         InitStridedRestriction(*mesh_fes, nelem, nqpts, ncomp, strides,
+         InitStridedRestriction(*mesh.GetNodalFESpace(),
+                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", quadCoeff->restr,
                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
@@ -254,22 +314,17 @@ public:
      switch (op.trial_op)
      {
         case EvalMode::None:
-            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim,
-                                  CEED_EVAL_NONE);
+            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim, CEED_EVAL_NONE);
            break;
         case EvalMode::Interp:
-            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim,
-                                  CEED_EVAL_INTERP);
+            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim, CEED_EVAL_INTERP);
            break;
         case EvalMode::Grad:
-            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim,
-                                  CEED_EVAL_GRAD);
+            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim, CEED_EVAL_GRAD);
            break;
         case EvalMode::InterpAndGrad:
-            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim,
-                                  CEED_EVAL_INTERP);
-            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim,
-                                  CEED_EVAL_GRAD);
+            CeedQFunctionAddInput(apply_qfunc, "u", trial_vdim, CEED_EVAL_INTERP);
+            CeedQFunctionAddInput(apply_qfunc, "gu", trial_vdim*dim, CEED_EVAL_GRAD);
            break;
      }
      // qdata
@@ -278,22 +333,17 @@ public:
      switch (op.test_op)
      {
         case EvalMode::None:
-            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim,
-                                   CEED_EVAL_NONE);
+            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim, CEED_EVAL_NONE);
            break;
         case EvalMode::Interp:
-            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim,
-                                   CEED_EVAL_INTERP);
+            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim, CEED_EVAL_INTERP);
            break;
         case EvalMode::Grad:
-            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim,
-                                   CEED_EVAL_GRAD);
+            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim, CEED_EVAL_GRAD);
            break;
         case EvalMode::InterpAndGrad:
-            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim,
-                                   CEED_EVAL_INTERP);
-            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim,
-                                   CEED_EVAL_GRAD);
+            CeedQFunctionAddOutput(apply_qfunc, "v", test_vdim, CEED_EVAL_INTERP);
+            CeedQFunctionAddOutput(apply_qfunc, "gv", test_vdim*dim, CEED_EVAL_GRAD);
            break;
      }
      CeedQFunctionSetContext(apply_qfunc, build_ctx);
@@ -308,18 +358,14 @@ public:
                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
-            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Grad:
-            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::InterpAndGrad:
-            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
-            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gu", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
            break;
      }
      // qdata
@@ -333,18 +379,14 @@ public:
                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
-            CeedOperatorSetField(oper, "v", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Grad:
-            CeedOperatorSetField(oper, "gv", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gv", test_restr, test_basis, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::InterpAndGrad:
-            CeedOperatorSetField(oper, "v", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
-            CeedOperatorSetField(oper, "gv", test_restr, test_basis,
-                                 CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
+            CeedOperatorSetField(oper, "gv", test_restr, test_basis, CEED_VECTOR_ACTIVE);
            break;
      }

@@ -402,6 +444,7 @@ protected:
   Coefficient *coeff;
   CeedQFunctionContext build_ctx;

+public:
   MFIntegrator()
      : Operator(),
        trial_basis(nullptr), test_basis(nullptr), mesh_basis(nullptr),
@@ -410,23 +453,51 @@ protected:
        apply_qfunc(nullptr), node_coords(nullptr),
        qdata(nullptr), coeff(nullptr), build_ctx(nullptr) { }

-public:
-   /** This method assembles the MFIntegrator.
+   /** @brief This method assembles the `MFIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.

-       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
-                       the `CeedOperatorInfo` type is expected to inherit from
-                       `OperatorInfo` and contain a `Context` type relevant to
-                       the qFunctions.
-       @param[in] fes the `FiniteElementSpace` for the form,
-       @param[in] ir the `IntegrationRule` for the numerical integration,
-       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] Q is the coefficient from the `Integrator`. */
   template <typename CeedOperatorInfo, typename CoeffType>
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
                 CoeffType *Q)
   {
-      Assemble(info, fes, fes, irm, Q);
+      Assemble(info, fes, ir, fes.GetNE(), nullptr, Q);
+   }
+
+   /** @brief This method assembles the `MFIntegrator` with the given
+       `CeedOperatorInfo` @a info, an `mfem::FiniteElementSpace` @a fes, an
+       `mfem::IntegrationRule` @a ir, and `mfem::Coefficient` or
+       `mfem::VectorCoefficient` @a Q for the elements given by the indices
+       @a indices.
+       The `CeedOperatorInfo` type is expected to inherit from `OperatorInfo`,
+       and contain a `Context` type relevant to the qFunctions.
+
+       @param[in] info is the structure describing the CeedOperator to assemble.
+       @param[in] fes is the finite element space.
+       @param[in] ir is the integration rule for the operator.
+       @param[in] nelem The number of elements.
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed.
+       @param[in] Q is the coefficient from the `Integrator`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
+                 CoeffType *Q)
+   {
+      Assemble(info, fes, fes, ir, nelem, indices, Q);
   }

   /** This method assembles the MFIntegrator for mixed forms.
@@ -443,12 +514,40 @@ public:
   void Assemble(CeedOperatorInfo &info,
                 const mfem::FiniteElementSpace &trial_fes,
                 const mfem::FiniteElementSpace &test_fes,
-                 const mfem::IntegrationRule &irm,
+                 const mfem::IntegrationRule &ir,
+                 CoeffType *Q)
+   {
+      Assemble(info, trial_fes, test_fes, ir, trial_fes.GetNE(), nullptr, Q);
+   }
+
+   /** This method assembles the MFIntegrator for mixed forms.
+
+       @param[in] info the `CeedOperatorInfo` describing the `CeedOperator`,
+                       the `CeedOperatorInfo` type is expected to inherit from
+                       `OperatorInfo` and contain a `Context` type relevant to
+                       the qFunctions.
+       @param[in] trial_fes the trial `FiniteElementSpace` for the form,
+       @param[in] test_fes the test `FiniteElementSpace` for the form,
+       @param[in] ir the `IntegrationRule` for the numerical integration,
+       @param[in] nelem The number of elements,
+       @param[in] indices The indices of the elements of same type in the
+                          `FiniteElementSpace`. If `indices == nullptr`, assumes
+                          that the `FiniteElementSpace` is not mixed,
+       @param[in] Q `Coefficient` or `VectorCoefficient`. */
+   template <typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &trial_fes,
+                 const mfem::FiniteElementSpace &test_fes,
+                 const mfem::IntegrationRule &ir,
+                 int nelem,
+                 const int* indices,
                 CoeffType *Q)
   {
      Ceed ceed(internal::ceed);
      Mesh &mesh = *trial_fes.GetMesh();
-      InitCoefficient(Q, mesh, irm, coeff, info.ctx);
+      MFEM_VERIFY(!(!indices && mesh.GetNumGeometries(mesh.Dimension()) > 1),
+                  "Use ceed::MixedIntegrator on mixed meshes.");
+      InitCoefficient(Q, mesh, ir, nelem, indices, coeff, info.ctx);
      bool const_coeff = coeff->IsConstant();
      std::string apply_func = const_coeff ? info.apply_func_mf_const
                               : info.apply_func_mf_quad;
@@ -459,7 +558,7 @@ public:
                     info.trial_op,
                     info.test_op
                    };
-      CeedInt nqpts, nelem = mesh.GetNE();
+
      CeedInt dim = mesh.SpaceDimension();
      CeedInt trial_vdim = trial_fes.GetVDim();
      CeedInt test_vdim = test_fes.GetVDim();
@@ -467,22 +566,22 @@ public:
      mesh.EnsureNodes();
      if ( &trial_fes == &test_fes )
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices, ceed,
                                 &trial_basis, &trial_restr);
         test_basis = trial_basis;
         test_restr = trial_restr;
      }
      else
      {
-         InitBasisAndRestriction(trial_fes, irm, ceed,
+         InitBasisAndRestriction(trial_fes, ir, nelem, indices, ceed,
                                 &trial_basis, &trial_restr);
-         InitBasisAndRestriction(test_fes, irm, ceed,
+         InitBasisAndRestriction(test_fes, ir, nelem, indices, ceed,
                                 &test_basis, &test_restr);
      }

      const mfem::FiniteElementSpace *mesh_fes = mesh.GetNodalFESpace();
      MFEM_VERIFY(mesh_fes, "the Mesh has no nodal FE space");
-      InitBasisAndRestriction(*mesh_fes, irm, ceed, &mesh_basis,
+      InitBasisAndRestriction(*mesh_fes, ir, nelem, indices, ceed, &mesh_basis,
                              &mesh_restr);

      CeedInt trial_nqpts, test_nqpts;
@@ -491,7 +590,7 @@ public:
      MFEM_VERIFY(trial_nqpts == test_nqpts,
                  "Trial and test basis must have the same number of quadrature"
                  " points.");
-      nqpts = trial_nqpts;
+      CeedInt nqpts = trial_nqpts;

      InitVector(*mesh.GetNodes(), node_coords);

@@ -572,8 +671,8 @@ public:
      // coefficient
      if (GridCoefficient *gridCoeff = dynamic_cast<GridCoefficient*>(coeff))
      {
-         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), irm, ceed,
-                                 &gridCoeff->basis, &gridCoeff->restr);
+         InitBasisAndRestriction(*gridCoeff->gf.FESpace(), ir, nelem, indices,
+                                 ceed, &gridCoeff->basis, &gridCoeff->restr);
         CeedOperatorSetField(oper, "coeff", gridCoeff->restr,
                              gridCoeff->basis, gridCoeff->coeffVector);
      }
@@ -22,6 +22,8 @@
 #include "coefficient.hpp"
 // PA or MF Operator using libCEED.
 #include "integrator.hpp"
+// PA Operator supporting mixed finite element spaces.
+#include "mixed_integrator.hpp"
 // Utility functions
 #include "util.hpp"
 // Wrapper to include <ceed.h>
@@ -0,0 +1,126 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_LIBCEED_MIXED_INTEGRATOR
+#define MFEM_LIBCEED_MIXED_INTEGRATOR
+
+#include "ceed.hpp"
+#include "integrator.hpp"
+#include <unordered_map>
+
+namespace mfem
+{
+
+namespace ceed
+{
+
+/** @brief This class wraps a `ceed::PAIntegrator` or `ceed::MFIntegrator` to
+    support mixed finite element spaces. */
+template <typename CeedInteg>
+class MixedIntegrator : public ceed::Operator
+{
+#ifdef MFEM_USE_CEED
+   using ElementKey = std::pair<int, int>; //< Element::Type, Order >
+   struct key_hash
+   {
+      std::size_t operator()(const ElementKey& k) const
+      {
+         return k.first + 2 * k.second;
+      }
+   };
+   using ElementsMap = std::unordered_map<const ElementKey, int*, key_hash>;
+   std::vector<CeedInteg*> sub_ops;
+
+public:
+   template <typename Integrator, typename CeedOperatorInfo, typename CoeffType>
+   void Assemble(const Integrator &integ,
+                 CeedOperatorInfo &info,
+                 const mfem::FiniteElementSpace &fes,
+                 CoeffType *Q)
+   {
+      ElementsMap count;
+      ElementsMap element_indices;
+      ElementsMap offsets;
+
+      // Count the number of elements of each type
+      for (int i = 0; i < fes.GetNE(); i++)
+      {
+         ElementKey key(fes.GetElementType(i), fes.GetElementOrder(i));
+         auto value = count.find(key);
+         if (value == count.end())
+         {
+            count[key] = new int(1);
+         }
+         else
+         {
+            (*value->second)++;
+         }
+      }
+
+      // Initialization of the arrays
+      for ( const auto& value : count )
+      {
+         element_indices[value.first] = new int[*value.second];
+         offsets[value.first] = new int(0);
+      }
+
+      // Populates the indices arrays for each element type
+      for (int i = 0; i < fes.GetNE(); i++)
+      {
+         ElementKey key(fes.GetElementType(i), fes.GetElementOrder(i));
+         int &offset = *(offsets[key]);
+         int* indices_array = element_indices[key];
+         indices_array[offset] = i;
+         offset++;
+      }
+
+      // Create composite CeedOperator
+      CeedCompositeOperatorCreate(internal::ceed, &oper);
+
+      // Create each sub-CeedOperator
+      sub_ops.reserve(element_indices.size());
+      for (const auto& value : element_indices)
+      {
+         const int* indices = value.second;
+         const int first_index = indices[0];
+         const mfem::FiniteElement &el = *fes.GetFE(first_index);
+         auto &T = *fes.GetMesh()->GetElementTransformation(first_index);
+         MFEM_ASSERT(!integ.GetIntegrationRule(),
+                     "Mixed mesh integrators should not have an"
+                     " IntegrationRule.");
+         const IntegrationRule &ir = GetRule(integ, el, el, T);
+         auto sub_op = new CeedInteg();
+         int nelem = *count[value.first];
+         sub_op->Assemble(info, fes, ir, nelem, indices, Q);
+         sub_ops.push_back(sub_op);
+         CeedCompositeOperatorAddSub(oper, sub_op->GetCeedOperator());
+      }
+
+      const int ndofs = fes.GetVDim() * fes.GetNDofs();
+      CeedVectorCreate(internal::ceed, ndofs, &u);
+      CeedVectorCreate(internal::ceed, ndofs, &v);
+   }
+
+   virtual ~MixedIntegrator()
+   {
+      for (auto sub_op : sub_ops)
+      {
+         delete sub_op;
+      }
+   }
+#endif
+};
+
+} // namespace ceed
+
+} // namespace mfem
+
+#endif // MFEM_LIBCEED_MIXED_INTEGRATOR
@@ -20,8 +20,8 @@ namespace ceed

 #ifdef MFEM_USE_CEED

-static void InitNonTensorRestriction(const mfem::FiniteElementSpace &fes,
-                                     Ceed ceed, CeedElemRestriction *restr)
+static void InitNativeRestr(const mfem::FiniteElementSpace &fes,
+                            Ceed ceed, CeedElemRestriction *restr)
 {
   const mfem::FiniteElement *fe = fes.GetFE(0);
   const int P = fe->GetDof();
@@ -31,77 +31,173 @@ static void InitNonTensorRestriction(const mfem::FiniteElementSpace &fes,
   const mfem::TensorBasisElement * tfe =
      dynamic_cast<const mfem::TensorBasisElement *>(fe);
   const int stride = compstride == 1 ? fes.GetVDim() : 1;
-   if (tfe) // Lexicographic ordering using dof_map
+   const mfem::Array<int>& dof_map = tfe->GetDofMap();
+
+   for (int i = 0; i < fes.GetNE(); i++)
   {
-      const mfem::Array<int>& dof_map = tfe->GetDofMap();
-      for (int i = 0; i < fes.GetNE(); i++)
+      const int el_offset = P * i;
+      for (int j = 0; j < P; j++)
      {
-         const int el_offset = P * i;
-         for (int j = 0; j < P; j++)
-         {
-            tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[dof_map[j]+el_offset];
-         }
-      }
-   }
-   else  // Native ordering
-   {
-      for (int e = 0; e < fes.GetNE(); e++)
-      {
-         for (int i = 0; i < P; i++)
-         {
-            tp_el_dof[i + e*P] = stride*el_dof.GetJ()[i + e*P];
-         }
+         tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[dof_map[j]+el_offset];
      }
   }
+
   CeedElemRestrictionCreate(ceed, fes.GetNE(), P, fes.GetVDim(),
                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
                             CEED_MEM_HOST, CEED_COPY_VALUES,
                             tp_el_dof.GetData(), restr);
 }

-// TODO fuse Tensor and NonTensor Restriction
-void InitTensorRestriction(const mfem::FiniteElementSpace &fes,
-                           Ceed ceed, CeedElemRestriction *restr)
+static void InitLexicoRestr(const mfem::FiniteElementSpace &fes,
+                            Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(0);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   const mfem::Table &el_dof = fes.GetElementToDofTable();
+   mfem::Array<int> tp_el_dof(el_dof.Size_of_connections());
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
+
+   for (int e = 0; e < fes.GetNE(); e++)
+   {
+      for (int i = 0; i < P; i++)
+      {
+         tp_el_dof[i + e*P] = stride*el_dof.GetJ()[i + e*P];
+      }
+   }
+
+   CeedElemRestrictionCreate(ceed, fes.GetNE(), P, fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+static void InitRestrictionImpl(const mfem::FiniteElementSpace &fes,
+                                Ceed ceed, CeedElemRestriction *restr)
 {
   const mfem::FiniteElement *fe = fes.GetFE(0);
   const mfem::TensorBasisElement * tfe =
      dynamic_cast<const mfem::TensorBasisElement *>(fe);
-   MFEM_VERIFY(tfe, "invalid FE");
+   if ( tfe && tfe->GetDofMap().Size()>0 ) // Native ordering using dof_map
+   {
+      InitNativeRestr(fes, ceed, restr);
+   }
+   else  // Lexicographic ordering
+   {
+      InitLexicoRestr(fes, ceed, restr);
+   }
+}
+
+static void InitNativeRestrWithIndices(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   mfem::Array<int> tp_el_dof(nelem*P);
+   const mfem::TensorBasisElement * tfe =
+      dynamic_cast<const mfem::TensorBasisElement *>(fe);
+   Array<int> dofs;
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
   const mfem::Array<int>& dof_map = tfe->GetDofMap();

-   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
-   const mfem::Table &el_dof = fes.GetElementToDofTable();
-   mfem::Array<int> tp_el_dof(el_dof.Size_of_connections());
-   const int dof = fe->GetDof();
-   const int stride = compstride == 1 ? fes.GetVDim() : 1;
-   if (dof_map.Size()>0)
+   for (int i = 0; i < nelem; i++)
   {
-      for (int i = 0; i < fes.GetNE(); i++)
+      const int elem_index = indices[i];
+      fes.GetElementDofs(elem_index, dofs);
+      const int el_offset = P * i;
+      for (int j = 0; j < P; j++)
      {
-         const int el_offset = dof * i;
-         for (int j = 0; j < dof; j++)
-         {
-            tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[dof_map[j]+el_offset];
-         }
+         tp_el_dof[j + el_offset] = stride*dofs[dof_map[j]];
      }
   }
-   else // dof_map.Size == 0, means dof_map[j]==j;
-   {
-      for (int i = 0; i < fes.GetNE(); i++)
-      {
-         const int el_offset = dof * i;
-         for (int j = 0; j < dof; j++)
-         {
-            tp_el_dof[j+el_offset] = stride*el_dof.GetJ()[j+el_offset];
-         }
-      }
-   }
-   CeedElemRestrictionCreate(ceed, fes.GetNE(), dof, fes.GetVDim(),
+
+   CeedElemRestrictionCreate(ceed, nelem, P, fes.GetVDim(),
                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
                             CEED_MEM_HOST, CEED_COPY_VALUES,
                             tp_el_dof.GetData(), restr);
 }

+static void InitLexicoRestrWithIndices(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const int P = fe->GetDof();
+   CeedInt compstride = fes.GetOrdering()==Ordering::byVDIM ? 1 : fes.GetNDofs();
+   mfem::Array<int> tp_el_dof(nelem*P);
+   Array<int> dofs;
+   const int stride = compstride == 1 ? fes.GetVDim() : 1;
+
+   for (int i = 0; i < nelem; i++)
+   {
+      const int elem_index = indices[i];
+      fes.GetElementDofs(elem_index, dofs);
+      const int el_offset = P * i;
+      for (int j = 0; j < P; j++)
+      {
+         tp_el_dof[j + el_offset] = stride*dofs[j];
+      }
+   }
+
+   CeedElemRestrictionCreate(ceed, nelem, P, fes.GetVDim(),
+                             compstride, (fes.GetVDim())*(fes.GetNDofs()),
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
+static void InitRestrictionWithIndicesImpl(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   Ceed ceed, CeedElemRestriction *restr)
+{
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const mfem::TensorBasisElement * tfe =
+      dynamic_cast<const mfem::TensorBasisElement *>(fe);
+   if ( tfe && tfe->GetDofMap().Size()>0 ) // Native ordering using dof_map
+   {
+      InitNativeRestrWithIndices(fes, nelem, indices, ceed, restr);
+   }
+   else  // Lexicographic ordering
+   {
+      InitLexicoRestrWithIndices(fes, nelem, indices, ceed, restr);
+   }
+}
+
+static void InitCoeffRestrictionWithIndicesImpl(
+   const mfem::FiniteElementSpace &fes,
+   int nelem,
+   const int* indices,
+   int nquads,
+   int ncomp,
+   Ceed ceed,
+   CeedElemRestriction *restr)
+{
+   mfem::Array<int> tp_el_dof(nelem*nquads);
+   const int stride_quad = ncomp;
+   const int stride_elem = ncomp*nquads;
+   // TODO generalize to support different #quads
+   for (int i = 0; i < nelem; i++)
+   {
+      const int elem_index = indices[i];
+      const int el_offset = elem_index * stride_elem;
+      for (int j = 0; j < nquads; j++)
+      {
+         tp_el_dof[j + nquads * i] = j * stride_quad + el_offset;
+      }
+   }
+   CeedElemRestrictionCreate(ceed, nelem, nquads, ncomp, 1,
+                             ncomp*fes.GetNE()*nquads,
+                             CEED_MEM_HOST, CEED_COPY_VALUES,
+                             tp_el_dof.GetData(), restr);
+}
+
 void InitStridedRestriction(const mfem::FiniteElementSpace &fes,
                            CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
                            const CeedInt *strides,
@@ -136,17 +232,60 @@ void InitRestriction(const FiniteElementSpace &fes,
   RestrKey restr_key(&fes, nelem, P, ncomp, restr_type::Standard);
   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);

-   // Init or retreive key values
+   // Init or retrieve key values
   if (restr_itr == mfem::internal::ceed_restr_map.end())
   {
-      if (UsesTensorBasis(fes))
-      {
-         InitTensorRestriction(fes, ceed, restr);
-      }
-      else
-      {
-         InitNonTensorRestriction(fes, ceed, restr);
-      }
+      InitRestrictionImpl(fes, ceed, restr);
+      mfem::internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
+   }
+}
+
+void InitRestrictionWithIndices(const FiniteElementSpace &fes,
+                                int nelem,
+                                const int* indices,
+                                Ceed ceed,
+                                CeedElemRestriction *restr)
+{
+   // Check for FES -> basis, restriction in hash tables
+   const mfem::FiniteElement *fe = fes.GetFE(indices[0]);
+   const int P = fe->GetDof();
+   const int ncomp = fes.GetVDim();
+   RestrKey restr_key(&fes, nelem, P, ncomp, restr_type::Standard);
+   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
+
+   // Init or retrieve key values
+   if (restr_itr == mfem::internal::ceed_restr_map.end())
+   {
+      InitRestrictionWithIndicesImpl(fes, nelem, indices, ceed, restr);
+      mfem::internal::ceed_restr_map[restr_key] = *restr;
+   }
+   else
+   {
+      *restr = restr_itr->second;
+   }
+}
+
+void InitCoeffRestrictionWithIndices(const FiniteElementSpace &fes,
+                                     int nelem,
+                                     const int* indices,
+                                     int nquads,
+                                     int ncomp,
+                                     Ceed ceed,
+                                     CeedElemRestriction *restr)
+{
+   // Check for FES -> basis, restriction in hash tables
+   RestrKey restr_key(&fes, nelem, nquads, ncomp, restr_type::Coeff);
+   auto restr_itr = mfem::internal::ceed_restr_map.find(restr_key);
+
+   // Init or retrieve key values
+   if (restr_itr == mfem::internal::ceed_restr_map.end())
+   {
+      InitCoeffRestrictionWithIndicesImpl(fes, nelem, indices, nquads, ncomp,
+                                          ceed, restr);
      mfem::internal::ceed_restr_map[restr_key] = *restr;
   }
   else
@@ -21,37 +21,63 @@ namespace ceed
 {

 #ifdef MFEM_USE_CEED
-/// @brief Initialize a strided CeedElemRestriction
-/** @a nelem is the number of elements,
-    @a nqpts is the total number of quadrature points
-    @a qdatasize is the number of data per quadrature point
-    @a strides Array for strides between [nodes, components, elements].
-    Data for node i, component j, element k can be found in the L-vector at
-    index i*strides[0] + j*strides[1] + k*strides[2]. CEED_STRIDES_BACKEND may
-    be used with vectors created by a Ceed backend. */
-void InitStridedRestriction(const mfem::FiniteElementSpace &fes,
-                            CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
-                            const CeedInt *strides,
-                            CeedElemRestriction *restr);
+/** @brief Initialize a CeedElemRestriction for non-mixed meshes.

-/** @brief Initialize a CeedElemRestriction.
- *
- * @param[in] fes Input finite element space.
- * @param[in] ceed Input Ceed object.
-   @param[out] restr The address of the initialized CeedElemRestriction object.
+    @param[in] fes Input finite element space.
+    @param[in] ceed Input Ceed object.
+    @param[out] restr The address of the initialized CeedElemRestriction object.
 */
 void InitRestriction(const FiniteElementSpace &fes,
                     Ceed ceed,
                     CeedElemRestriction *restr);

-/** @brief Initialize a CeedElemRestriction.
- *
- * @param[in] fes Input finite element space.
- * @param[in] ceed Input Ceed object.
-   @param[out] restr The address of the initialized CeedElemRestriction object.
-*/
-void InitTensorRestriction(const FiniteElementSpace &fes,
-                           Ceed ceed, CeedElemRestriction *restr);
+/** @brief Initialize a CeedElemRestriction for mixed meshes.
+
+    @param[in] fes The finite element space.
+    @param[in] ceed The Ceed object.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[out] restr The `CeedElemRestriction` to initialize. */
+void InitRestrictionWithIndices(const FiniteElementSpace &fes,
+                                int nelem,
+                                const int* indices,
+                                Ceed ceed,
+                                CeedElemRestriction *restr);
+
+/** @brief Initialize a strided CeedElemRestriction
+
+    @param[in] nelem is the number of elements.
+    @param[in] nqpts is the total number of quadrature points.
+    @param[in] qdatasize is the number of data per quadrature point.
+    @param[in] strides Array for strides between [nodes, components, elements].
+    Data for node i, component j, element k can be found in the L-vector at
+    index i*strides[0] + j*strides[1] + k*strides[2]. CEED_STRIDES_BACKEND may
+    be used with vectors created by a Ceed backend.
+    @param[out] restr The `CeedElemRestriction` to initialize. */
+void InitStridedRestriction(const mfem::FiniteElementSpace &fes,
+                            CeedInt nelem, CeedInt nqpts, CeedInt qdatasize,
+                            const CeedInt *strides,
+                            CeedElemRestriction *restr);
+
+/** @brief Initialize a CeedElemRestriction for a mfem::Coefficient on a mixed
+    mesh.
+
+    @param[in] fes The finite element space.
+    @param[in] nelem is the number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`.
+    @param[in] nquads is the total number of quadrature points
+    @param[in] ncomp is the number of data per quadrature point
+    @param[in] ceed The Ceed object.
+    @param[out] restr The `CeedElemRestriction` to initialize. */
+void InitCoeffRestrictionWithIndices(const FiniteElementSpace &fes,
+                                     int nelem,
+                                     const int* indices,
+                                     int nquads,
+                                     int ncomp,
+                                     Ceed ceed,
+                                     CeedElemRestriction *restr);

 #endif

@@ -99,6 +99,34 @@ void InitBasisAndRestriction(const FiniteElementSpace &fes,
   InitRestriction(fes, ceed, restr);
 }

+void InitBasisAndRestrictionWithIndices(const FiniteElementSpace &fes,
+                                        const IntegrationRule &irm,
+                                        int nelem,
+                                        const int* indices,
+                                        Ceed ceed, CeedBasis *basis,
+                                        CeedElemRestriction *restr)
+{
+   InitBasisWithIndices(fes, irm, nelem, indices, ceed, basis);
+   InitRestrictionWithIndices(fes, nelem, indices, ceed, restr);
+}
+
+void InitBasisAndRestriction(const FiniteElementSpace &fes,
+                             const IntegrationRule &irm,
+                             int nelem,
+                             const int* indices,
+                             Ceed ceed, CeedBasis *basis,
+                             CeedElemRestriction *restr)
+{
+   if (indices)
+   {
+      InitBasisAndRestrictionWithIndices(fes,irm,nelem,indices,ceed,basis,restr);
+   }
+   else
+   {
+      InitBasisAndRestriction(fes,irm,ceed,basis,restr);
+   }
+}
+
 // Assumes a tensor-product operator with one active field
 int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
 {
@@ -158,6 +186,66 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   return 0;
 }

+template <>
+const IntegrationRule & GetRule<MassIntegrator>(
+   const MassIntegrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &trans)
+{
+   return MassIntegrator::GetRule(trial_fe, test_fe, trans);
+}
+
+template <>
+const IntegrationRule & GetRule<VectorMassIntegrator>(
+   const VectorMassIntegrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &trans)
+{
+   return MassIntegrator::GetRule(trial_fe, test_fe, trans);
+}
+
+template <>
+const IntegrationRule & GetRule<ConvectionIntegrator>(
+   const ConvectionIntegrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &trans)
+{
+   return ConvectionIntegrator::GetRule(trial_fe, test_fe, trans);
+}
+
+template <>
+const IntegrationRule & GetRule<VectorConvectionNLFIntegrator>(
+   const VectorConvectionNLFIntegrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &trans)
+{
+   return VectorConvectionNLFIntegrator::GetRule(trial_fe, trans);
+}
+
+template <>
+const IntegrationRule & GetRule<DiffusionIntegrator>(
+   const DiffusionIntegrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &trans)
+{
+   return DiffusionIntegrator::GetRule(trial_fe, test_fe);
+}
+
+template <>
+const IntegrationRule & GetRule<VectorDiffusionIntegrator>(
+   const VectorDiffusionIntegrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &trans)
+{
+   return DiffusionIntegrator::GetRule(trial_fe, test_fe);
+}
+
 std::string ceed_path;

 const std::string &GetCeedPath()
@@ -26,7 +26,9 @@
 namespace mfem
 {

+class FiniteElement;
 class FiniteElementSpace;
+class ElementTransformation;
 class IntegrationRule;
 class Vector;

@@ -55,15 +57,51 @@ void RemoveBasisAndRestriction(const mfem::FiniteElementSpace *fes);
 /// Initialize a CeedVector from an mfem::Vector
 void InitVector(const mfem::Vector &v, CeedVector &cv);

-/** Initialize a CeedBasis and a CeedElemRestriction based on an
-    mfem::FiniteElementSpace @a fes, and an mfem::IntegrationRule @a ir. */
+/** @brief Initialize a CeedBasis and a CeedElemRestriction based on an
+    mfem::FiniteElementSpace @a fes, and an mfem::IntegrationRule @a ir.
+
+    @param[in] fes The finite element space.
+    @param[in] ir The integration rule.
+    @param[in] ceed The Ceed object.
+    @param[out] basis The `CeedBasis` to initialize.
+    @param[out] restr The `CeedElemRestriction` to initialize.
+
+    @warning Only for non-mixed finite element spaces. */
 void InitBasisAndRestriction(const mfem::FiniteElementSpace &fes,
                             const mfem::IntegrationRule &ir,
                             Ceed ceed, CeedBasis *basis,
                             CeedElemRestriction *restr);

+/** @brief Initialize a CeedBasis and a CeedElemRestriction based on an
+    mfem::FiniteElementSpace @a fes, and an mfem::IntegrationRule @a ir,
+    and a list of @a nelem elements of indices @a indices.
+
+    @param[in] fes The finite element space.
+    @param[in] ir The integration rule.
+    @param[in] nelem The number of elements.
+    @param[in] indices The indices of the elements of same type in the
+                       `FiniteElementSpace`. If `indices == nullptr`, assumes
+                       that the `FiniteElementSpace` is not mixed.
+    @param[in] ceed The Ceed object.
+    @param[out] basis The `CeedBasis` to initialize.
+    @param[out] restr The `CeedElemRestriction` to initialize. */
+void InitBasisAndRestriction(const FiniteElementSpace &fes,
+                             const IntegrationRule &ir,
+                             int nelem,
+                             const int* indices,
+                             Ceed ceed, CeedBasis *basis,
+                             CeedElemRestriction *restr);
+
 int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field);

+
+template <typename Integrator>
+const IntegrationRule & GetRule(
+   const Integrator &integ,
+   const FiniteElement &trial_fe,
+   const FiniteElement &test_fe,
+   ElementTransformation &Trans);
+
 /// Return the path to the libCEED q-function headers.
 const std::string &GetCeedPath();

@@ -87,7 +125,7 @@ struct BasisHash
 };
 using BasisMap = std::unordered_map<const BasisKey, CeedBasis, BasisHash>;

-enum restr_type {Standard, Strided};
+enum restr_type {Standard, Strided, Coeff};

 // Hash table for CeedElemRestriction
 using RestrKey =
@@ -117,6 +155,8 @@ namespace internal
 {

 #ifdef MFEM_USE_CEED
+/** @warning These maps have a tendency to create bugs when adding new "types"
+    of CeedBasis and CeedElemRestriction. */
 extern ceed::BasisMap ceed_basis_map;
 extern ceed::RestrMap ceed_restr_map;
 #endif
@@ -633,7 +633,7 @@ AlgebraicSpaceHierarchy::AlgebraicSpaceHierarchy(FiniteElementSpace &fes)
   current_order = order;

   Ceed ceed = internal::ceed;
-   InitTensorRestriction(fes, ceed, &fine_er);
+   InitRestriction(fes, ceed, &fine_er);
   CeedElemRestriction er = fine_er;

   int dim = fes.GetMesh()->Dimension();
@@ -745,7 +745,7 @@ ParAlgebraicCoarseSpace::ParAlgebraicCoarseSpace(
   ldof_group.SetSize(lsize);
   ldof_group = 0;

-   GroupTopology &group_topo = gc_fine->GetGroupTopology();
+   const GroupTopology &group_topo = gc_fine->GetGroupTopology();
   gc = new GroupCommunicator(group_topo);
   Table &group_ldof = gc->GroupLDofTable();
   group_ldof.MakeI(group_ldof_fine.Size());
@@ -822,11 +822,11 @@ HypreParMatrix *ParAlgebraicCoarseSpace::GetProlongationHypreParMatrix()

   ParMesh *pmesh = dynamic_cast<ParMesh*>(mesh);
   MFEM_VERIFY(pmesh != NULL, "");
-   Array<HYPRE_Int> dof_offsets, tdof_offsets, tdof_nb_offsets;
-   Array<HYPRE_Int> *offsets[2] = {&dof_offsets, &tdof_offsets};
+   Array<HYPRE_BigInt> dof_offsets, tdof_offsets, tdof_nb_offsets;
+   Array<HYPRE_BigInt> *offsets[2] = {&dof_offsets, &tdof_offsets};
   int lsize = P->Height();
   int ltsize = P->Width();
-   HYPRE_Int loc_sizes[2] = {lsize, ltsize};
+   HYPRE_BigInt loc_sizes[2] = {lsize, ltsize};
   pmesh->GenerateOffsets(2, loc_sizes, offsets);

   MPI_Comm comm = pmesh->GetComm();
@@ -870,12 +870,12 @@ HypreParMatrix *ParAlgebraicCoarseSpace::GetProlongationHypreParMatrix()
   HYPRE_Int *j_offd = Memory<HYPRE_Int>(lsize-ltsize);
   int offd_counter;

-   HYPRE_Int *cmap   = Memory<HYPRE_Int>(lsize-ltsize);
+   HYPRE_BigInt *cmap   = Memory<HYPRE_BigInt>(lsize-ltsize);

-   HYPRE_Int *col_starts = tdof_offsets;
-   HYPRE_Int *row_starts = dof_offsets;
+   HYPRE_BigInt *col_starts = tdof_offsets;
+   HYPRE_BigInt *row_starts = dof_offsets;

-   Array<Pair<HYPRE_Int, int> > cmap_j_offd(lsize-ltsize);
+   Array<Pair<HYPRE_BigInt, int> > cmap_j_offd(lsize-ltsize);

   i_diag[0] = i_offd[0] = 0;
   diag_counter = offd_counter = 0;
@@ -909,7 +909,7 @@ HypreParMatrix *ParAlgebraicCoarseSpace::GetProlongationHypreParMatrix()
      i_offd[i_ldof+1] = offd_counter;
   }

-   SortPairs<HYPRE_Int, int>(cmap_j_offd, offd_counter);
+   SortPairs<HYPRE_BigInt, int>(cmap_j_offd, offd_counter);

   for (int i = 0; i < offd_counter; i++)
   {
@@ -134,7 +134,6 @@ public:
   }
   ~AlgebraicSpaceHierarchy()
   {
-      CeedElemRestrictionDestroy(&fine_er);
      for (int i=0; i<R_tr.Size(); ++i)
      {
         delete R_tr[i];
@@ -48,6 +48,31 @@ ElementTransformation *RefinedToCoarse(
   return coarse_T;
 }

+void Coefficient::Project(QuadratureFunction &qf)
+{
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   Vector values;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         const int iq_p = qspace.GetPermutedIndex(iel, iq);
+         values[iq_p] = Eval(T, ip);
+      }
+   }
+}
+
+void ConstantCoefficient::Project(QuadratureFunction &qf)
+{
+   qf = constant;
+}
+
 double PWConstCoefficient::Eval(ElementTransformation & T,
                                const IntegrationPoint & ip)
 {
@@ -135,6 +160,11 @@ double GridFunctionCoefficient::Eval (ElementTransformation &T,
   }
 }

+void GridFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf.ProjectGridFunction(*GridF);
+}
+
 void TransformedCoefficient::SetTime(double t)
 {
   if (Q1) { Q1->SetTime(t); }
@@ -203,6 +233,29 @@ void VectorCoefficient::Eval(DenseMatrix &M, ElementTransformation &T,
   }
 }

+void VectorCoefficient::Project(QuadratureFunction &qf)
+{
+   MFEM_VERIFY(vdim == qf.GetVDim(), "Wrong sizes.");
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   DenseMatrix values;
+   Vector col;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         const int iq_p = qspace.GetPermutedIndex(iel, iq);
+         values.GetColumnReference(iq_p, col);
+         Eval(col, T, ip);
+      }
+   }
+}
+
 void PWVectorCoefficient::InitMap(const Array<int> & attr,
                                  const Array<VectorCoefficient*> & coefs)
 {
@@ -368,6 +421,11 @@ void VectorGridFunctionCoefficient::Eval(
   }
 }

+void VectorGridFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf.ProjectGridFunction(*GridFunc);
+}
+
 GradientGridFunctionCoefficient::GradientGridFunctionCoefficient (
   const GridFunction *gf)
   : VectorCoefficient((gf) ?
@@ -517,6 +575,29 @@ void VectorRestrictedCoefficient::Eval(
   }
 }

+void MatrixCoefficient::Project(QuadratureFunction &qf, bool transpose)
+{
+   MFEM_VERIFY(qf.GetVDim() == height*width, "Wrong sizes.");
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   DenseMatrix values, matrix;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         const int iq_p = qspace.GetPermutedIndex(iel, iq);
+         matrix.UseExternalData(&values(0, iq_p), height, width);
+         Eval(matrix, T, ip);
+         if (transpose) { matrix.Transpose(); }
+      }
+   }
+}
+
 void PWMatrixCoefficient::InitMap(const Array<int> & attr,
                                  const Array<MatrixCoefficient*> & coefs)
 {
@@ -669,6 +750,31 @@ void MatrixFunctionCoefficient::EvalSymmetric(Vector &K,
   }
 }

+void SymmetricMatrixCoefficient::ProjectSymmetric(QuadratureFunction &qf)
+{
+   const int vdim = qf.GetVDim();
+   MFEM_VERIFY(vdim == height*(height+1)/2, "Wrong sizes.");
+
+   QuadratureSpaceBase &qspace = *qf.GetSpace();
+   const int ne = qspace.GetNE();
+   DenseMatrix values;
+   DenseSymmetricMatrix matrix;
+   for (int iel = 0; iel < ne; ++iel)
+   {
+      qf.GetValues(iel, values);
+      const IntegrationRule &ir = qspace.GetIntRule(iel);
+      ElementTransformation& T = *qspace.GetTransformation(iel);
+      for (int iq = 0; iq < ir.Size(); ++iq)
+      {
+         const IntegrationPoint &ip = ir[iq];
+         T.SetIntPoint(&ip);
+         matrix.UseExternalData(&values(0, iq), vdim);
+         Eval(matrix, T, ip);
+      }
+   }
+}
+
+
 void SymmetricMatrixCoefficient::Eval(DenseMatrix &K, ElementTransformation &T,
                                      const IntegrationPoint &ip)
 {
@@ -1437,12 +1543,12 @@ void VectorQuadratureFunctionCoefficient::Eval(Vector &V,

   if (index == 0 && vdim == QuadF.GetVDim())
   {
-      QuadF.GetElementValues(T.ElementNo, ip.index, V);
+      QuadF.GetValues(T.ElementNo, ip.index, V);
   }
   else
   {
      Vector temp;
-      QuadF.GetElementValues(T.ElementNo, ip.index, temp);
+      QuadF.GetValues(T.ElementNo, ip.index, temp);
      V.SetSize(vdim);
      for (int i = 0; i < vdim; i++)
      {
@@ -1453,6 +1559,11 @@ void VectorQuadratureFunctionCoefficient::Eval(Vector &V,
   return;
 }

+void VectorQuadratureFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf = QuadF;
+}
+
 QuadratureFunctionCoefficient::QuadratureFunctionCoefficient(
   QuadratureFunction &qf) : QuadF(qf)
 {
@@ -1464,8 +1575,210 @@ double QuadratureFunctionCoefficient::Eval(ElementTransformation &T,
 {
   QuadF.HostRead();
   Vector temp(1);
-   QuadF.GetElementValues(T.ElementNo, ip.index, temp);
+   QuadF.GetValues(T.ElementNo, ip.index, temp);
   return temp[0];
 }

+void QuadratureFunctionCoefficient::Project(QuadratureFunction &qf)
+{
+   qf = QuadF;
+}
+
+
+CoefficientVector::CoefficientVector(
+   QuadratureSpaceBase &qs_, CoefficientStorage storage_)
+   : Vector(), storage(storage_), vdim(0), qs(qs_), qf(NULL)
+{
+   UseDevice(true);
+}
+
+CoefficientVector::CoefficientVector(Coefficient *coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   if (coeff == NULL)
+   {
+      SetConstant(1.0);
+   }
+   else
+   {
+      Project(*coeff);
+   }
+}
+
+CoefficientVector::CoefficientVector(Coefficient &coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   Project(coeff);
+}
+
+CoefficientVector::CoefficientVector(VectorCoefficient &coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   Project(coeff);
+}
+
+CoefficientVector::CoefficientVector(MatrixCoefficient &coeff,
+                                     QuadratureSpaceBase &qs_,
+                                     CoefficientStorage storage_)
+   : CoefficientVector(qs_, storage_)
+{
+   Project(coeff);
+}
+
+void CoefficientVector::Project(Coefficient &coeff)
+{
+   vdim = 1;
+   if (auto *const_coeff = dynamic_cast<ConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_coeff->constant);
+   }
+   else if (auto *qf_coeff = dynamic_cast<QuadratureFunctionCoefficient*>(&coeff))
+   {
+      MakeRef(qf_coeff->GetQuadFunction());
+   }
+   else
+   {
+      if (qf == nullptr) { qf = new QuadratureFunction(qs); }
+      qf->SetVDim(1);
+      coeff.Project(*qf);
+      Vector::MakeRef(*qf, 0, qf->Size());
+   }
+}
+
+void CoefficientVector::Project(VectorCoefficient &coeff)
+{
+   vdim = coeff.GetVDim();
+   if (auto *const_coeff = dynamic_cast<VectorConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_coeff->GetVec());
+   }
+   else if (auto *qf_coeff =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(&coeff))
+   {
+      MakeRef(qf_coeff->GetQuadFunction());
+   }
+   else
+   {
+      if (qf == nullptr) { qf = new QuadratureFunction(qs, vdim); }
+      qf->SetVDim(vdim);
+      coeff.Project(*qf);
+      Vector::MakeRef(*qf, 0, qf->Size());
+   }
+}
+
+void CoefficientVector::Project(MatrixCoefficient &coeff, bool transpose)
+{
+   if (auto *const_coeff = dynamic_cast<MatrixConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_coeff->GetMatrix());
+   }
+   else if (auto *const_sym_coeff =
+               dynamic_cast<SymmetricMatrixConstantCoefficient*>(&coeff))
+   {
+      SetConstant(const_sym_coeff->GetMatrix());
+   }
+   else
+   {
+      auto *sym_coeff = dynamic_cast<SymmetricMatrixCoefficient*>(&coeff);
+      const bool sym = sym_coeff && (storage & CoefficientStorage::SYMMETRIC);
+      const int height = coeff.GetHeight();
+      const int width = coeff.GetWidth();
+      vdim = sym ? height*(height + 1)/2 : width*height;
+
+      if (qf == nullptr) { qf = new QuadratureFunction(qs, vdim); }
+      qf->SetVDim(vdim);
+      if (sym) { sym_coeff->ProjectSymmetric(*qf); }
+      else { coeff.Project(*qf, transpose); }
+      Vector::MakeRef(*qf, 0, qf->Size());
+   }
+}
+
+void CoefficientVector::ProjectTranspose(MatrixCoefficient &coeff)
+{
+   Project(coeff, true);
+}
+
+void CoefficientVector::MakeRef(const QuadratureFunction &qf_)
+{
+   vdim = qf_.GetVDim();
+   const QuadratureSpaceBase *qs2 = qf_.GetSpace();
+   MFEM_CONTRACT_VAR(qs2); // qs2 used only for asserts
+   MFEM_VERIFY(qs2 != NULL, "Invalid QuadratureSpace.")
+   MFEM_VERIFY(qs2->GetMesh() == qs.GetMesh(), "Meshes differ.");
+   MFEM_VERIFY(qs2->GetOrder() == qs.GetOrder(), "Orders differ.");
+   Vector::MakeRef(const_cast<QuadratureFunction&>(qf_), 0, qf_.Size());
+}
+
+void CoefficientVector::SetConstant(double constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   vdim = 1;
+   SetSize(nq);
+   Vector::operator=(constant);
+}
+
+void CoefficientVector::SetConstant(const Vector &constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   vdim = constant.Size();
+   SetSize(nq*vdim);
+   for (int iq = 0; iq < nq; ++iq)
+   {
+      for (int vd = 0; vd<vdim; ++vd)
+      {
+         (*this)[vd + iq*vdim] = constant[vd];
+      }
+   }
+}
+
+void CoefficientVector::SetConstant(const DenseMatrix &constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   const int width = constant.Width();
+   const int height = constant.Height();
+   vdim = width*height;
+   SetSize(nq*vdim);
+   for (int iq = 0; iq < nq; ++iq)
+   {
+      for (int j = 0; j < width; ++j)
+      {
+         for (int i = 0; i < height; ++i)
+         {
+            (*this)[i + j*height + iq*vdim] = constant(i, j);
+         }
+      }
+   }
+}
+
+void CoefficientVector::SetConstant(const DenseSymmetricMatrix &constant)
+{
+   const int nq = (storage & CoefficientStorage::CONSTANTS) ? 1 : qs.GetSize();
+   const int height = constant.Height();
+   const bool sym = storage & CoefficientStorage::SYMMETRIC;
+   vdim = sym ? height*(height + 1)/2 : height*height;
+   SetSize(nq*vdim);
+   for (int iq = 0; iq < nq; ++iq)
+   {
+      for (int vd = 0; vd < vdim; ++vd)
+      {
+         const double value = sym ? constant.GetData()[vd] : constant(vd % height,
+                                                                      vd / height);
+         (*this)[vd + iq*vdim] = value;
+      }
+   }
+}
+
+int CoefficientVector::GetVDim() const { return vdim; }
+
+CoefficientVector::~CoefficientVector()
+{
+   delete qf;
+}
+
 }
@@ -23,6 +23,8 @@ namespace mfem
 {

 class Mesh;
+class QuadratureSpaceBase;
+class QuadratureFunction;

 #ifdef MFEM_USE_MPI
 class ParMesh;
@@ -70,6 +72,10 @@ public:
      return Eval(T, ip);
   }

+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   virtual void Project(QuadratureFunction &qf);
+
   virtual ~Coefficient() { }
 };

@@ -87,6 +93,9 @@ public:
   virtual double Eval(ElementTransformation &T,
                       const IntegrationPoint &ip)
   { return (constant); }
+
+   /// Fill the QuadratureFunction @a qf with the constant value.
+   void Project(QuadratureFunction &qf);
 };

 /** @brief A piecewise constant coefficient with the constants keyed
@@ -274,6 +283,13 @@ public:
   /// Evaluate the coefficient at @a ip.
   virtual double Eval(ElementTransformation &T,
                       const IntegrationPoint &ip);
+
+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// This function uses the efficient QuadratureFunction::ProjectGridFunction
+   /// to fill the QuadratureFunction.
+   virtual void Project(QuadratureFunction &qf);
 };


@@ -471,6 +487,13 @@ public:
   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
                     const IntegrationRule &ir);

+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// The @a vdim of the VectorCoefficient should be equal to the @a vdim of
+   /// the QuadratureFunction.
+   virtual void Project(QuadratureFunction &qf);
+
   virtual ~VectorCoefficient() { }
 };

@@ -491,7 +514,7 @@ public:
                     const IntegrationPoint &ip) { V = vec; }

   /// Return a reference to the constant vector in this class.
-   const Vector& GetVec() { return vec; }
+   const Vector& GetVec() const { return vec; }
 };

 /** @brief A piecewise vector-valued coefficient with the pieces keyed off the
@@ -688,6 +711,13 @@ public:
   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
                     const IntegrationRule &ir);

+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// This function uses the efficient QuadratureFunction::ProjectGridFunction
+   /// to fill the QuadratureFunction.
+   virtual void Project(QuadratureFunction &qf);
+
   virtual ~VectorGridFunctionCoefficient() { }
 };

@@ -915,6 +945,14 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip) = 0;

+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points. The matrix will be transposed or not according to
+   /// the boolean argument @a transpose.
+   ///
+   /// The @a vdim of the QuadratureFunction should be equal to the height times
+   /// the width of the matrix.
+   virtual void Project(QuadratureFunction &qf, bool transpose=false);
+
   /// (DEPRECATED) Evaluate a symmetric matrix coefficient.
   /** @brief Evaluate the upper triangular entries of the matrix coefficient
       in the symmetric case, similarly to Eval. Matrix entry (i,j) is stored
@@ -943,6 +981,8 @@ public:
   /// Evaluate the matrix coefficient at @a ip.
   virtual void Eval(DenseMatrix &M, ElementTransformation &T,
                     const IntegrationPoint &ip) { M = mat; }
+   /// Return a reference to the constant matrix.
+   const DenseMatrix& GetMatrix() { return mat; }
 };


@@ -1146,6 +1186,8 @@ public:
       can be overridden with the @a own parameter. */
   void Set(int i, int j, Coefficient * c, bool own=true);

+   using MatrixCoefficient::Eval;
+
   /// Evaluate coefficient located at (i,j) in the matrix using integration
   /// point @a ip.
   double Eval(int i, int j, ElementTransformation &T, const IntegrationPoint &ip)
@@ -1260,6 +1302,15 @@ public:
   /// Get the size of the matrix.
   int GetSize() const { return height; }

+   /// @brief Fill the QuadratureFunction @a qf by evaluating the coefficient at
+   /// the quadrature points.
+   ///
+   /// @note As opposed to MatrixCoefficient::Project, this function stores only
+   /// the @a symmetric part of the matrix at each quadrature point.
+   ///
+   /// The @a vdim of the coefficient should be equal to height*(height+1)/2.
+   virtual void ProjectSymmetric(QuadratureFunction &qf);
+
   /** @brief Evaluate the matrix coefficient in the element described by @a T
       at the point @a ip, storing the result as a symmetric matrix @a K. */
   /** @note When this method is called, the caller must make sure that the
@@ -1280,6 +1331,9 @@ public:
   virtual void Eval(DenseMatrix &K, ElementTransformation &T,
                     const IntegrationPoint &ip);

+   /// Return a reference to the constant matrix.
+   const DenseSymmetricMatrix& GetMatrix() { return mat; }
+
   virtual ~SymmetricMatrixCoefficient() { }
 };

@@ -2049,8 +2103,6 @@ public:
 };
 ///@}

-class QuadratureFunction;
-
 /** @brief Vector quadrature function coefficient which requires that the
    quadrature rules used for this vector coefficient be the same as those that
    live within the supplied QuadratureFunction. */
@@ -2075,6 +2127,8 @@ public:
   virtual void Eval(Vector &V, ElementTransformation &T,
                     const IntegrationPoint &ip);

+   virtual void Project(QuadratureFunction &qf);
+
   virtual ~VectorQuadratureFunctionCoefficient() { }
 };

@@ -2094,9 +2148,123 @@ public:

   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip);

+   virtual void Project(QuadratureFunction &qf);
+
   virtual ~QuadratureFunctionCoefficient() { }
 };

+/// Flags that determine what storage optimizations to use in CoefficientVector
+enum class CoefficientStorage : int
+{
+   FULL = 0, ///< Store the coefficient as a full QuadratureFunction.
+   CONSTANTS = 1 << 0, ///< Store constants using only @a vdim entries.
+   SYMMETRIC = 1 << 1, ///< Store the triangular part of symmetric matrices.
+   COMPRESSED = CONSTANTS | SYMMETRIC ///< Enable all above compressions.
+};
+
+inline CoefficientStorage operator|(CoefficientStorage a, CoefficientStorage b)
+{
+   return CoefficientStorage(int(a) | int(b));
+}
+
+inline int operator&(CoefficientStorage a, CoefficientStorage b)
+{
+   return int(a) & int(b);
+}
+
+
+/// @brief Class to represent a coefficient evaluated at quadrature points.
+///
+/// In the general case, a CoefficientVector is the same as a QuadratureFunction
+/// with a coefficient projected onto it.
+///
+/// This class allows for some "compression" of the coefficient data, according
+/// to the storage flags given by CoefficientStorage. For example, constant
+/// coefficients can be stored using only @a vdim values, and symmetric matrices
+/// can be stored using e.g. the upper triangular part of the matrix.
+class CoefficientVector : public Vector
+{
+protected:
+   CoefficientStorage storage; ///< Storage optimizations (see CoefficientStorage).
+   int vdim; ///< Number of values per quadrature point.
+   QuadratureSpaceBase &qs; ///< Associated QuadratureSpaceBase.
+   QuadratureFunction *qf; ///< Internal QuadratureFunction (owned, may be NULL).
+public:
+   /// Create an empty CoefficientVector.
+   CoefficientVector(QuadratureSpaceBase &qs_,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given Coefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// If @a coeff is NULL, it will be interpreted as a constant with value one.
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(Coefficient *coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given Coefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(Coefficient &coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given VectorCoefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(VectorCoefficient &coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Create a CoefficientVector from the given MatrixCoefficient and
+   /// QuadratureSpaceBase.
+   ///
+   /// @sa CoefficientStorage for a description of @a storage_.
+   CoefficientVector(MatrixCoefficient &coeff, QuadratureSpaceBase &qs,
+                     CoefficientStorage storage_ = CoefficientStorage::FULL);
+
+   /// @brief Evaluate the given Coefficient at the quadrature points defined by
+   /// @ref qs.
+   void Project(Coefficient &coeff);
+
+   /// @brief Evaluate the given VectorCoefficient at the quadrature points
+   /// defined by @ref qs.
+   ///
+   /// @sa CoefficientVector for a description of the @a compress argument.
+   void Project(VectorCoefficient &coeff);
+
+   /// @brief Evaluate the given MatrixCoefficient at the quadrature points
+   /// defined by @ref qs.
+   ///
+   /// @sa CoefficientVector for a description of the @a compress argument.
+   void Project(MatrixCoefficient &coeff, bool transpose=false);
+
+   /// @brief Project the tranpose of @a coeff.
+   ///
+   /// @sa Project(MatrixCoefficient&, QuadratureSpace&, bool, bool)
+   void ProjectTranspose(MatrixCoefficient &coeff);
+
+   /// Make this vector a reference to the given QuadratureFunction.
+   void MakeRef(const QuadratureFunction &qf_);
+
+   /// Set this vector to the given constant.
+   void SetConstant(double constant);
+
+   /// Set this vector to the given constant vector.
+   void SetConstant(const Vector &constant);
+
+   /// Set this vector to the given constant matrix.
+   void SetConstant(const DenseMatrix &constant);
+
+   /// Set this vector to the given constant symmetric matrix.
+   void SetConstant(const DenseSymmetricMatrix &constant);
+
+   /// Return the number of values per quadrature point.
+   int GetVDim() const;
+
+   ~CoefficientVector();
+};
+
 /** @brief Compute the Lp norm of a function f.
    \f$ \| f \|_{Lp} = ( \int_\Omega | f |^p d\Omega)^{1/p} \f$ */
 double ComputeLpNorm(double p, Coefficient &coeff, Mesh &mesh,
@@ -442,7 +442,7 @@ void VisItDataCollection::RegisterQField(const std::string& name,
   {
      int locLOD = GlobGeometryRefiner.GetRefinementLevelFromElems(
                      mesh->GetElementBaseGeometry(e),
-                      qf->GetElementIntRule(e).GetNPoints());
+                      qf->GetIntRule(e).GetNPoints());

      LOD = std::max(LOD,locLOD);
   }
@@ -1078,7 +1078,7 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
         it->second->GetValues(i, RefG->RefPts, val, pmat);
         for (int j = 0; j < val.Size(); j++)
         {
-            WriteBinaryOrASCII(out, buf, val(j), "\n", pv_data_format);
+            WriteBinaryOrASCII(os, buf, val(j), "\n", pv_data_format);
         }
      }
   }
@@ -1094,7 +1094,7 @@ void ParaViewDataCollection::SaveGFieldVTU(std::ostream &os, int ref_,
         {
            for (int ii = 0; ii < vval.Height(); ii++)
            {
-               WriteBinaryOrASCII(out, buf, vval(ii,jj), " ", pv_data_format);
+               WriteBinaryOrASCII(os, buf, vval(ii,jj), " ", pv_data_format);
            }
            if (pv_data_format == VTKFormat::ASCII) { os << '\n'; }
         }
@@ -14,6 +14,7 @@

 #include "../config/config.hpp"
 #include "gridfunc.hpp"
+#include "qfunction.hpp"
 #ifdef MFEM_USE_MPI
 #include "pgridfunc.hpp"
 #endif
@@ -0,0 +1,315 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#include "dgmassinv.hpp"
+#include "bilinearform.hpp"
+#include "dgmassinv_kernels.hpp"
+#include "../general/forall.hpp"
+
+namespace mfem
+{
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_orig, Coefficient *coeff,
+                             const IntegrationRule *ir,
+                             int btype)
+   : Solver(fes_orig.GetTrueVSize()),
+     fec(fes_orig.GetMaxElementOrder(),
+         fes_orig.GetMesh()->Dimension(),
+         btype,
+         fes_orig.GetFE(0)->GetMapType()),
+     fes(fes_orig.GetMesh(), &fec)
+{
+   MFEM_VERIFY(fes.IsDGSpace(), "Space must be DG.");
+   MFEM_VERIFY(!fes.IsVariableOrder(), "Variable orders not supported.");
+
+   const int btype_orig =
+      static_cast<const L2_FECollection*>(fes_orig.FEColl())->GetBasisType();
+
+   if (btype_orig == btype)
+   {
+      // No change of basis required
+      d2q = nullptr;
+   }
+   else
+   {
+      // original basis to solver basis
+      const auto mode = DofToQuad::TENSOR;
+      d2q = &fes_orig.GetFE(0)->GetDofToQuad(fes.GetFE(0)->GetNodes(), mode);
+
+      int n = d2q->ndof;
+      Array<double> B_inv = d2q->B; // deep copy
+      Array<int> ipiv(n);
+      // solver basis to original
+      LUFactors lu(B_inv.HostReadWrite(), ipiv.HostWrite());
+      lu.Factor(n);
+      B_.SetSize(n*n);
+      lu.GetInverseMatrix(n, B_.HostWrite());
+      Bt_.SetSize(n*n);
+      DenseMatrix B_matrix(B_.HostReadWrite(), n, n);
+      DenseMatrix Bt_matrix(Bt_.HostWrite(), n, n);
+      Bt_matrix.Transpose(B_matrix);
+   }
+
+   if (coeff) { m = new MassIntegrator(*coeff, ir); }
+   else { m = new MassIntegrator(ir); }
+
+   diag_inv.SetSize(height);
+   // Workspace vectors used for CG
+   r_.SetSize(height);
+   d_.SetSize(height);
+   z_.SetSize(height);
+   // Only need transformed RHS if basis is different
+   if (btype_orig != btype) { b2_.SetSize(height); }
+
+   M = new BilinearForm(&fes);
+   M->AddDomainIntegrator(m); // M assumes ownership of m
+   M->SetAssemblyLevel(AssemblyLevel::PARTIAL);
+
+   // Assemble the bilinear form and its diagonal (for preconditioning).
+   Update();
+}
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                             int btype)
+   : DGMassInverse(fes_, &coeff, nullptr, btype) { }
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                             const IntegrationRule &ir, int btype)
+   : DGMassInverse(fes_, &coeff, &ir, btype) { }
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_,
+                             const IntegrationRule &ir, int btype)
+   : DGMassInverse(fes_, nullptr, &ir, btype) { }
+
+DGMassInverse::DGMassInverse(FiniteElementSpace &fes_, int btype)
+   : DGMassInverse(fes_, nullptr, nullptr, btype) { }
+
+void DGMassInverse::SetOperator(const Operator &op)
+{
+   MFEM_ABORT("SetOperator not supported with DGMassInverse.")
+}
+
+void DGMassInverse::SetRelTol(const double rel_tol_) { rel_tol = rel_tol_; }
+
+void DGMassInverse::SetAbsTol(const double abs_tol_) { abs_tol = abs_tol_; }
+
+void DGMassInverse::SetMaxIter(const double max_iter_) { max_iter = max_iter_; }
+
+void DGMassInverse::Update()
+{
+   M->Assemble();
+   M->AssembleDiagonal(diag_inv);
+   internal::MakeReciprocal(diag_inv.Size(), diag_inv.ReadWrite());
+}
+
+DGMassInverse::~DGMassInverse()
+{
+   delete M;
+}
+
+template<int DIM, int D1D, int Q1D>
+void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
+{
+   using namespace internal; // host/device kernel functions
+
+   const int NE = fes.GetNE();
+   const int d1d = m->dofs1D;
+   const int q1d = m->quad1D;
+
+   const int ND = static_cast<int>(pow(d1d, DIM));
+
+   const auto B = m->maps->B.Read();
+   const auto Bt = m->maps->Bt.Read();
+   const auto pa_data = m->pa_data.Read();
+   const auto dinv = diag_inv.Read();
+   auto r = r_.Write();
+   auto d = d_.Write();
+   auto z = z_.Write();
+   auto u = u_.ReadWrite();
+
+   const double RELTOL = rel_tol;
+   const double ABSTOL = abs_tol;
+   const double MAXIT = max_iter;
+   const bool IT_MODE = iterative_mode;
+   const bool CHANGE_BASIS = (d2q != nullptr);
+
+   // b is the right-hand side (if no change of basis, this just points to the
+   // incoming RHS vector, if we have to change basis, this points to the
+   // internal b2 vector where we put the transformed RHS)
+   const double *b;
+   // the following are non-null if we have to change basis
+   double *b2 = nullptr; // non-const access to b2
+   const double *b_orig = nullptr; // RHS vector in "original" basis
+   const double *d2q_B = nullptr; // matrix to transform initial guess
+   const double *q2d_B = nullptr; // matrix to transform solution
+   const double *q2d_Bt = nullptr; // matrix to transform RHS
+   if (CHANGE_BASIS)
+   {
+      d2q_B = d2q->B.Read();
+      q2d_B = B_.Read();
+      q2d_Bt = Bt_.Read();
+
+      b2 = b2_.Write();
+      b_orig = b_.Read();
+      b = b2;
+   }
+   else
+   {
+      b = b_.Read();
+   }
+
+   constexpr int NB = Q1D ? Q1D : 1; // block size
+
+   MFEM_FORALL_2D(e, NE, NB, NB, 1,
+   {
+      constexpr int NB = Q1D ? Q1D : 1; // redefine here for some compilers
+
+      // Perform change of basis if needed
+      if (CHANGE_BASIS)
+      {
+         // Transform RHS
+         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
+         if (IT_MODE)
+         {
+            // Transform initial guess
+            DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, d2q_B, u, u, d1d);
+         }
+      }
+
+      const int tid = MFEM_THREAD_ID(x) + NB*MFEM_THREAD_ID(y);
+
+      // Compute first residual
+      if (IT_MODE)
+      {
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, u, r, d1d, q1d);
+         DGMassAxpy(e, NE, ND, 1.0, b, -1.0, r, r); // r = b - r
+      }
+      else
+      {
+         // if not in iterative mode, use zero initial guess
+         const int BX = MFEM_THREAD_SIZE(x);
+         const int BY = MFEM_THREAD_SIZE(y);
+         const int bxy = BX*BY;
+         const auto B = ConstDeviceMatrix(b, ND, NE);
+         auto U = DeviceMatrix(u, ND, NE);
+         auto R = DeviceMatrix(r, ND, NE);
+         for (int i = tid; i < ND; i += bxy)
+         {
+            U(i, e) = 0.0;
+            R(i, e) = B(i, e);
+         }
+         MFEM_SYNC_THREAD;
+      }
+
+      DGMassPreconditioner(e, NE, ND, dinv, r, z);
+      DGMassAxpy(e, NE, ND, 1.0, z, 0.0, z, d); // d = z
+
+      double nom = DGMassDot<NB>(e, NE, ND, d, r);
+      if (nom < 0.0) { return; /* Not positive definite */ }
+      double r0 = fmax(nom*RELTOL*RELTOL, ABSTOL*ABSTOL);
+      if (nom <= r0) { return; /* Converged */ }
+
+      DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d);
+      double den = DGMassDot<NB>(e, NE, ND, z, d);
+      if (den <= 0.0)
+      {
+         DGMassDot<NB>(e, NE, ND, d, d);
+         // d2 > 0 => not positive definite
+         if (den == 0.0) { return; }
+      }
+
+      // start iteration
+      int i = 1;
+      while (true)
+      {
+         const double alpha = nom/den;
+         DGMassAxpy(e, NE, ND, 1.0, u, alpha, d, u); // u = u + alpha*d
+         DGMassAxpy(e, NE, ND, 1.0, r, -alpha, z, r); // r = r - alpha*A*d
+
+         DGMassPreconditioner(e, NE, ND, dinv, r, z);
+
+         double betanom = DGMassDot<NB>(e, NE, ND, r, z);
+         if (betanom < 0.0) { return; /* Not positive definite */ }
+         if (betanom <= r0) { break; /* Converged */ }
+
+         if (++i > MAXIT) { break; }
+
+         const double beta = betanom/nom;
+         DGMassAxpy(e, NE, ND, 1.0, z, beta, d, d); // d = z + beta*d
+         DGMassApply<DIM,D1D,Q1D>(e, NE, B, Bt, pa_data, d, z, d1d, q1d); // z = A d
+         den = DGMassDot<NB>(e, NE, ND, d, z);
+         if (den <= 0.0)
+         {
+            DGMassDot<NB>(e, NE, ND, d, d);
+            // d2 > 0 => not positive definite
+            if (den == 0.0) { break; }
+         }
+         nom = betanom;
+      }
+
+      if (CHANGE_BASIS)
+      {
+         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_B, u, u, d1d);
+      }
+   });
+}
+
+void DGMassInverse::Mult(const Vector &Mu, Vector &u) const
+{
+   // Dispatch to templated version based on dim, d1d, and q1d.
+   const int dim = fes.GetMesh()->Dimension();
+   const int d1d = m->dofs1D;
+   const int q1d = m->quad1D;
+
+   const int id = (d1d << 4) | q1d;
+
+   if (dim == 2)
+   {
+      switch (id)
+      {
+         case 0x11: return DGMassCGIteration<2,1,1>(Mu, u);
+         case 0x22: return DGMassCGIteration<2,2,2>(Mu, u);
+         case 0x33: return DGMassCGIteration<2,3,3>(Mu, u);
+         case 0x35: return DGMassCGIteration<2,3,5>(Mu, u);
+         case 0x44: return DGMassCGIteration<2,4,4>(Mu, u);
+         case 0x46: return DGMassCGIteration<2,4,6>(Mu, u);
+         case 0x55: return DGMassCGIteration<2,5,5>(Mu, u);
+         case 0x57: return DGMassCGIteration<2,5,7>(Mu, u);
+         case 0x66: return DGMassCGIteration<2,6,6>(Mu, u);
+         case 0x68: return DGMassCGIteration<2,6,8>(Mu, u);
+         default: return DGMassCGIteration<2>(Mu, u); // Fallback
+      }
+   }
+   else if (dim == 3)
+   {
+      switch (id)
+      {
+         case 0x22: return DGMassCGIteration<3,2,2>(Mu, u);
+         case 0x23: return DGMassCGIteration<3,2,3>(Mu, u);
+         case 0x33: return DGMassCGIteration<3,3,3>(Mu, u);
+         case 0x34: return DGMassCGIteration<3,3,4>(Mu, u);
+         case 0x35: return DGMassCGIteration<3,3,5>(Mu, u);
+         case 0x44: return DGMassCGIteration<3,4,4>(Mu, u);
+         case 0x45: return DGMassCGIteration<3,4,5>(Mu, u);
+         case 0x46: return DGMassCGIteration<3,4,6>(Mu, u);
+         case 0x48: return DGMassCGIteration<3,4,8>(Mu, u);
+         case 0x55: return DGMassCGIteration<3,5,5>(Mu, u);
+         case 0x56: return DGMassCGIteration<3,5,6>(Mu, u);
+         case 0x57: return DGMassCGIteration<3,5,7>(Mu, u);
+         case 0x58: return DGMassCGIteration<3,5,8>(Mu, u);
+         case 0x66: return DGMassCGIteration<3,6,6>(Mu, u);
+         case 0x67: return DGMassCGIteration<3,6,7>(Mu, u);
+         default: return DGMassCGIteration<3>(Mu, u); // Fallback
+      }
+   }
+}
+
+} // namespace mfem
@@ -0,0 +1,112 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DGMASSINV_HPP
+#define MFEM_DGMASSINV_HPP
+
+#include "../linalg/operator.hpp"
+#include "fespace.hpp"
+
+namespace mfem
+{
+
+/// @brief Solver for the discontinuous Galerkin mass matrix.
+///
+/// This class performs a @a local (diagonally preconditioned) conjugate
+/// gradient iteration for each element. Optionally, a change of basis is
+/// performed to iterate on a better-conditioned system. This class fully
+/// supports execution on device (GPU).
+class DGMassInverse : public Solver
+{
+protected:
+   DG_FECollection fec; ///< FE collection in requested basis.
+   FiniteElementSpace fes; ///< FE space in requested basis.
+   const DofToQuad *d2q; ///< Change of basis. Not owned.
+   Array<double> B_; ///< Inverse of change of basis.
+   Array<double> Bt_; ///< Inverse of change of basis, transposed.
+   class BilinearForm *M; ///< Mass bilinear form, owned.
+   class MassIntegrator *m; ///< Mass integrator, owned by the form @ref M.
+   Vector diag_inv; ///< Jacobi preconditioner.
+   double rel_tol = 1e-12; ///< Relative CG tolerance.
+   double abs_tol = 1e-12; ///< Absolute CG tolerance.
+   int max_iter = 100; ///< Maximum number of CG iterations;
+
+   /// @name Intermediate vectors needed for CG three-term recurrence.
+   ///@{
+   mutable Vector r_, d_, z_, b2_;
+   ///@}
+
+   /// @brief Protected constructor, used internally.
+   ///
+   /// Custom coefficient and integration rule are used if @a coeff and @a ir
+   /// are non-NULL.
+   DGMassInverse(FiniteElementSpace &fes_, Coefficient *coeff,
+                 const IntegrationRule *ir, int btype);
+public:
+   /// @brief Construct the DG inverse mass operator for @a fes_.
+   ///
+   /// The basis type @a btype determines which basis should be used internally
+   /// in the solver. This <b>does not</b> have to be the same basis as @a fes_.
+   /// The best choice is typically BasisType::GaussLegendre because it is
+   /// well-preconditioned by its diagonal.
+   ///
+   /// The solution and right-hand side used for the solver are not affected by
+   /// this basis (they correspond to the basis of @a fes_). @a btype is only
+   /// used internally, and only has an effect on the convergence rate.
+   DGMassInverse(FiniteElementSpace &fes_, int btype=BasisType::GaussLegendre);
+   /// @brief Construct the DG inverse mass operator for @a fes_ with
+   /// Coefficient @a coeff.
+   ///
+   /// @sa DGMassInverse(FiniteElementSpace&, int) for information about @a
+   /// btype.
+   DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                 int btype=BasisType::GaussLegendre);
+   /// @brief Construct the DG inverse mass operator for @a fes_ with
+   /// Coefficient @a coeff and IntegrationRule @a ir.
+   ///
+   /// @sa DGMassInverse(FiniteElementSpace&, int) for information about @a
+   /// btype.
+   DGMassInverse(FiniteElementSpace &fes_, Coefficient &coeff,
+                 const IntegrationRule &ir, int btype=BasisType::GaussLegendre);
+   /// @brief Construct the DG inverse mass operator for @a fes_ with
+   /// IntegrationRule @a ir.
+   ///
+   /// @sa DGMassInverse(FiniteElementSpace&, int) for information about @a
+   /// btype.
+   DGMassInverse(FiniteElementSpace &fes_, const IntegrationRule &ir,
+                 int btype=BasisType::GaussLegendre);
+   /// @brief Solve the system M b = u.
+   ///
+   /// If @ref iterative_mode is @a true, @a u is used as an initial guess.
+   void Mult(const Vector &b, Vector &u) const;
+   /// Not implemented. Aborts.
+   void SetOperator(const Operator &op);
+   /// Set the relative tolerance.
+   void SetRelTol(const double rel_tol_);
+   /// Set the absolute tolerance.
+   void SetAbsTol(const double abs_tol_);
+   /// Set the maximum number of iterations.
+   void SetMaxIter(const double max_iter_);
+   /// Recompute operator and preconditioner (when coefficient or mesh changes).
+   void Update();
+
+   ~DGMassInverse();
+
+   /// @brief Solve the system M b = u. <b>Not part of the public interface.</b>
+   /// @note This member function must be public because it contains an
+   /// MFEM_FORALL kernel (nvcc limitation)
+   template<int DIM, int D1D = 0, int Q1D = 0>
+   void DGMassCGIteration(const Vector &b_, Vector &u_) const;
+};
+
+} // namespace mfem
+
+#endif
@@ -0,0 +1,295 @@
+// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+// LICENSE and NOTICE for details. LLNL-CODE-806117.
+//
+// This file is part of the MFEM library. For more information and source code
+// availability visit https://mfem.org.
+//
+// MFEM is free software; you can redistribute it and/or modify it under the
+// terms of the BSD-3 license. We welcome feedback and contributions, see file
+// CONTRIBUTING.md for details.
+
+#ifndef MFEM_DGMASSINV_KERNELS_HPP
+#define MFEM_DGMASSINV_KERNELS_HPP
+
+#include "bilininteg_mass_pa.hpp"
+#include "../linalg/kernels.hpp"
+#include "kernels.hpp"
+
+namespace mfem
+{
+
+namespace internal
+{
+
+void MakeReciprocal(int n, double *x)
+{
+   MFEM_FORALL(i, n, x[i] = 1.0/x[i]; );
+}
+
+template <int DIM, int D1D, int Q1D>
+MFEM_HOST_DEVICE inline
+void DGMassApply(const int e,
+                 const int NE,
+                 const double *B,
+                 const double *Bt,
+                 const double *pa_data,
+                 const double *x,
+                 double *y,
+                 const int d1d = 0,
+                 const int q1d = 0)
+{
+   constexpr bool use_smem = (D1D > 0 && Q1D > 0);
+   constexpr bool ACCUM = false;
+   constexpr int NBZ = 1;
+   if (use_smem)
+   {
+      // cannot specialize functions below with D1D or Q1D equal to zero
+      // (this branch only runs with D1D and Q1D are both positive)
+      constexpr int TD1D = D1D ? D1D : 1;
+      constexpr int TQ1D = Q1D ? Q1D : 1;
+      if (DIM == 2)
+      {
+         SmemPAMassApply2D_Element<TD1D,TQ1D,NBZ,ACCUM>(e, NE, B, pa_data, x, y);
+      }
+      else if (DIM == 3)
+      {
+         SmemPAMassApply3D_Element<TD1D,TQ1D,ACCUM>(e, NE, B, pa_data, x, y);
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("Unsupported dimension.");
+      }
+   }
+   else
+   {
+      if (DIM == 2)
+      {
+         PAMassApply2D_Element<ACCUM>(e, NE, B, Bt, pa_data, x, y, d1d, q1d);
+      }
+      else if (DIM == 3)
+      {
+         PAMassApply3D_Element<ACCUM>(e, NE, B, Bt, pa_data, x, y, d1d, q1d);
+      }
+      else
+      {
+         MFEM_ABORT_KERNEL("Unsupported dimension.");
+      }
+   }
+}
+
+MFEM_HOST_DEVICE inline
+void DGMassPreconditioner(const int e,
+                          const int NE,
+                          const int ND,
+                          const double *dinv,
+                          const double *x,
+                          double *y)
+{
+   const auto X = ConstDeviceMatrix(x, ND, NE);
+   const auto D = ConstDeviceMatrix(dinv, ND, NE);
+   auto Y = DeviceMatrix(y, ND, NE);
+
+   const int tid = MFEM_THREAD_ID(x) + MFEM_THREAD_SIZE(x)*MFEM_THREAD_ID(y);
+   const int bxy = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
+
+   for (int i = tid; i < ND; i += bxy)
+   {
+      Y(i, e) = D(i, e)*X(i, e);
+   }
+   MFEM_SYNC_THREAD;
+}
+
+MFEM_HOST_DEVICE inline
+void DGMassAxpy(const int e,
+                const int NE,
+                const int ND,
+                const double a,
+                const double *x,
+                const double b,
+                const double *y,
+                double *z)
+{
+   const auto X = ConstDeviceMatrix(x, ND, NE);
+   const auto Y = ConstDeviceMatrix(y, ND, NE);
+   auto Z = DeviceMatrix(z, ND, NE);
+
+   const int tid = MFEM_THREAD_ID(x) + MFEM_THREAD_SIZE(x)*MFEM_THREAD_ID(y);
+   const int bxy = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
+
+   for (int i = tid; i < ND; i += bxy)
+   {
+      Z(i, e) = a*X(i, e) + b*Y(i, e);
+   }
+   MFEM_SYNC_THREAD;
+}
+
+template <int NB>
+MFEM_HOST_DEVICE inline
+double DGMassDot(const int e,
+                 const int NE,
+                 const int ND,
+                 const double *x,
+                 const double *y)
+{
+   const auto X = ConstDeviceMatrix(x, ND, NE);
+   const auto Y = ConstDeviceMatrix(y, ND, NE);
+
+   const int tid = MFEM_THREAD_ID(x) + MFEM_THREAD_SIZE(x)*MFEM_THREAD_ID(y);
+   const int bxy = MFEM_THREAD_SIZE(x)*MFEM_THREAD_SIZE(y);
+
+   MFEM_SHARED double s_dot[NB*NB];
+   s_dot[tid] = 0.0;
+
+   for (int i = tid; i < ND; i += bxy) { s_dot[tid] += X(i,e)*Y(i,e); }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 512 && tid + 512 < bxy) { s_dot[tid] += s_dot[tid + 512]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 256 && tid < 256 && tid + 256 < bxy) { s_dot[tid] += s_dot[tid + 256]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 128 && tid < 128 && tid + 128 < bxy) { s_dot[tid] += s_dot[tid + 128]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 64 && tid < 64 && tid + 64 < bxy) { s_dot[tid] += s_dot[tid + 64]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 32 && tid < 32 && tid + 32 < bxy) { s_dot[tid] += s_dot[tid + 32]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 16 && tid < 16 && tid + 16 < bxy) { s_dot[tid] += s_dot[tid + 16]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 8 && tid < 8 && tid + 8 < bxy) { s_dot[tid] += s_dot[tid + 8]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 4 && tid < 4 && tid + 4 < bxy) { s_dot[tid] += s_dot[tid + 4]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 2 && tid < 2 && tid + 2 < bxy) { s_dot[tid] += s_dot[tid + 2]; }
+   MFEM_SYNC_THREAD;
+
+   if (bxy > 1 && tid < 1 && tid + 1 < bxy) { s_dot[tid] += s_dot[tid + 1]; }
+   MFEM_SYNC_THREAD;
+
+   return s_dot[0];
+}
+
+template<int T_D1D = 0, int MAX_D1D = 0>
+MFEM_HOST_DEVICE inline
+void DGMassBasis2D(const int e,
+                   const int NE,
+                   const double *b_,
+                   const double *x_,
+                   double *y_,
+                   const int d1d = 0)
+{
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   const int D1D = T_D1D ? T_D1D : d1d;
+
+   const auto b = Reshape(b_, D1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, NE);
+   auto y = Reshape(y_, D1D, D1D, NE);
+
+   MFEM_SHARED double sB[MD1*MD1];
+   MFEM_SHARED double sm0[MD1*MD1];
+   MFEM_SHARED double sm1[MD1*MD1];
+
+   kernels::internal::LoadB<MD1,MD1>(D1D,D1D,b,sB);
+
+   ConstDeviceMatrix B(sB, D1D,D1D);
+   DeviceMatrix DD(sm0, MD1, MD1);
+   DeviceMatrix DQ(sm1, MD1, MD1);
+   DeviceMatrix QQ(sm0, MD1, MD1);
+
+   kernels::internal::LoadX(e,D1D,x,DD);
+   kernels::internal::EvalX(D1D,D1D,B,DD,DQ);
+   kernels::internal::EvalY(D1D,D1D,B,DQ,QQ);
+   MFEM_SYNC_THREAD; // sync here to allow in-place evaluations
+   MFEM_FOREACH_THREAD(qy,y,D1D)
+   {
+      MFEM_FOREACH_THREAD(qx,x,D1D)
+      {
+         y(qx,qy,e) = QQ(qx,qy);
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+template<int T_D1D = 0, int MAX_D1D = 0>
+MFEM_HOST_DEVICE inline
+void DGMassBasis3D(const int e,
+                   const int NE,
+                   const double *b_,
+                   const double *x_,
+                   double *y_,
+                   const int d1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+
+   const auto b = Reshape(b_, D1D, D1D);
+   const auto x = Reshape(x_, D1D, D1D, D1D, NE);
+   auto y = Reshape(y_, D1D, D1D, D1D, NE);
+
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+
+   MFEM_SHARED double sB[MD1*MD1];
+   MFEM_SHARED double sm0[MD1*MD1*MD1];
+   MFEM_SHARED double sm1[MD1*MD1*MD1];
+
+   kernels::internal::LoadB<MD1,MD1>(D1D,D1D,b,sB);
+
+   ConstDeviceMatrix B(sB, D1D,D1D);
+   DeviceCube DDD(sm0, MD1,MD1,MD1);
+   DeviceCube DDQ(sm1, MD1,MD1,MD1);
+   DeviceCube DQQ(sm0, MD1,MD1,MD1);
+   DeviceCube QQQ(sm1, MD1,MD1,MD1);
+
+   kernels::internal::LoadX(e,D1D,x,DDD);
+   kernels::internal::EvalX(D1D,D1D,B,DDD,DDQ);
+   kernels::internal::EvalY(D1D,D1D,B,DDQ,DQQ);
+   kernels::internal::EvalZ(D1D,D1D,B,DQQ,QQQ);
+   MFEM_SYNC_THREAD; // sync here to allow in-place evaluation
+   MFEM_FOREACH_THREAD(qz,z,D1D)
+   {
+      MFEM_FOREACH_THREAD(qy,y,D1D)
+      {
+         for (int qx = 0; qx < D1D; ++qx)
+         {
+            y(qx,qy,qz,e) = QQQ(qz,qy,qx);
+         }
+      }
+   }
+   MFEM_SYNC_THREAD;
+}
+
+template<int DIM, int T_D1D = 0, int MAX_D1D = 0>
+MFEM_HOST_DEVICE inline
+void DGMassBasis(const int e,
+                 const int NE,
+                 const double *b_,
+                 const double *x_,
+                 double *y_,
+                 const int d1d = 0)
+{
+   if (DIM == 2)
+   {
+      DGMassBasis2D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
+   }
+   else if (DIM == 3)
+   {
+      DGMassBasis3D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
+   }
+   else
+   {
+      MFEM_ABORT_KERNEL("Dimension not supported.");
+   }
+}
+
+} // namespace internal
+
+} // namespace mfem
+
+#endif
@@ -53,8 +53,12 @@ public:

   virtual int DofForGeometry(Geometry::Type GeomType) const = 0;

-   /** @brief Returns an array, say p, that maps a local permuted index i to
-       a local base index: base_i = p[i]. */
+   /** @brief Returns an array, say p, that maps a local permuted index i to a
+       local base index: base_i = p[i].
+
+       @note Only provides information about interior dofs. See
+       FiniteElementCollection::SubDofOrder if interior \a and boundary dof
+       order is needed. */
   virtual const int *DofOrderForOrientation(Geometry::Type GeomType,
                                             int Or) const = 0;

@@ -95,10 +99,10 @@ public:
   | RT_ValTrace_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
   | RT_Trace@[BTYPE]_[DIM]_[ORDER] | H^{1/2} | * | 1 / 0 | INTEGRAL | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
   | RT_ValTrace@[BTYPE]_[DIM]_[ORDER] |  H^{1/2} | * | 1 / 0 | VALUE | H^{1/2}-conforming trace elements for H(div) defined on the interface between mesh elements (faces) |
-   | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinous L2 elements |
-   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinous L2 elements |
-   | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinous L2 elements |
-   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinous L2 elements |
+   | L2_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
+   | L2_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | VALUE | Discontinuous L2 elements |
+   | L2Int_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
+   | L2Int_T[BTYPE]_[DIM]_[ORDER] | L2 | * | 0 | INTEGRAL | Discontinuous L2 elements |
   | DG_Iface_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
   | DG_Iface@[BTYPE]_[DIM]_[ORDER] | - | * | 0 | VALUE | Discontinuous elements on the interface between mesh elements (faces) |
   | DG_IntIface_[DIM]_[ORDER] | - | * | 0 | INTEGRAL | Discontinuous elements on the interface between mesh elements (faces) |
@@ -45,6 +45,7 @@
 #include "multigrid.hpp"
 #include "ceed/solvers/algebraic.hpp"
 #include "lor/lor.hpp"
+#include "dgmassinv.hpp"

 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"
--- a/Show More
+++ b/Show More