alternative preconditioner

add mesh file
minor
2024-12-04 10:03:43 -05:00 · 2024-12-04 09:40:55 -05:00 · 2024-01-22 10:52:39 -08:00 · 2024-01-12 13:07:25 -08:00 · 2024-01-12 13:06:59 -08:00 · 2024-01-04 16:09:52 -08:00
341 changed files with 145804 additions and 8221 deletions
@@ -1,7 +1,6 @@
 name: "Docker"

 on:
-
  # Always have a base image ready to go - this is a nightly build
  schedule:
    - cron: 0 3 * * *
@@ -26,7 +25,6 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-
        # Dockerfiles to build, a matrix supports future expanded builds
        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
@@ -34,15 +32,20 @@ jobs:
    runs-on: ubuntu-latest
    name: Build
    steps:
+      - name: Run Actions Cleaner
+        uses: easimon/maximize-build-space@v8
+        with:
+          overprovision-lvm: 'true'
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+
      - name: Checkout
        uses: actions/checkout@v3

-      - name: Make Space For Build
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-
-       # It's easier to reference named variables than indexes of the matrix
+      # It's easier to reference named variables than indexes of the matrix
      - name: Set Environment
        env:
          dockerfile: ${{ matrix.container[0] }}
@@ -65,7 +65,7 @@ jobs:
        # - Add a new combination.
        #   'build-system: cmake' and 'hypre-target: int64'
        #
-        # note: we will gather coverage info for any non-debug run except the
+        # Note: we will gather coverage info for any non-debug run except the
        # CMake build.
        include:
          - target: dbg
@@ -101,181 +101,189 @@ jobs:
    runs-on: ${{ matrix.os }}

    steps:
-    # This external action allows to interrupt a workflow already running on
-    # the same branch to save resource
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
-      with:
-        access_token: ${{ github.token }}
+      # This external action allows to interrupt a workflow already running on
+      # the same branch to save resources.
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}

-    # Checkout MFEM in "mfem" subdirectory. Final path:
-    # /home/runner/work/mfem/mfem/mfem
-    # Note: Done now to access "install-hypre" and "install-metis" actions.
-    - name: checkout mfem
-      uses: actions/checkout@v3
-      with:
-        path: ${{ env.MFEM_TOP_DIR }}
-        # Fetch the complete history for codecov to access commits ID
-        fetch-depth: 0
+      # Fix 'No space left on device' errors for Ubuntu builds.
+      - name: Run Actions Cleaner
+        if: matrix.os == 'ubuntu-latest'
+        uses: easimon/maximize-build-space@v8
+        with:
+          overprovision-lvm: 'true'
+          remove-android: 'true'

-    # Only get MPI if defined for the job.
-    # TODO: It would be nice to have only one step, e.g. with a dedicated
-    # action, but I (@adrienbernede) don't see how at the moment.
-    - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
-      run: |
-        sudo apt-get install mpich libmpich-dev
+      # Checkout MFEM in "mfem" subdirectory. Final path:
+      # /home/runner/work/mfem/mfem/mfem
+      # Note: Done now to access "install-hypre" and "install-metis" actions.
+      - name: checkout mfem
+        uses: actions/checkout@v3
+        with:
+          path: ${{ env.MFEM_TOP_DIR }}
+          # Fetch the complete history for codecov to access commits ID
+          fetch-depth: 0

-    - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
-      run: |
-        sudo apt-get install lcov
+      # Only get MPI if defined for the job.
+      # TODO: It would be nice to have only one step, e.g. with a dedicated
+      # action, but I (@adrienbernede) don't see how at the moment.
+      - name: get MPI (Linux)
+        if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
+        run: |
+          sudo apt-get install mpich libmpich-dev

-    # Keep the following section in case we need it again in the future,
-    # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-    # - name: Set up Homebrew
-    #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-    #   uses: Homebrew/actions/setup-homebrew@master
+      - name: get lcov (Linux)
+        if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
+        run: |
+          sudo apt-get install lcov

-    - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install openmpi
+      # Keep the following section in case we need it again in the future,
+      # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
+      # - name: Set up Homebrew
+      #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
+      #   uses: Homebrew/actions/setup-homebrew@master

-    - name: get lcov (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
-      run: |
-        export HOMEBREW_NO_INSTALL_CLEANUP=1
-        brew install lcov
+      - name: get MPI (MacOS)
+        if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
+        run: |
+          export HOMEBREW_NO_INSTALL_CLEANUP=1
+          brew install openmpi

-    - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-      uses: mpi4py/setup-mpi@v1.1.4
+      - name: get lcov (MacOS)
+        if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
+        run: |
+          export HOMEBREW_NO_INSTALL_CLEANUP=1
+          brew install lcov

-    # Get Hypre through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache hypre
-      id: hypre-cache
-      if: matrix.mpi == 'par'
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+      - name: get MPI (Windows)
+        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+        uses: mpi4py/setup-mpi@v1.1.4

-    - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.4
-      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: ${{ matrix.hypre-target }}
-        build-system: make
+      # Get Hypre through cache, or build it.
+      # Install will only run on cache miss.
+      - name: cache hypre
+        id: hypre-cache
+        if: matrix.mpi == 'par'
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.HYPRE_TOP_DIR }}
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2

-    - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-      uses: mfem/github-actions/build-hypre@v2.4
-      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: ${{ matrix.hypre-target }}
-        build-system: cmake
+      - name: get hypre
+        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
+        uses: mfem/github-actions/build-hypre@v2.4
+        with:
+          archive: ${{ env.HYPRE_ARCHIVE }}
+          dir: ${{ env.HYPRE_TOP_DIR }}
+          target: ${{ matrix.hypre-target }}
+          build-system: make

-    # Get Metis through cache, or build it.
-    # Install will only run on cache miss.
-    - name: cache metis
-      id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+      - name: get hypre (Windows)
+        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
+        uses: mfem/github-actions/build-hypre@v2.4
+        with:
+          archive: ${{ env.HYPRE_ARCHIVE }}
+          dir: ${{ env.HYPRE_TOP_DIR }}
+          target: ${{ matrix.hypre-target }}
+          build-system: cmake

-    - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.4
-      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
+      # Get Metis through cache, or build it.
+      # Install will only run on cache miss.
+      - name: cache metis
+        id: metis-cache
+        if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.METIS_TOP_DIR }}
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

-    - name: cache vcpkg (Windows)
-      id: vcpkg-cache
-      if: matrix.os == 'windows-latest'
-      uses: actions/cache@v3
-      with:
-        path: vcpkg_cache
-        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
+      - name: install metis
+        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-metis@v2.4
+        with:
+          archive: ${{ env.METIS_ARCHIVE }}
+          dir: ${{ env.METIS_TOP_DIR }}

-    - name: prepare vcpkg binary cache location (Windows)
-      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
-      run: |
-        mkdir -p vcpkg_cache
+      - name: cache vcpkg (Windows)
+        id: vcpkg-cache
+        if: matrix.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: vcpkg_cache
+          key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1

-    - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-      env:
-        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-      run: |
-        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
+      - name: prepare vcpkg binary cache location (Windows)
+        if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p vcpkg_cache

-    # MFEM build and test
-    - name: build
-      uses: mfem/github-actions/build-mfem@v2.4
-      env:
-        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-      with:
-        os: ${{ matrix.os }}
-        target: ${{ matrix.target }}
-        codecov: ${{ matrix.codecov }}
-        mpi: ${{ matrix.mpi }}
-        build-system: ${{ matrix.build-system }}
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: ${{ env.MFEM_TOP_DIR }}
-        config-options: ${{ matrix.config-opts }}
-        library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
+      - name: install metis (Windows)
+        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+        env:
+          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+        run: |
+          vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports

-    # Run checks (and only checks) on debug targets
-    - name: checks
-      if: matrix.build-system == 'make' && matrix.target == 'dbg'
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }}  && make check
+      # MFEM build and test
+      - name: build
+        uses: mfem/github-actions/build-mfem@v2.4
+        env:
+          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+        with:
+          os: ${{ matrix.os }}
+          target: ${{ matrix.target }}
+          codecov: ${{ matrix.codecov }}
+          mpi: ${{ matrix.mpi }}
+          build-system: ${{ matrix.build-system }}
+          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+          metis-dir: ${{ env.METIS_TOP_DIR }}
+          mfem-dir: ${{ env.MFEM_TOP_DIR }}
+          config-options: ${{ matrix.config-opts }}
+          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}

-    # Note: 'tests' include the unit tests
-    - name: tests
-      if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
-      run: |
-        cd ${{ env.MFEM_TOP_DIR }} && make test
+      # Run checks (and only checks) on debug targets
+      - name: checks
+        if: matrix.build-system == 'make' && matrix.target == 'dbg'
+        run: |
+          cd ${{ env.MFEM_TOP_DIR }}  && make check

-    - name: cmake checks
-      if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
-      run: |
-        CTEST_CONFIG="Debug"
-        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
-      shell: bash
+      # Note: 'tests' include the unit tests
+      - name: tests
+        if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
+        run: |
+          cd ${{ env.MFEM_TOP_DIR }} && make test

-    - name: cmake unit tests (Ubuntu)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
-      run: |
-        CTEST_CONFIG="Release"
-        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
-        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
-      shell: bash
+      - name: cmake checks
+        if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
+        run: |
+          CTEST_CONFIG="Debug"
+          cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
+        shell: bash

-    - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
-      run: |
-        CTEST_CONFIG="Release"
-        cd ${{ env.MFEM_TOP_DIR }}/build && \
-        ctest --output-on-failure -C ${CTEST_CONFIG} || \
-        ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
-      shell: bash
+      - name: cmake unit tests (Ubuntu)
+        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
+        run: |
+          CTEST_CONFIG="Release"
+          [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
+          cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
+        shell: bash

-    # Code coverage (process and upload reports)
-    - name: codecov
-      if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.4
-      with:
-        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-        project_dir: ${{ env.MFEM_TOP_DIR }}
-        directories: "fem general linalg mesh"
+      - name: cmake tests
+        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
+        run: |
+          CTEST_CONFIG="Release"
+          cd ${{ env.MFEM_TOP_DIR }}/build && \
+          ctest --output-on-failure -C ${CTEST_CONFIG} || \
+          ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
+        shell: bash
+
+      # Code coverage (process and upload reports)
+      - name: codecov
+        if: matrix.codecov == 'YES'
+        uses: mfem/github-actions/upload-coverage@v2.4
+        with:
+          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+          project_dir: ${{ env.MFEM_TOP_DIR }}
+          directories: "fem general linalg mesh"
@@ -13,10 +13,10 @@ name: "Static Analysis"

 on:
  push:
-    branches: [ "master", "next"]
+    branches: ["master", "next"]
  pull_request:
    # The branches below must be a subset of the branches above
-    branches: [ "master" ]
+    branches: ["master"]

 jobs:
  analyze:
@@ -35,36 +35,35 @@ jobs:
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3

-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.

-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
+          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+          # queries: security-extended,security-and-quality

+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2

-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+      #   If the Autobuild fails above, remove it and uncomment the following three lines.
+      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+      # - run: |
+      #   echo "Run, Build Application using script"
+      #   ./location_of_script_within_repo/buildscript.sh

-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
@@ -34,67 +34,67 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
-      with:
-        access_token: ${{ github.token }}
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}

-    - name: checkout MFEM
-      uses: actions/checkout@v3
-      with:
-        path: mfem
+      - name: checkout MFEM
+        uses: actions/checkout@v3
+        with:
+          path: mfem

-    - name: Get MPI (Linux)
-      run: |
-        sudo apt-get install mpich libmpich-dev
+      - name: Get MPI (Linux)
+        run: |
+          sudo apt-get install mpich libmpich-dev

-    - name: Cache Hypre Install
-      id: hypre-cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.HYPRE_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+      - name: Cache Hypre Install
+        id: hypre-cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.HYPRE_TOP_DIR }}
+          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2

-    - name: Get Hypre
-      if: steps.hypre-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-hypre@v2.4
-      with:
-        archive: ${{ env.HYPRE_ARCHIVE }}
-        dir: ${{ env.HYPRE_TOP_DIR }}
-        target: int32
+      - name: Get Hypre
+        if: steps.hypre-cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-hypre@v2.4
+        with:
+          archive: ${{ env.HYPRE_ARCHIVE }}
+          dir: ${{ env.HYPRE_TOP_DIR }}
+          target: int32

-    - name: Cache Metis Install
-      id: metis-cache
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.METIS_TOP_DIR }}
-        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+      - name: Cache Metis Install
+        id: metis-cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.METIS_TOP_DIR }}
+          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

-    - name: Install Metis
-      if: steps.metis-cache.outputs.cache-hit != 'true'
-      uses: mfem/github-actions/build-metis@v2.4
-      with:
-        archive: ${{ env.METIS_ARCHIVE }}
-        dir: ${{ env.METIS_TOP_DIR }}
+      - name: Install Metis
+        if: steps.metis-cache.outputs.cache-hit != 'true'
+        uses: mfem/github-actions/build-metis@v2.4
+        with:
+          archive: ${{ env.METIS_ARCHIVE }}
+          dir: ${{ env.METIS_TOP_DIR }}

-    # MFEM build and test
-    - name: build-mfem
-      uses: mfem/github-actions/build-mfem@v2.4
-      with:
-        os: ${{ runner.os }}
-        target: opt
-        codecov: NO
-        mpi: par
-        build-system: make
-        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-        metis-dir: ${{ env.METIS_TOP_DIR }}
-        mfem-dir: mfem
+      # MFEM build and test
+      - name: build-mfem
+        uses: mfem/github-actions/build-mfem@v2.4
+        with:
+          os: ${{ runner.os }}
+          target: opt
+          codecov: NO
+          mpi: par
+          build-system: make
+          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+          metis-dir: ${{ env.METIS_TOP_DIR }}
+          mfem-dir: mfem

-    - name: test (no clean)
-      run: |
-        cd mfem && make test-noclean
+      - name: test (no clean)
+        run: |
+          cd mfem && make test-noclean

-    - name: gitignore
-      run: |
-        cd mfem/tests/scripts
-        ./runtest gitignore
+      - name: gitignore
+        run: |
+          cd mfem/tests/scripts
+          ./runtest gitignore
@@ -27,44 +27,44 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
-      with:
-        access_token: ${{ github.token }}
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}

-    - name: MFEM Checkout
-      uses: actions/checkout@v3
-      with:
-        path: mfem
+      - name: MFEM Checkout
+        uses: actions/checkout@v3
+        with:
+          path: mfem

-    - name: MFEM Build
-      uses: mfem/github-actions/build-mfem@v2.4
-      with:
-        os: ${{ runner.os }}
-        target: opt
-        mpi: seq
-        hypre-dir: unused-hypre-dir
-        metis-dir: unused-metis-dir
-        mfem-dir: mfem
-        build-system: make
-        library-only: false
-        config-options:
-          CXX="clang++-14"
-          CXXFLAGS="-g -O1 -std=c++11
-                    -fsanitize=address
-                    -fno-omit-frame-pointer
-                    -fsanitize-address-use-after-scope"
+      - name: MFEM Build
+        uses: mfem/github-actions/build-mfem@v2.4
+        with:
+          os: ${{ runner.os }}
+          target: opt
+          mpi: seq
+          hypre-dir: unused-hypre-dir
+          metis-dir: unused-metis-dir
+          mfem-dir: mfem
+          build-system: make
+          library-only: false
+          config-options:
+            CXX="clang++-14"
+            CXXFLAGS="-g -O1 -std=c++11
+                      -fsanitize=address
+                      -fno-omit-frame-pointer
+                      -fsanitize-address-use-after-scope"

-    - name: MFEM Info
-      working-directory: mfem
-      run: make info
+      - name: MFEM Info
+        working-directory: mfem
+        run: make info

-    - name: MFEM Sanitize
-      working-directory: mfem
-      run:
-        ASAN_OPTIONS="detect_leaks=1,
-                      strict_init_order=1,
-                      strict_string_checks=1,
-                      check_initialization_order=1,
-                      detect_stack_use_after_return=1"
-        make test
+      - name: MFEM Sanitize
+        working-directory: mfem
+        run:
+          ASAN_OPTIONS="detect_leaks=1,
+                        strict_init_order=1,
+                        strict_string_checks=1,
+                        check_initialization_order=1,
+                        detect_stack_use_after_return=1"
+          make test
@@ -33,49 +33,49 @@ jobs:
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
-      with:
-        access_token: ${{ github.token }}
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}

-    - name: checkout mfem
-      uses: actions/checkout@v3
+      - name: checkout mfem
+        uses: actions/checkout@v3

-    - name: copyright check
-      id: copyright
-      run: |
-        ./config/githooks/pre-push --copyright
+      - name: copyright check
+        id: copyright
+        run: |
+          ./config/githooks/pre-push --copyright

-      continue-on-error: true
+        continue-on-error: true

-    - name: license check
-      id: license
-      run: |
-        ./config/githooks/pre-push --license
-      continue-on-error: true
+      - name: license check
+        id: license
+        run: |
+          ./config/githooks/pre-push --license
+        continue-on-error: true

-    - name: release check
-      id: release
-      run: |
-        ./config/githooks/pre-push --release
-      continue-on-error: true
+      - name: release check
+        id: release
+        run: |
+          ./config/githooks/pre-push --release
+        continue-on-error: true

-    - name: wrap-up
-      if: |
-        steps.copyright.outcome != 'success' ||
-        steps.license.outcome != 'success' ||
-        steps.release.outcome != 'success'
-      run: |
-        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-          echo "copyright check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-          echo "license check failed, unroll log for details"
-        fi
-        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-          echo "release check failed, unroll log for details"
-        fi
-        exit 1
+      - name: wrap-up
+        if: |
+          steps.copyright.outcome != 'success' ||
+          steps.license.outcome != 'success' ||
+          steps.release.outcome != 'success'
+        run: |
+          if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
+            echo "copyright check failed, unroll log for details"
+          fi
+          if [[ "${{ steps.license.outcome }}" != "success" ]]; then
+            echo "license check failed, unroll log for details"
+          fi
+          if [[ "${{ steps.release.outcome }}" != "success" ]]; then
+            echo "release check failed, unroll log for details"
+          fi
+          exit 1

  code-style:
    runs-on: ubuntu-latest
@@ -83,16 +83,16 @@ jobs:
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v3
+      - name: checkout mfem
+        uses: actions/checkout@v3

-    - name: get astyle
-      run: |
-        sudo apt-get install astyle
+      - name: get astyle
+        run: |
+          sudo apt-get install astyle

-    - name: style check
-      run: |
-        ./config/githooks/pre-push --style
+      - name: style check
+        run: |
+          ./config/githooks/pre-push --style

  documentation:
    runs-on: ubuntu-latest
@@ -100,22 +100,22 @@ jobs:
      (github.event_name == 'push' ||
       github.event.pull_request.head.repo.full_name != github.repository)
    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v3
+      - name: checkout mfem
+        uses: actions/checkout@v3

-    - name: get doxygen and graphviz
-      run: |
-        sudo apt-get install doxygen graphviz
+      - name: get doxygen and graphviz
+        run: |
+          sudo apt-get install doxygen graphviz

-    - name: update doxygen config file
-      run: |
-        cd doc
-        doxygen -u CodeDocumentation.conf.in
+      - name: update doxygen config file
+        run: |
+          cd doc
+          doxygen -u CodeDocumentation.conf.in

-    - name: build documentation
-      run: |
-        cd tests/scripts
-        ./runtest documentation
+      - name: build documentation
+        run: |
+          cd tests/scripts
+          ./runtest documentation

  branch-history:
    if: |
@@ -125,16 +125,16 @@ jobs:
       github.event.pull_request.head.repo.full_name != github.repository)
    runs-on: ubuntu-latest
    steps:
-    - name: checkout mfem
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
+      - name: checkout mfem
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0

-    - name: branch-history
-      run: |
-        # We override origin to make sure we point to the main repo.
-        # This is to have consistent test results on PRs from forks.
-        git remote remove origin
-        git remote add origin https://github.com/mfem/mfem.git
-        git checkout -b gh-actions-branch-history
-        ./config/githooks/pre-push --history
+      - name: branch-history
+        run: |
+          # We override origin to make sure we point to the main repo.
+          # This is to have consistent test results on PRs from forks.
+          git remote remove origin
+          git remote add origin https://github.com/mfem/mfem.git
+          git checkout -b gh-actions-branch-history
+          ./config/githooks/pre-push --history
@@ -128,6 +128,13 @@ examples/amgx/sol.gf
 examples/amgx/mesh.*
 examples/amgx/sol.*

+examples/caliper/ex1
+examples/caliper/ex1p
+examples/caliper/refined.mesh
+examples/caliper/sol.gf
+examples/caliper/mesh.*
+examples/caliper/sol.*
+
 examples/ginkgo/ex1
 examples/ginkgo/refined.mesh
 examples/ginkgo/sol.gf
@@ -266,6 +273,7 @@ miniapps/navier/*_output
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex11p
+miniapps/nurbs/nurbs_printfunc
 miniapps/nurbs/nurbs_patch_ex1
 miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
@@ -275,6 +283,10 @@ miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
 miniapps/nurbs/sin-fit.mesh
 miniapps/nurbs/CurveInt
+miniapps/nurbs/nurbs_naca_cmesh
+miniapps/nurbs/naca-cmesh.mesh
+miniapps/nurbs/glvis_naca-cmesh.mesh
+miniapps/nurbs/Naca_cmesh

 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -299,8 +311,12 @@ miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/plor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric
+miniapps/tools/tmop-check-metric
 miniapps/tools/tmop-metric-magnitude
+miniapps/tools/nodal-transfer
+miniapps/tools/ParaView
+miniapps/tools/gridfunc_*
+miniapps/tools/mesh_*

 miniapps/toys/automata
 miniapps/toys/life
@@ -8,117 +8,164 @@
                               https://mfem.org


-Version 4.5.3 (development)
+Version 4.6.1 (development)
 ===========================
- Added curve interpolation method for NURBS.
+
+Discretization improvements
+---------------------------
+- Introduced support for higher order non conformal Nedelec elements on
+  simplices in ParMesh.
+- Introduced support for internal boundary elements in nonconformal adapted
+  meshes.
+
+- Added functionality for construction of cut-surface and cut-volume
+  IntegrationRules through a moment-fitting approach. The cut is specified by
+  the zero level set of a Coefficient. See fem/intrules_cut.hpp and Example 38.
+
+Miscellaneous
+-------------
+- The ReadCubit Genesis mesh importer has been rewritten to improve readability.
+
+- Updated the Doxygen documentation style, which now requires Doxygen version
+  1.9.8 or later. See the doc/ directory.
+
+- Improved thread safety for global variables in the library, for example
+  IntegrationRules IntRules, RefinedIntRules, GeometryRefiner
+  GlobGeometryRefiner, and FiniteElement::dof2quad_array.
+
+
+Version 4.6, released on September 27, 2023
+===========================================
+
+- MFEM is now available in Homebrew and can be installed on a Mac with just
+  "brew install mfem". See https://formulae.brew.sh/formula/mfem.
+
+Meshing improvements
+--------------------
+- Added asymptotically-balanced TMOP compound metrics 90, 94, 328, 338. A new
+  tool, tmop-metric-magnitude, can be used to track how metrics change under
+  geometric perturbations. See miniapps/tools.
+
+- Several NURBS meshing improvements:
+  * Support for free connectivity of NURBS patches allowing for more complex
+    patch configurations such as C-meshes.
+  * New methods to set and get attributes on NURBS patches and patch boundaries.
+  * The edge to knot map for NURBS meshes can be determined automatically. It is
+    no longer needed to specify this in the NURBS mesh.
+  * Added curve interpolation method for NURBS.
+  * Added new small miniapp for printing of shape functions of a KnotVector
+  * See miniapps/nurbs for example meshes and miniapps.
+
+Discretization improvements
+---------------------------
+- SubMesh and ParSubMesh have been extended to support the transfer of
+  Nedelec and Raviart-Thomas finite element spaces.
+
+- Added support for partial assembly on NURBS patches, and NURBS-patch sparse
+  matrix assembly. Patch matrix assembly includes the option to use reduced
+  approximate integration rules, computed by the newly implemented non-negative
+  least-squares (NNLS) solver.
+
+- Support for parallel transfer of H1 fields using the low-order refined (LOR)
+  transfer operators in L2ProjectionGridTransfer
+
+- Added KDTree class for 2D/3D set of points, which is then utilized in the new
+  KDTreeNodalProjection class to project a function defined on an arbitrary set
+  of points onto an MFEM grid function. This functionality is demonstrated in
+  the nodal-transfer miniapp. The current implementation is serial only. Further
+  extensions can include search in arbitrary dimensional spaces.
+
+- Added support for p-refined meshes in GSLIB-FindPoints.
+
+- Device kernels can now access device-specific DOF and quadrature limits using
+  the DofQuadLimits structure, allowing increased limits when executing on CPU.
+  The limits for the runtime selected device can be accessed in host code using
+  DeviceDofQuadLimits::Get(). The global constants MAX_D1D and MAX_Q1D are no
+  longer available.
+
+- Face restriction operators for Nedelec and Raviart-Thomas finite element
+  spaces are now supported through the ConformingFaceRestriction class.
+
+- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
+
+Linear and nonlinear solvers
+----------------------------
+- Updated the MUMPS interface to support multiple right-hand sides, block
+  low-rank compression, builds using 64-bit integers, and other improvements.
+
+- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
+  The interface provides a serial (OpenMP shared memory) version of Pardiso for
+  use with SparseMatrix. This complements the existing parallel (MPI distributed
+  memory) version already available through the CPardiso MFEM integration.
+
+- Added HIP support to the PETSc and SUNDIALS interfaces.
+
+- Efficient GPU-accelerated LOR assembly now supports surface meshes.

 New and updated examples and miniapps
 -------------------------------------
- Added a new example code, Example 36/36p, to demonstrate the solution of
-  the obstacle problem with a new finite element method.
-
- Added a new miniapp, Mesh Quality, for evaluating mesh quality using size,
-  skewness, and aspect-ratio computed from the Jacobian of the transformation.
-
- Added a new miniapp for interface and boundary fitting to implicit domains
-  defined using level-set functions. See miniapps/meshing/pmesh-fitting.cpp.
-
- Added a new miniapp for fitting of selected mesh nodes to specified positions,
-  while maintaining mesh quality. See miniapps/meshing/fit-node-position.cpp.
+- Added a new H(div) solver miniapp demonstrating the use of a matrix-free
+  saddle-point solver methodology, suitable for high-order discretizations and
+  for GPU acceleration. Examples illustrating the solution of Darcy and grad-div
+  problems are included. See miniapps/hdiv-linear-solver.

 - Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
  and parallel examples for diffusion, convection-diffusion, acoustics and
  Maxwell equations. The miniapp includes new classes such as (Par)DPGWeakForm,
  (Par)ComplexDPGWeakForm and (Complex)BlockStaticCondensation. Three new
  integrators are added in support of DPG systems: TraceIntegrator,
-  NormalTraceIntegrator and TangentTraceIntegrator.
+  NormalTraceIntegrator and TangentTraceIntegrator. See miniapps/dpg.

- Added new SubMesh examples demonstrating source terms and boundary conditions
-  transferred from SubMesh objects.
+- Added a new miniapp that implements the SPDE method for generating Gaussian
+  random fields of Matern covariance. The resulting random field can be used,
+  e.g., to model material uncertainties. See miniapps/spde.

- Added a miniapp for interpolation of NURBS.
+- Added a new parallel LOR transfer miniapp, plor-transfer, which mirrors the
+  functionality of the serial LOR transfer miniapp. See miniapps/tools.

- Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
-  demonstrating the use of a matrix-free saddle-point solver methodology,
-  suitable for high-order discretizations and for GPU acceleration. Examples
-  illustrating the solution of Darcy and grad-div problems are included.
+- New serial miniapp, nodal-transfer, demonstrating the use of KDTree to map a
+  parallel grid function to a different parallel partitioning of the same mesh.
+
+- Added 3 additional TMOP miniapps in miniapps/meshing:
+  * Mesh-Quality evaluates quality using size, skewness, and aspect-ratio
+    computed from the Jacobian of the transformation.
+  * Mesh-Fitting can be used for interface and boundary fitting to implicit
+    domains defined using level-set functions.
+  * Fit-Node-Position fits selected mesh nodes to specified positions, while
+    maintaining overall mesh quality.
+
+- Added 4 new example codes:
+  * Example 34/34p solves a simple magnetostatic problem where source terms and
+    boundary conditions are transferred with SubMesh objects.
+  * Example 35p implements H1, H(curl) and H(div) variants of a damped harmonic
+    oscillator with field transfer using SubMesh objects.
+  * Example 36/36p demonstrates the solution of the obstacle problem with a new
+    finite element method (proximal Galerkin).
+  * Example 37/37p demonstrates topology optimization with MFEM.

 - Added a random refinement option to the mesh-explorer miniapp to assist users
  in experimenting with nonconforming meshes.

 - Moved the distance solver methods from miniapps/shifted to miniapps/common.

- Added a new parallel LOR transfer miniapp, miniapps/tools/plor-transfer, which
-  mirrors the functionality of the serial LOR transfer miniapp,
-  miniapps/tools/lor-transfer
-
-Meshing improvements
--------------------
- Added support for free connectivity of NURBS patches allowing for more complex
-  patch configurations such as C-meshes. This is demonstrated in a new NURBS
-  miniapp.
-  
- The edge to knot map for NURBS meshes can be determined automatically. It is no
-  longer needed to specify this in the NURBS mesh. A mesh in the NURBS miniapp
-  demonstrates this.
-
- Added new methods in the Mesh class to set and get attributes on NURBS patches
-  and patch boundaries.
-
- Added HIP support to the SUNDIALS interface.
-
- TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
-  338. Added the tmop-metric-magnitude tool for tracking how metrics change
-  under geometric perturbations.
-
-Discretization improvements
---------------------------
- Face restriction operators for Nedelec and Raviart-Thomas finite element
-  spaces are now supported through the ConformingFaceRestriction class.
-
- SubMesh and ParSubMesh have been extended to support the transfer of
-  Nedelec and Raviart-Thomas finite element spaces.
-
- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
-
- Added support for partial assembly on NURBS patches and NURBS patch sparse
-  matrix assembly. Patch matrix assembly includes the option to use reduced
-  approximate integration rules, computed by the newly implemented non-negative
-  least-squares (NNLS) solver.
-
- Added support for p-refined meshes in FindPointsGSLIB.
-
- Support for parallel transfer of H1 fields using the low-order refined (LOR)
-  transfer operators in L2ProjectionGridTransfer
-
-Linear and nonlinear solvers
----------------------------
- Updated interface to MUMPS direct solver to support multiple right-hand
-  sides, block low-rank compression, builds using 64-bit integers, and other
-  improvements.
-
- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
-  This interface provides a serial (OpenMP shared memory) version of Pardiso for
-  use with SparseMatrix. This complements the existing parallel (MPI distributed
-  memory) version already available through the CPardiso MFEM integration.
-
-Integrations, testing and documentation
---------------------------------------
- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
-  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
-
 Miscellaneous
 -------------
 - Improved lambda body debugging with the addition of mfem::forall functions.
  These functions can take the place of the MFEM_FORALL macros, which have been
  preserved for backwards compatibility.

+- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
+  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
+
 - Reorganized files for bilinear form, linear form, and nonlinear form integrators
  in the fem/integ/ subdirectory.

 - FiniteElementSpace::GetFE has been updated to abort instead of returning NULL for
  an empty partition.

+- Various other simplifications, extensions, and bugfixes in the code.
+
+
 Version 4.5.2, released on March 23, 2023
 =========================================

@@ -57,7 +57,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.3)
+set(${PROJECT_NAME}_VERSION 4.6.1)

 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -139,10 +139,9 @@ if (MFEM_USE_CUDA)
      set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
   endif()
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
+   find_package(CUDAToolkit REQUIRED)
   set(CUSPARSE_FOUND TRUE)
-   set(CUSPARSE_LIBRARIES "cusparse")
-   set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
+   get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
 endif()

 if (XSDK_ENABLE_C)
@@ -531,7 +530,7 @@ find_package(Threads REQUIRED)
 set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
    NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
+    ADIOS2 CUSPARSE MKL_CPARDISO MKL_PARDISO AMGX CALIPER CODIPACK
    BENCHMARK PARELAG MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)

 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
@@ -641,16 +640,34 @@ if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
  foreach(Header mfem.hpp mfem-performance.hpp)
    message(STATUS
      "Writing substitute header --> \"${Header}\"")
-    file(WRITE "${PROJECT_BINARY_DIR}/${Header}"
+    file(WRITE "${PROJECT_BINARY_DIR}/${Header}.tmp"
 "// Auto-generated file.
 #define MFEM_CONFIG_FILE \"${PROJECT_BINARY_DIR}/config/_config.hpp\"
 #include \"${PROJECT_SOURCE_DIR}/${Header}\"
 ")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+      "${PROJECT_BINARY_DIR}/${Header}.tmp"
+      "${PROJECT_BINARY_DIR}/${Header}"
+    )
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
+      "${PROJECT_BINARY_DIR}/${Header}.tmp"
+    )
+
    # This version will be installed in the top include directory:
-    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
+    file(WRITE "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
 "// Auto-generated file.
 #include \"mfem/${Header}\"
 ")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different 
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}"
+    )
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove
+      "${PROJECT_BINARY_DIR}/InstallHeaders/${Header}.tmp"
+    )
+
  endforeach()
 endif()

@@ -135,6 +135,7 @@ The MFEM source code has the following structure:
  │   ├── adjoint
  │   ├── autodiff
  │   ├── common
+  │   ├── dpg
  │   ├── electromagnetics
  │   ├── gslib
  │   ├── hdiv-linear-solver
@@ -148,6 +149,7 @@ The MFEM source code has the following structure:
  │   ├── performance
  │   ├── shifted
  │   ├── solvers
+  │   ├── spde
  │   ├── tools
  │   └── toys
  └── tests
@@ -659,8 +659,7 @@ The specific libraries and their options are:
  requires the PT-Scotch and Scalapack libraries as well as ParMETIS, which
  includes METIS 5 in its distribution. Starting with STRUMPACK v2.2.0, ParMETIS
  and PT-Scotch are optional dependencies.
-  The support for STRUMPACK was added in MFEM v3.3.2 and it requires STRUMPACK
-  2.0.0 or later.
+  The support for STRUMPACK was added in MFEM v3.3.2.
  URL: http://portal.nersc.gov/project/sparse/strumpack
  Options: STRUMPACK_OPT, STRUMPACK_LIB.
  Versions: STRUMPACK >= 3.0.0.
@@ -699,12 +698,15 @@ The specific libraries and their options are:
  PETSc has been cloned on the same level as mfem and hypre:
    ./configure --download-fblaslapack=yes --download-scalapack=yes \
                --download-mumps=yes --download-suitesparse=yes \
-                --with-hypre-dir=../hypre-2.10.0b/src/hypre \
+                --with-hypre-dir=../hypre/src/hypre \
                --with-shared-libraries=0
+  When building PETSc with HIP, one may need to add a flag like -std=c2x to
+  CFLAGS to allow proper parsing of the hipsparse header under C.
  URL: https://www.mcs.anl.gov/petsc
  Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA)
+  Versions: PETSc >= 3.8.0   (PETSc build without CUDA/HIP)
            PETSc >= 3.15.0  (PETSc built with CUDA)
+            PETSc >= 3.19.0  (PETSc built with HIP, older versions may work too)

 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
  uses some of the PETSc options when compiled.
@@ -794,7 +796,7 @@ The specific libraries and their options are:
  URL: https://github.com/CEED/libCEED
       https://ceed.exascaleproject.org/libceed
  Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.10.
+  Versions: libCEED >= 0.12.

 - RAJA (optional), used when MFEM_USE_RAJA = YES.
  Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.
@@ -14,9 +14,13 @@
 #   - HYPRE_LIBRARIES
 #   - HYPRE_INCLUDE_DIRS
 #   - HYPRE_VERSION
+#   - HYPRE_USING_CUDA (internal)
 #   - HYPRE_USING_HIP (internal)

 if (HYPRE_FOUND)
+  if (HYPRE_USING_CUDA)
+    find_package(CUDAToolkit REQUIRED)
+  endif()
  if (HYPRE_USING_HIP)
    find_package(rocsparse REQUIRED)
    find_package(rocrand REQUIRED)
@@ -27,6 +31,20 @@ endif()
 include(MfemCmakeUtilities)
 mfem_find_package(HYPRE HYPRE HYPRE_DIR "include" "HYPRE.h" "lib" "HYPRE"
  "Paths to headers required by HYPRE." "Libraries required by HYPRE."
+  CHECK_BUILD HYPRE_USING_CUDA FALSE
+  "
+#undef HYPRE_USING_CUDA
+#include <HYPRE_config.h>
+
+#ifndef HYPRE_USING_CUDA
+#error HYPRE is built without CUDA.
+#endif
+
+int main()
+{
+   return 0;
+}
+"
  CHECK_BUILD HYPRE_USING_HIP FALSE
  "
 #undef HYPRE_USING_HIP
@@ -57,6 +75,16 @@ if (HYPRE_FOUND AND (NOT HYPRE_VERSION))
  endif()
 endif()

+if (HYPRE_FOUND AND HYPRE_USING_CUDA)
+  find_package(CUDAToolkit REQUIRED)
+  get_target_property(CUSPARSE_LIBRARIES CUDA::cusparse LOCATION)
+  get_target_property(CURAND_LIBRARIES CUDA::curand LOCATION)
+  list(APPEND HYPRE_LIBRARIES ${CUSPARSE_LIBRARIES} ${CURAND_LIBRARIES})
+  set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
+      "HYPRE libraries + dependencies." FORCE)
+  message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")
+endif()
+
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
  find_package(rocsparse REQUIRED)
  find_package(rocrand REQUIRED)
@@ -106,12 +106,7 @@ set(HYPRE_DIR "${MFEM_DIR}/../hypre/src/hypre" CACHE PATH
 # If hypre was compiled to depend on BLAS and LAPACK:
 # set(HYPRE_REQUIRED_PACKAGES "BLAS" "LAPACK" CACHE STRING
 #     "Packages that HYPRE depends on.")
-if (MFEM_USE_CUDA)
-   # This is only necessary when hypre is built with cuda:
-   set(HYPRE_REQUIRED_LIBRARIES "-lcusparse" "-lcurand" CACHE STRING
-       "Libraries that HYPRE depends on.")
-endif()
-# HIP dependency for HYPRE is handled in FindHYPRE.cmake.
+# CUDA and HIP dependencies for HYPRE are handled in FindHYPRE.cmake.

 set(METIS_DIR "${MFEM_DIR}/../metis-4.0" CACHE PATH "Path to the METIS library.")

@@ -157,7 +152,8 @@ set(STRUMPACK_DIR "${MFEM_DIR}/../STRUMPACK-build" CACHE PATH
 # STRUMPACK may also depend on "OpenMP", depending on how it was compiled.
 # Starting with v2.2.0 of STRUMPACK, ParMETIS and Scotch are optional.
 set(STRUMPACK_REQUIRED_PACKAGES "MPI" "MPI_Fortran" "ParMETIS" "METIS"
-    "ScaLAPACK" "Scotch/ptscotch/ptscotcherr/scotch/scotcherr" CACHE STRING
+    "Scotch/ptscotch/ptscotcherr/scotch/scotcherr"
+    "ScaLAPACK" "LAPACK" "BLAS" CACHE STRING
    "Additional packages required by STRUMPACK.")
 # If the MPI package does not find all required Fortran libraries:
 # set(STRUMPACK_REQUIRED_LIBRARIES "gfortran" "mpi_mpifh" CACHE STRING
@@ -331,16 +331,30 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
 $(SCOTCH_LIB) $(SCALAPACK_LIB)

-# Ginkgo library configuration (currently not needed)
+# Ginkgo library configuration
 GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
+GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
 GINKGO_BUILD_TYPE=Release
 ifeq ($(MFEM_USE_GINKGO),YES)
   BASE_FLAGS = -std=c++14
 endif
 GINKGO_OPT = -isystem $(GINKGO_DIR)/include
-GINKGO_LIB_DIR = $(sort $(dir $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
-ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*d.a  $(GINKGO_DIR)/lib*/libginkgo*d.so $(GINKGO_DIR)/lib*/libginkgo*d.dylib $(GINKGO_DIR)/lib*/libginkgo*d.dll)))
-ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard $(GINKGO_DIR)/lib*/libginkgo*.a  $(GINKGO_DIR)/lib*/libginkgo*.so $(GINKGO_DIR)/lib*/libginkgo*.dylib $(GINKGO_DIR)/lib*/libginkgo*.dll)))
+GINKGO_LIB_DIR = $(sort $(dir $(wildcard\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
+GINKGO_LINK_LIB_DIR = $(GINKGO_DIR)$(subst $(GINKGO_SEARCH_DIR),,$(GINKGO_LIB_DIR))
+ALL_GINKGO_LIBS_DEBUG = $(notdir $(basename $(wildcard\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.a\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.so\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dylib\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*d.dll)))
+ALL_GINKGO_LIBS = $(notdir $(basename $(wildcard\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.a\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.so\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dylib\
+ $(GINKGO_SEARCH_DIR)/lib*/libginkgo*.dll)))
 ALL_GINKGO_LIBS_RELEASE = $(filter-out $(ALL_GINKGO_LIBS_DEBUG),$(ALL_GINKGO_LIBS))
 GINKGO_LINK = $(subst libginkgo,-lginkgo,$(ALL_GINKGO_LIBS_RELEASE))
 ifeq ($(GINKGO_BUILD_TYPE),Debug)
@@ -349,7 +363,8 @@ ifeq ($(GINKGO_BUILD_TYPE),Debug)
  endif
 else
 endif
-GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LIB_DIR) -L$(GINKGO_LIB_DIR) $(GINKGO_LINK)
+GINKGO_LIB = $(XLINKER)-rpath,$(GINKGO_LINK_LIB_DIR) -L$(GINKGO_LINK_LIB_DIR)\
+ $(GINKGO_LINK)

 # AmgX library configuration
 AMGX_DIR = @MFEM_DIR@/../amgx
@@ -38,14 +38,14 @@ all: header config-mk
 MPI = $(MFEM_USE_MPI:NO=)
 GHV_CXX ?= $(MFEM_CXX)
 GHV = get_hypre_version
-GHV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
+GHV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(HYPRE_OPT))
 SMX = $(if $(MFEM_USE_PUMI:NO=),MFEM_USE_SIMMETRIX)
 SMX_PATH = $(PUMI_DIR)/include/gmi_sim.h
 SMX_FILE = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(SMX_PATH))
 MUMPS = $(MFEM_USE_MUMPS:NO=)
 GMV_CXX ?= $(MFEM_CXX)
 GMV = get_mumps_version
-GMV_FLAGS = $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))
+GMV_FLAGS = $(MFEM_CXXFLAGS) $(subst @MFEM_DIR@,$(if $(MFEM_DIR),$(MFEM_DIR),..),$(MUMPS_OPT))

 $(GHV): $(SRC)$(GHV).cpp
 	$(call mfem-info, Determining HYPRE version ...)
@@ -110,3 +110,4 @@ config-mk:

 clean:
 	rm -f $(CONFIG_HPP) $(CONFIG_MK) sample-runs-build.log
+	rm -f $(GHV) $(GHV).out $(GMV) $(GMV).out
@@ -315,7 +315,7 @@ function extract_sample_runs()
   sruns=`grep -v "^//.*  mpirun .* ${app}" "${src}" |
          grep "^//.*  ${app}" |
          sed -e "s/.*  ${app}/${vg_app}/g"`
-   runs="${sruns}${pruns}"
+   runs="${sruns}"$'\n'"${pruns}"
   if [ "$skip_gen_meshes" == "yes" ]; then
      runs=`printf "%s" "$runs" | grep -v ".* -m .*\.gen"`
   fi
@@ -0,0 +1,37 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+1
+
+elements
+4
+1 1 0 1
+1 1 1 2
+1 1 2 3
+1 1 3 4
+
+boundary
+2
+1 0 0
+2 0 4
+
+vertices
+5
+2
+0 0
+0.25 0.25
+0.50 0.50
+0.75 0.75
+1 1
@@ -0,0 +1,37 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+1
+
+elements
+4
+1 1 0 1
+1 1 1 2
+1 1 2 3
+1 1 3 4
+
+boundary
+2
+1 0 0
+2 0 4
+
+vertices
+5
+3
+0 0 0
+0.25 0.25 0.25
+0.50 0.50 0.50
+0.75 0.75 0.75
+1 1 1
@@ -105,8 +105,14 @@ namespace mfem {
 * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
 * - <a class="el" href="ex33_8cpp_source.html">Example 33</a>: nodal H1 FEM for the fractional Laplacian problem
 * - <a class="el" href="ex33p_8cpp_source.html">Example 33p</a>: parallel nodal H1 FEM for the fractional Laplacian problem
+ * - <a class="el" href="ex34_8cpp_source.html">Example 34</a>: multi-domain magnetostatics
+ * - <a class="el" href="ex34p_8cpp_source.html">Example 34p</a>: parallel multi-domain magnetostatics
+ * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
 * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
 * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
+ * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
+ * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
+ * - <a class="el" href="ex38_8cpp_source.html">Example 38</a>: cut-surface and cut-volume integration
 *
 * <H4>AmgX Examples</H4>
 * - Variants of Examples
@@ -14,7 +14,7 @@ If not already available, Doxygen can be downloaded from

   http://www.doxygen.org

-We recommend using version 1.8 or later.
+We recommend using version 1.9.8 or later.

 To build the documentation, simply type "make" in the doc/ directory. This will
 create the file CodeDocumentation.html, which can be viewed in any web browser.
@@ -0,0 +1,3 @@
+html {
+    --content-maxwidth: auto;
+}
@@ -0,0 +1,78 @@
+<!-- HTML header for doxygen 1.9.6-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="$langISO">
+<head>
+<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
+<meta http-equiv="X-UA-Compatible" content="IE=11"/>
+<meta name="generator" content="Doxygen $doxygenversion"/>
+<meta name="viewport" content="width=device-width, initial-scale=1"/>
+<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
+<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
+<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<script type="text/javascript">var page_layout=1;</script>
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+<script type="text/javascript" src="$relpath^jquery.js"></script>
+<script type="text/javascript" src="$relpath^dynsections.js"></script>
+$treeview
+$search
+$mathjax
+$darkmode
+<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+$extrastylesheet
+<script type="text/javascript" src="$relpath^doxygen-awesome-darkmode-toggle.js"></script>
+<script type="text/javascript">
+    DoxygenAwesomeDarkModeToggle.init()
+</script>
+</head>
+<body>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<div id="side-nav" class="ui-resizable side-nav-resizable"><!-- do not remove this div, it is closed by doxygen! -->
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+
+<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
+
+<!--BEGIN TITLEAREA-->
+<div id="titlearea">
+<table cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr id="projectrow">
+  <!--BEGIN PROJECT_LOGO-->
+  <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
+  <!--END PROJECT_LOGO-->
+  <!--BEGIN PROJECT_NAME-->
+  <td id="projectalign">
+   <div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
+   </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
+  </td>
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+   <!--BEGIN PROJECT_BRIEF-->
+    <td>
+    <div id="projectbrief">$projectbrief</div>
+    </td>
+   <!--END PROJECT_BRIEF-->
+  <!--END !PROJECT_NAME-->
+  <!--BEGIN DISABLE_INDEX-->
+   <!--BEGIN SEARCHENGINE-->
+     <!--BEGIN !FULL_SIDEBAR-->
+    <td>$searchbox</td>
+     <!--END !FULL_SIDEBAR-->
+   <!--END SEARCHENGINE-->
+  <!--END DISABLE_INDEX-->
+ </tr>
+  <!--BEGIN SEARCHENGINE-->
+   <!--BEGIN FULL_SIDEBAR-->
+   <tr><td colspan="2">$searchbox</td></tr>
+   <!--END FULL_SIDEBAR-->
+  <!--END SEARCHENGINE-->
+ </tbody>
+</table>
+</div>
+<!--END TITLEAREA-->
+<!-- end header part -->
@@ -0,0 +1,157 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2021 - 2023 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+class DoxygenAwesomeDarkModeToggle extends HTMLElement {
+    // SVG icons from https://fonts.google.com/icons
+    // Licensed under the Apache 2.0 license:
+    // https://www.apache.org/licenses/LICENSE-2.0.html
+    static lightModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FCBF00"><rect fill="none" height="24" width="24"/><circle cx="12" cy="12" opacity=".3" r="3"/><path d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg>`
+    static darkModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FE9700"><rect fill="none" height="24" width="24"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27 C17.45,17.19,14.93,19,12,19c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z" opacity=".3"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg>`
+    static title = "Toggle Light/Dark Mode"
+
+    static prefersLightModeInDarkModeKey = "prefers-light-mode-in-dark-mode"
+    static prefersDarkModeInLightModeKey = "prefers-dark-mode-in-light-mode"
+
+    static _staticConstructor = function() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.userPreference)
+        // Update the color scheme when the browsers preference changes
+        // without user interaction on the website.
+        window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+        })
+        // Update the color scheme when the tab is made visible again.
+        // It is possible that the appearance was changed in another tab 
+        // while this tab was in the background.
+        document.addEventListener("visibilitychange", visibilityState => {
+            if (document.visibilityState === 'visible') {
+                DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+            }
+        });
+    }()
+
+    static init() {
+        $(function() {
+            $(document).ready(function() {
+                const toggleButton = document.createElement('doxygen-awesome-dark-mode-toggle')
+                toggleButton.title = DoxygenAwesomeDarkModeToggle.title
+                toggleButton.updateIcon()
+
+                window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+                    toggleButton.updateIcon()
+                })
+                document.addEventListener("visibilitychange", visibilityState => {
+                    if (document.visibilityState === 'visible') {
+                        toggleButton.updateIcon()
+                    }
+                });
+
+                $(document).ready(function(){
+                    document.getElementById("MSearchBox").parentNode.appendChild(toggleButton)
+                })
+                $(window).resize(function(){
+                    document.getElementById("MSearchBox").parentNode.appendChild(toggleButton)
+                })
+            })
+        })
+    }
+
+    constructor() {
+        super();
+        this.onclick=this.toggleDarkMode
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode system preference
+     */
+    static get systemPreference() {
+        return window.matchMedia('(prefers-color-scheme: dark)').matches
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode user preference
+     */
+    static get userPreference() {
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
+    }
+
+    static set userPreference(userPreference) {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = userPreference
+        if(!userPreference) {
+            if(DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)
+            }
+        } else {
+            if(!DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey)
+            }
+        }
+        DoxygenAwesomeDarkModeToggle.onUserPreferenceChanged()
+    }
+
+    static enableDarkMode(enable) {
+        if(enable) {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = true
+            document.documentElement.classList.add("dark-mode")
+            document.documentElement.classList.remove("light-mode")
+        } else {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = false
+            document.documentElement.classList.remove("dark-mode")
+            document.documentElement.classList.add("light-mode")
+        }
+    }
+
+    static onSystemPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = DoxygenAwesomeDarkModeToggle.userPreference
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    static onUserPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    toggleDarkMode() {
+        DoxygenAwesomeDarkModeToggle.userPreference = !DoxygenAwesomeDarkModeToggle.userPreference
+        this.updateIcon()
+    }
+
+    updateIcon() {
+        if(DoxygenAwesomeDarkModeToggle.darkModeEnabled) {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.darkModeIcon
+        } else {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.lightModeIcon
+        }
+    }
+}
+
+customElements.define("doxygen-awesome-dark-mode-toggle", DoxygenAwesomeDarkModeToggle);
@@ -16,7 +16,7 @@ DOXYGEN_CONF = CodeDocumentation.conf
 # doxygen uses: graphviz, latex
 html: $(DOXYGEN_CONF)
 	@# Generate the html documentation
-	@( cat $(DOXYGEN_CONF) ; echo "$(MFEM_DOXYGEN_FLAGS)" ) | doxygen -
+	@( cat $(DOXYGEN_CONF) ; printf "$(MFEM_DOXYGEN_FLAGS)\n" ) | doxygen -
 	@echo "<meta http-equiv=\"REFRESH\" content=\"0;URL=CodeDocumentation/html/index.html\">" > CodeDocumentation.html
 	@cat warnings.log 1>&2
 	@# Generate the log of undocumented methods
@@ -42,8 +42,15 @@ list(APPEND ALL_EXE_SRCS
  ex33.cpp
  ex34.cpp
  ex36.cpp
+  ex37.cpp
  )

+if(MFEM_USE_LAPACK)
+list(APPEND ALL_EXE_SRCS
+  ex38.cpp
+  )
+endif()
+
 if (MFEM_USE_MPI)
  list(APPEND ALL_EXE_SRCS
    ex0p.cpp
@@ -82,6 +89,7 @@ if (MFEM_USE_MPI)
    ex34p.cpp
    ex35p.cpp
    ex36p.cpp
+    ex37p.cpp
    )
 endif()

@@ -107,6 +115,8 @@ if (MFEM_ENABLE_TESTING)
      list(APPEND THIS_TEST_OPTIONS "-e" "1")
    elseif(${TEST_NAME} MATCHES "ex27p*")
      list(APPEND THIS_TEST_OPTIONS "-dg")
+    elseif(${TEST_NAME} MATCHES "ex37p*")
+      list(APPEND THIS_TEST_OPTIONS "-mi" "3")
    endif()

    if (NOT (${TEST_NAME} MATCHES ".*p$"))
@@ -262,12 +262,13 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
      if (sp_solver)
      {
-         STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+         STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
         strumpack->SetPrintFactorStatistics(true);
         strumpack->SetPrintSolveStatistics(false);
         strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
         strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-         strumpack->DisableMatching();
+         strumpack->SetMatching(strumpack::MatchingJob::NONE);
+         strumpack->SetCompression(strumpack::CompressionType::NONE);
         strumpack->SetOperator(*Arow);
         strumpack->SetFromCommandLine();
         precond = strumpack;
@@ -5,6 +5,7 @@
 // Sample runs:  mpirun -np 4 ex13p -m ../data/star.mesh
 //               mpirun -np 4 ex13p -m ../data/square-disc.mesh -o 2 -n 4
 //               mpirun -np 4 ex13p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex13p -m ../data/beam-tet.mesh -nc -o 2 -rs 1
 //               mpirun -np 4 ex13p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex13p -m ../data/escher.mesh
 //               mpirun -np 4 ex13p -m ../data/fichera.mesh
@@ -54,6 +55,7 @@ int main(int argc, char *argv[])
   int par_ref_levels = 1;
   int order = 1;
   int nev = 5;
+   bool nc = false;
   bool visualization = 1;
   const char *device_config = "cpu";

@@ -69,6 +71,9 @@ int main(int argc, char *argv[])
                  " isoparametric space.");
   args.AddOption(&nev, "-n", "--num-eigs",
                  "Number of desired eigenmodes.");
+   args.AddOption(&nc, "-nc", "--non-conforming", "-c",
+                  "--conforming",
+                  "Mark the mesh as nonconforming before partitioning.");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
                  "Enable or disable GLVis visualization.");
@@ -98,6 +103,10 @@ int main(int argc, char *argv[])
   //    and volume meshes with the same code.
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   int dim = mesh->Dimension();
+   if (nc)
+   {
+      mesh->EnsureNCMesh(true);
+   }

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement (2 by default, or
@@ -13,6 +13,7 @@
 //               mpirun -np 4 ex15p -m ../data/square-disc-nurbs.mesh
 //               mpirun -np 4 ex15p -m ../data/disc-nurbs.mesh
 //               mpirun -np 4 ex15p -m ../data/fichera.mesh -tf 0.5
+//               mpirun -np 4 ex15p -m ../data/fichera-mixed.mesh -tf 0.5
 //               mpirun -np 4 ex15p -m ../data/ball-nurbs.mesh -tf 0.5
 //               mpirun -np 4 ex15p -m ../data/mobius-strip.mesh
 //               mpirun -np 4 ex15p -m ../data/amr-quad.mesh
@@ -26,13 +26,12 @@
 using namespace std;
 using namespace mfem;

-/** After spatial discretization, the conduction model can be written as:
+/** After spatial discretization, the wave model can be written as:
 *
 *     d^2u/dt^2 = M^{-1}(-Ku)
 *
- *  where u is the vector representing the temperature, M is the mass matrix,
- *  and K is the diffusion operator with diffusivity depending on u:
- *  (\kappa + \alpha u).
+ *  where u is the vector representing the temperature, M is the mass,
+ *  and K is the stiffness matrix.
 *
 *  Class WaveOperator represents the right-hand side of the above ODE.
 */
@@ -301,7 +300,7 @@ int main(int argc, char *argv[])
   Vector dudt;
   dudt_gf.GetTrueDofs(dudt);

-   // 7. Initialize the conduction operator and the visualization.
+   // 7. Initialize the wave operator and the visualization.
   Array<int> ess_bdr;
   if (mesh->bdr_attributes.Size())
   {
@@ -356,7 +355,7 @@ int main(int argc, char *argv[])
      else
      {
         sout.precision(precision);
-         sout << "solution\n" << *mesh << dudt_gf;
+         sout << "solution\n" << *mesh << u_gf;
         sout << "pause\n";
         sout << flush;
         cout << "GLVis visualization paused."
@@ -170,6 +170,7 @@ int main(int argc, char *argv[])
   bool herm_conv = true;
   bool slu_solver  = false;
   bool mumps_solver = false;
+   bool strumpack_solver = false;
   bool visualization = 1;
   bool pa = false;
   const char *device_config = "cpu";
@@ -200,6 +201,11 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_MUMPS
   args.AddOption(&mumps_solver, "-mumps", "--mumps-solver", "-no-mumps",
                  "--no-mumps-solver", "Use the MUMPS Solver.");
+#endif
+#ifdef MFEM_USE_STRUMPACK
+   args.AddOption(&strumpack_solver, "-strumpack", "--strumpack-solver",
+                  "-no-strumpack", "--no-strumpack-solver",
+                  "Use the STRUMPACK Solver.");
 #endif
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                  "--no-visualization",
@@ -209,13 +215,14 @@ int main(int argc, char *argv[])
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.Parse();
-   if (slu_solver && mumps_solver)
+   if (slu_solver + mumps_solver + strumpack_solver > 1)
   {
      if (myid == 0)
-         cout << "WARNING: Both SuperLU and MUMPS have been selected,"
-              << " please choose either one." << endl
+         cout << "WARNING: More than one of SuperLU, MUMPS, and STRUMPACK have"
+              << " been selected, please choose only one." << endl
              << "         Defaulting to SuperLU." << endl;
      mumps_solver = false;
+      strumpack_solver = false;
   }

   if (iprob > 4) { iprob = 4; }
@@ -474,6 +481,24 @@ int main(int argc, char *argv[])
      delete A;
   }
 #endif
+#ifdef MFEM_USE_STRUMPACK
+   if (!pa && strumpack_solver)
+   {
+      HypreParMatrix *A = Ah.As<ComplexHypreParMatrix>()->GetSystemMatrix();
+      STRUMPACKRowLocMatrix SA(*A);
+      STRUMPACKSolver strumpack(MPI_COMM_WORLD, argc, argv);
+      strumpack.SetPrintFactorStatistics(false);
+      strumpack.SetPrintSolveStatistics(false);
+      strumpack.SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
+      strumpack.SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
+      strumpack.SetMatching(strumpack::MatchingJob::NONE);
+      strumpack.SetCompression(strumpack::CompressionType::NONE);
+      strumpack.SetFromCommandLine();
+      strumpack.SetOperator(SA);
+      strumpack.Mult(B, X);
+      delete A;
+   }
+#endif
 #ifdef MFEM_USE_MUMPS
   if (!pa && mumps_solver)
   {
@@ -493,7 +518,7 @@ int main(int argc, char *argv[])
   //
   //    In PML:   1/mu (abs(1/det(J) J^T J) Curl E, Curl F)
   //              + omega^2 * epsilon (abs(det(J) * (J^T J)^-1) * E, F)
-   if (pa || (!slu_solver && !mumps_solver))
+   if (pa || (!slu_solver && !mumps_solver && !strumpack_solver))
   {
      ConstantCoefficient absomeg(pow(omega, 2) * epsilon);
      RestrictedCoefficient restr_absomeg(absomeg,attr);
@@ -63,6 +63,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
+   bool nc = false;
   const char *device_config = "cpu";
   bool visualization = 1;

@@ -77,6 +78,9 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&nc, "-nc", "--non-conforming", "-c",
+                  "--conforming",
+                  "Mark the mesh as nonconforming before partitioning.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -102,6 +106,11 @@ int main(int argc, char *argv[])
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();
+   if (nc)
+   {
+      // Can set to false to use conformal refinement for simplices.
+      mesh->EnsureNCMesh(true);
+   }

   // 4. Refine the mesh to increase the resolution. In this example we do
   //    'ref_levels' of uniform refinement. We choose 'ref_levels' to be the
@@ -267,9 +267,9 @@ int main(int argc, char *argv[])
                << "window_geometry 400 0 400 350" << flush;
   }

-   // 7. Define a parallel finite element space on the full mesh. Here we
-   //    use the H(curl) finite elements for the vector potential and H(div)
-   //    for the current density.
+   // 7. Define a parallel finite element space on the full mesh. Here we use
+   //    the H(curl) finite elements for the vector potential and H(div) for the
+   //    current density.
   ND_FECollection fec_nd(order, dim);
   RT_FECollection fec_rt(order - 1, dim);
   FiniteElementSpace fespace_nd(&mesh, &fec_nd);
@@ -292,10 +292,10 @@ int main(int argc, char *argv[])
   }

   // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined
-   //    by marking all the boundary attributes except for those on a symmetry
-   //    plane as essential (Dirichlet) and converting them to a list of
-   //    true dofs.
+   //    boundary dofs. In this example, the boundary conditions are defined by
+   //    marking all the boundary attributes except for those on a symmetry
+   //    plane as essential (Dirichlet) and converting them to a list of true
+   //    dofs.
   Array<int> ess_tdof_list;
   Array<int> ess_bdr;
   if (mesh.bdr_attributes.Size())
@@ -324,14 +324,13 @@ int main(int argc, char *argv[])
   GridFunction x(&fespace_nd);
   x = 0.0;

-   // 11. Set up the parallel bilinear form corresponding to the EM
-   //     diffusion operator curl muinv curl + delta I, by adding the
-   //     curl-curl and the mass domain integrators. For standard
-   //     magnetostatics equations choose delta << 1. Larger values of
-   //     delta should make the linear system easier to solve at the
-   //     expense of resembling a diffusive quasistatic magnetic field.
-   //     A reasonable balance must be found whenever the mesh or problem
-   //     setup is altered.
+   // 11. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + delta I, by adding the curl-curl and the
+   //     mass domain integrators. For standard magnetostatics equations choose
+   //     delta << 1. Larger values of delta should make the linear system
+   //     easier to solve at the expense of resembling a diffusive quasistatic
+   //     magnetic field.  A reasonable balance must be found whenever the mesh
+   //     or problem setup is altered.
   ConstantCoefficient muinv(1.0);
   ConstantCoefficient delta(delta_const);
   BilinearForm a(&fespace_nd);
@@ -423,8 +422,8 @@ int main(int argc, char *argv[])
   GridFunction dx(&fespace_rt);
   curl.Mult(x, dx);

-   // 18. Save the curl of the solution in parallel. This output can
-   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g dsol".
+   // 18. Save the curl of the solution in parallel. This output can be viewed
+   //     later using GLVis: "glvis -np <np> -m mesh -g dsol".
   {
      ostringstream dsol_name;
      dsol_name << "dsol.gf";
@@ -456,18 +455,18 @@ void ComputeCurrentDensityOnSubMesh(int order,
                                    const Array<int> &jn_zero_attr,
                                    GridFunction &j_cond)
 {
-   // Exract the finite element space and mesh on which j_cond is defined
+   // Extract the finite element space and mesh on which j_cond is defined
   FiniteElementSpace &fes_cond_rt = *j_cond.FESpace();
   Mesh &mesh_cond = *fes_cond_rt.GetMesh();
   int dim  = mesh_cond.Dimension();

-   // Define a parallel finite element space on the SubMesh. Here we use the
-   // H1 finite elements for the electrostatic potential.
+   // Define a parallel finite element space on the SubMesh. Here we use the H1
+   // finite elements for the electrostatic potential.
   H1_FECollection fec_h1(order, dim);
   FiniteElementSpace fes_cond_h1(&mesh_cond, &fec_h1);

-   // Define the conductivity coefficient and the boundaries associated with
-   // the fixed potentials phi0 and phi1 which will drive the current.
+   // Define the conductivity coefficient and the boundaries associated with the
+   // fixed potentials phi0 and phi1 which will drive the current.
   ConstantCoefficient sigmaCoef(1.0);
   Array<int> ess_bdr_phi(mesh_cond.bdr_attributes.Max());
   Array<int> ess_bdr_j(mesh_cond.bdr_attributes.Max());
@@ -578,9 +577,9 @@ void ComputeCurrentDensityOnSubMesh(int order,
                << "window_geometry 0 0 400 350" << flush;
   }

-   // Solve for the current density J = -sigma Grad phi with boundary
-   // conditions J.n = 0 on the walls of the conductor but not on the
-   // ports where phi=0 and phi=1.
+   // Solve for the current density J = -sigma Grad phi with boundary conditions
+   // J.n = 0 on the walls of the conductor but not on the ports where phi=0 and
+   // phi=1.

   // J will be computed in H(div) so we need an RT mass matrix
   BilinearForm m_rt(&fes_cond_rt);
@@ -302,9 +302,9 @@ int main(int argc, char *argv[])
                << "window_geometry 400 0 400 350" << flush;
   }

-   // 8. Define a parallel finite element space on the full mesh. Here we
-   //    use the H(curl) finite elements for the vector potential and H(div)
-   //    for the current density.
+   // 8. Define a parallel finite element space on the full mesh. Here we use
+   //    the H(curl) finite elements for the vector potential and H(div) for the
+   //    current density.
   ND_FECollection fec_nd(order, dim);
   RT_FECollection fec_rt(order - 1, dim);
   ParFiniteElementSpace fespace_nd(&pmesh, &fec_nd);
@@ -360,14 +360,13 @@ int main(int argc, char *argv[])
   ParGridFunction x(&fespace_nd);
   x = 0.0;

-   // 12. Set up the parallel bilinear form corresponding to the EM
-   //     diffusion operator curl muinv curl + delta I, by adding the
-   //     curl-curl and the mass domain integrators. For standard
-   //     magnetostatics equations choose delta << 1. Larger values of
-   //     delta should make the linear system easier to solve at the
-   //     expense of resembling a diffusive quasistatic magnetic field.
-   //     A reasonable balance must be found whenever the mesh or problem
-   //     setup is altered.
+   // 12. Set up the parallel bilinear form corresponding to the EM diffusion
+   //     operator curl muinv curl + delta I, by adding the curl-curl and the
+   //     mass domain integrators. For standard magnetostatics equations choose
+   //     delta << 1. Larger values of delta should make the linear system
+   //     easier to solve at the expense of resembling a diffusive quasistatic
+   //     magnetic field.  A reasonable balance must be found whenever the mesh
+   //     or problem setup is altered.
   ConstantCoefficient muinv(1.0);
   ConstantCoefficient delta(delta_const);
   ParBilinearForm a(&fespace_nd);
@@ -504,7 +503,7 @@ void ComputeCurrentDensityOnSubMesh(int order,
                                    const Array<int> &jn_zero_attr,
                                    ParGridFunction &j_cond)
 {
-   // Exract the finite element space and mesh on which j_cond is defined
+   // Extract the finite element space and mesh on which j_cond is defined
   ParFiniteElementSpace &fes_cond_rt = *j_cond.ParFESpace();
   ParMesh &pmesh_cond = *fes_cond_rt.GetParMesh();
   int myid = fes_cond_rt.GetMyRank();
@@ -515,8 +514,8 @@ void ComputeCurrentDensityOnSubMesh(int order,
   H1_FECollection fec_h1(order, dim);
   ParFiniteElementSpace fes_cond_h1(&pmesh_cond, &fec_h1);

-   // Define the conductivity coefficient and the boundaries associated with
-   // the fixed potentials phi0 and phi1 which will drive the current.
+   // Define the conductivity coefficient and the boundaries associated with the
+   // fixed potentials phi0 and phi1 which will drive the current.
   ConstantCoefficient sigmaCoef(1.0);
   Array<int> ess_bdr_phi(pmesh_cond.bdr_attributes.Max());
   Array<int> ess_bdr_j(pmesh_cond.bdr_attributes.Max());
@@ -599,9 +598,9 @@ void ComputeCurrentDensityOnSubMesh(int order,
                << "window_geometry 0 0 400 350" << flush;
   }

-   // Solve for the current density J = -sigma Grad phi with boundary
-   // conditions J.n = 0 on the walls of the conductor but not on the
-   // ports where phi=0 and phi=1.
+   // Solve for the current density J = -sigma Grad phi with boundary conditions
+   // J.n = 0 on the walls of the conductor but not on the ports where phi=0 and
+   // phi=1.

   // J will be computed in H(div) so we need an RT mass matrix
   ParBilinearForm m_rt(&fes_cond_rt);
@@ -35,10 +35,10 @@
 //               conductivity, sigma = c. The user can specify these constants
 //               using either set of names.
 //
-//               This example demonstrates how to transfer fields computed on
-//               a boundary generated SubMesh to the full mesh and apply them
-//               as boundary conditions. The default mesh and corresponding
-//               boundary attriburtes were chosen to verify proper behavior on
+//               This example demonstrates how to transfer fields computed on a
+//               boundary generated SubMesh to the full mesh and apply them as
+//               boundary conditions. The default mesh and corresponding
+//               boundary attributes were chosen to verify proper behavior on
 //               both triangular and quadrilateral faces of tetrahedral,
 //               wedge-shaped, and hexahedral elements.
 //
@@ -420,7 +420,6 @@ int main(int argc, char *argv[])
      //
      //      2) A vector H(Div) field
      //         -Grad(a Div) - omega^2 b + i omega c
-      //
      ParBilinearForm pcOp(&fespace);
      if (pa) { pcOp.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
      switch (prob)
@@ -445,8 +444,8 @@ int main(int argc, char *argv[])
      pcOp.Assemble();

      // 14b. Define and apply a parallel FGMRES solver for AU=B with a block
-      //     diagonal preconditioner based on the appropriate multigrid
-      //     preconditioner from hypre.
+      //      diagonal preconditioner based on the appropriate multigrid
+      //      preconditioner from hypre.
      Array<int> blockTrueOffsets;
      blockTrueOffsets.SetSize(3);
      blockTrueOffsets[0] = 0;
@@ -609,10 +608,9 @@ int main(int argc, char *argv[])
 }

 /**
-   Solves the eigenvalue problem -Div(Grad x) = lambda x with
-   homogeneous Dirichlet boundary conditions on the boundary of the
-   domain. Returns mode number "mode" (counting from zero) in the
-   ParGridFunction "x".
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with homogeneous
+   Dirichlet boundary conditions on the boundary of the domain. Returns mode
+   number "mode" (counting from zero) in the ParGridFunction "x".
 */
 void ScalarWaveGuide(int mode, ParGridFunction &x)
 {
@@ -667,10 +665,10 @@ void ScalarWaveGuide(int mode, ParGridFunction &x)
 }

 /**
-   Solves the eigenvalue problem -Curl(Curl x) = lambda x with
-   homogeneous Dirichlet boundary conditions, on the tangential
-   component of x, on the boundary of the domain. Returns mode number
-   "mode" (counting from zero) in the ParGridFunction "x".
+   Solves the eigenvalue problem -Curl(Curl x) = lambda x with homogeneous
+   Dirichlet boundary conditions, on the tangential component of x, on the
+   boundary of the domain. Returns mode number "mode" (counting from zero) in
+   the ParGridFunction "x".
 */
 void VectorWaveGuide(int mode, ParGridFunction &x)
 {
@@ -723,13 +721,12 @@ void VectorWaveGuide(int mode, ParGridFunction &x)
 }

 /**
-   Solves the eigenvalue problem -Div(Grad x) = lambda x with
-   homogeneous Neumann boundary conditions on the boundary of the
-   domain. Returns mode number "mode" (counting from zero) in the
-   ParGridFunction "x_l2". Note that mode 0 is a constant field so
-   higher mode numbers are often more interesting. The eigenmode is
-   solved using continuous H1 basis of the appropriate order and then
-   projected onto the L2 basis and returned.
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with homogeneous
+   Neumann boundary conditions on the boundary of the domain. Returns mode
+   number "mode" (counting from zero) in the ParGridFunction "x_l2". Note that
+   mode 0 is a constant field so higher mode numbers are often more
+   interesting. The eigenmode is solved using continuous H1 basis of the
+   appropriate order and then projected onto the L2 basis and returned.
 */
 void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
 {
@@ -791,9 +788,9 @@ void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
   delete M;
 }

-// Compute eigenmode "mode" of either a Dirichlet or Neumann Laplacian
-// or of a Dirichlet curl curl operator based on the problem type and
-// dimension of the domain.
+// Compute eigenmode "mode" of either a Dirichlet or Neumann Laplacian or of a
+// Dirichlet curl curl operator based on the problem type and dimension of the
+// domain.
 void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc)
 {
   switch (prob)
@@ -1,12 +1,10 @@
 //                                MFEM Example 36
 //
-//
 // Compile with: make ex36
 //
 // Sample runs: ex36 -o 2
 //              ex36 -o 2 -r 4
 //
-//
 // Description: This example code demonstrates the use of MFEM to solve the
 //              bound-constrained energy minimization problem
 //
@@ -28,12 +26,10 @@
 //              order solutions to variation inequality problems and
 //              showcases how to set up and solve nonlinear mixed methods.
 //
-//
 // [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
 //     preserving finite element method for pointwise bound constraints.
 //     arXiv:2307.12444 [math.NA]

-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -63,7 +59,7 @@ public:
 class ExponentialGridFunctionCoefficient : public Coefficient
 {
 protected:
-   GridFunction *u; // grid function
+   GridFunction *u;
   Coefficient *obstacle;
   double min_val;
   double max_val;
@@ -88,7 +84,7 @@ int main(int argc, char *argv[])

   OptionsParser args(argc, argv);
   args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree)");
+                  "Finite element order (polynomial degree).");
   args.AddOption(&ref_levels, "-r", "--refs",
                  "Number of h-refinements.");
   args.AddOption(&max_it, "-mi", "--max-it",
@@ -198,7 +194,7 @@ int main(int argc, char *argv[])
   u_gf.ProjectCoefficient(IC_coef);
   u_old_gf = u_gf;

-   // 9. Initialize the slack variable ψₕ = exp(uₕ)
+   // 9. Initialize the slack variable ψₕ = ln(uₕ)
   LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
   psi_gf.ProjectCoefficient(ln_u);
   psi_old_gf = psi_gf;
@@ -1,12 +1,10 @@
-//                         MFEM Example 36 - Parallel Version
-//
+//                       MFEM Example 36 - Parallel Version
 //
 // Compile with: make ex36p
 //
 // Sample runs: mpirun -np 4 ex36p -o 2
 //              mpirun -np 4 ex36p -o 2 -r 4
 //
-//
 // Description: This example code demonstrates the use of MFEM to solve the
 //              bound-constrained energy minimization problem
 //
@@ -28,12 +26,10 @@
 //              order solutions to variation inequality problems and
 //              showcases how to set up and solve nonlinear mixed methods.
 //
-//
 // [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
 //     preserving finite element method for pointwise bound constraints.
 //     arXiv:2307.12444 [math.NA]

-
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -63,7 +59,7 @@ public:
 class ExponentialGridFunctionCoefficient : public Coefficient
 {
 protected:
-   GridFunction *u; // grid function
+   GridFunction *u;
   Coefficient *obstacle;
   double min_val;
   double max_val;
@@ -220,7 +216,6 @@ int main(int argc, char *argv[])
   u_old_gf = 0.0;
   psi_old_gf = 0.0;

-
   // 8. Define the function coefficients for the solution and use them to
   //    initialize the initial guess
   FunctionCoefficient exact_coef(exact_solution_obstacle);
@@ -231,7 +226,7 @@ int main(int argc, char *argv[])
   u_gf.ProjectCoefficient(IC_coef);
   u_old_gf = u_gf;

-   // 9. Initialize the slack variable ψₕ = exp(uₕ)
+   // 9. Initialize the slack variable ψₕ = ln(uₕ)
   LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
   psi_gf.ProjectCoefficient(ln_u);
   psi_old_gf = psi_gf;
@@ -0,0 +1,466 @@
+//                                MFEM Example 37
+//
+// Compile with: make ex37
+//
+// Sample runs:
+//     ex37 -alpha 10
+//     ex37 -alpha 10 -pv
+//     ex37 -lambda 0.1 -mu 0.1
+//     ex37 -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
+//     ex37 -r 6 -o 1 -alpha 25.0 -epsilon 0.02 -mi 50 -ntol 1e-5
+//
+// Description: This example code demonstrates the use of MFEM to solve a
+//              density-filtered [3] topology optimization problem. The
+//              objective is to minimize the compliance
+//
+//                  minimize ∫_Ω f⋅u dx over u ∈ [H¹(Ω)]² and ρ ∈ L¹(Ω)
+//
+//                  subject to
+//
+//                    -Div(r(ρ̃)Cε(u)) = f       in Ω + BCs
+//                    -ϵ²Δρ̃ + ρ̃ = ρ             in Ω + Neumann BCs
+//                    0 ≤ ρ ≤ 1                 in Ω
+//                    ∫_Ω ρ dx = θ vol(Ω)
+//
+//              Here, r(ρ̃) = ρ₀ + ρ̃³ (1-ρ₀) is the solid isotropic material
+//              penalization (SIMP) law, C is the elasticity tensor for an
+//              isotropic linearly elastic material, ϵ > 0 is the design
+//              length scale, and 0 < θ < 1 is the volume fraction.
+//
+//              The problem is discretized and gradients are computing using
+//              finite elements [1]. The design is optimized using an entropic
+//              mirror descent algorithm introduced by Keith and Surowiec [2]
+//              that is tailored to the bound constraint 0 ≤ ρ ≤ 1.
+//
+//              This example highlights the ability of MFEM to deliver high-
+//              order solutions to inverse design problems and showcases how
+//              to set up and solve PDE-constrained optimization problems
+//              using the so-called reduced space approach.
+//
+// [1] Andreassen, E., Clausen, A., Schevenels, M., Lazarov, B. S., & Sigmund, O.
+//     (2011). Efficient topology optimization in MATLAB using 88 lines of
+//     code. Structural and Multidisciplinary Optimization, 43(1), 1-16.
+// [2] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//     preserving finite element method for pointwise bound constraints.
+//     arXiv:2307.12444 [math.NA]
+// [3] Lazarov, B. S., & Sigmund, O. (2011). Filters in topology optimization
+//     based on Helmholtz‐type differential equations. International Journal
+//     for Numerical Methods in Engineering, 86(6), 765-781.
+
+#include "mfem.hpp"
+#include <iostream>
+#include <fstream>
+#include "ex37.hpp"
+
+using namespace std;
+using namespace mfem;
+
+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return double Final volume, ∫_Ω sigmoid(ψ)
+ */
+double proj(GridFunction &psi, double target_volume, double tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   LinearForm int_sigmoid_psi(psi.FESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   LinearForm int_der_sigmoid_psi(psi.FESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      const double f = int_sigmoid_psi.Sum() - target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      const double df = int_der_sigmoid_psi.Sum();
+
+      const double dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   return int_sigmoid_psi.Sum();
+}
+
+/**
+ * ---------------------------------------------------------------
+ *                      ALGORITHM PREAMBLE
+ * ---------------------------------------------------------------
+ *
+ *  The Lagrangian for this problem is
+ *
+ *          L(u,ρ,ρ̃,w,w̃) = (f,u) - (r(ρ̃) C ε(u),ε(w)) + (f,w)
+ *                       - (ϵ² ∇ρ̃,∇w̃) - (ρ̃,w̃) + (ρ,w̃)
+ *
+ *  where
+ *
+ *    r(ρ̃) = ρ₀ + ρ̃³ (1 - ρ₀)       (SIMP rule)
+ *
+ *    ε(u) = (∇u + ∇uᵀ)/2           (symmetric gradient)
+ *
+ *    C e = λtr(e)I + 2μe           (isotropic material)
+ *
+ *  NOTE: The Lame parameters can be computed from Young's modulus E
+ *        and Poisson's ratio ν as follows:
+ *
+ *             λ = E ν/((1+ν)(1-2ν)),      μ = E/(2(1+ν))
+ *
+ * ---------------------------------------------------------------
+ *
+ *  Discretization choices:
+ *
+ *     u ∈ V ⊂ (H¹)ᵈ (order p)
+ *     ψ ∈ L² (order p - 1), ρ = sigmoid(ψ)
+ *     ρ̃ ∈ H¹ (order p)
+ *     w ∈ V  (order p)
+ *     w̃ ∈ H¹ (order p)
+ *
+ * ---------------------------------------------------------------
+ *                          ALGORITHM
+ * ---------------------------------------------------------------
+ *
+ *  Update ρ with projected mirror descent via the following algorithm.
+ *
+ *  1. Initialize ψ = inv_sigmoid(vol_fraction) so that ∫ sigmoid(ψ) = θ vol(Ω)
+ *
+ *  While not converged:
+ *
+ *     2. Solve filter equation ∂_w̃ L = 0; i.e.,
+ *
+ *           (ϵ² ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)   ∀ v ∈ H¹.
+ *
+ *     3. Solve primal problem ∂_w L = 0; i.e.,
+ *
+ *      (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)   ∀ v ∈ V.
+ *
+ *     NB. The dual problem ∂_u L = 0 is the negative of the primal problem due to symmetry.
+ *
+ *     4. Solve for filtered gradient ∂_ρ̃ L = 0; i.e.,
+ *
+ *      (ϵ² ∇ w̃ , ∇ v ) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)   ∀ v ∈ H¹.
+ *
+ *     5. Project the gradient onto the discrete latent space; i.e., solve
+ *
+ *                         (G,v) = (w̃,v)   ∀ v ∈ L².
+ *
+ *     6. Bregman proximal gradient update; i.e.,
+ *
+ *                            ψ ← ψ - αG + c,
+ *
+ *     where α > 0 is a step size parameter and c ∈ R is a constant ensuring
+ *
+ *                     ∫_Ω sigmoid(ψ - αG + c) dx = θ vol(Ω).
+ *
+ *  end
+ */
+
+int main(int argc, char *argv[])
+{
+   // 1. Parse command-line options.
+   int ref_levels = 5;
+   int order = 2;
+   double alpha = 1.0;
+   double epsilon = 0.01;
+   double vol_fraction = 0.5;
+   int max_it = 1e3;
+   double itol = 1e-1;
+   double ntol = 1e-4;
+   double rho_min = 1e-6;
+   double lambda = 1.0;
+   double mu = 1.0;
+   bool glvis_visualization = true;
+   bool paraview_output = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&alpha, "-alpha", "--alpha-step-length",
+                  "Step length for gradient descent.");
+   args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
+                  "Length scale for ρ.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of gradient descent iterations.");
+   args.AddOption(&ntol, "-ntol", "--rel-tol",
+                  "Normalized exit tolerance.");
+   args.AddOption(&itol, "-itol", "--abs-tol",
+                  "Increment exit tolerance.");
+   args.AddOption(&vol_fraction, "-vf", "--volume-fraction",
+                  "Volume fraction for the material density.");
+   args.AddOption(&lambda, "-lambda", "--lambda",
+                  "Lamé constant λ.");
+   args.AddOption(&mu, "-mu", "--mu",
+                  "Lamé constant μ.");
+   args.AddOption(&rho_min, "-rmin", "--psi-min",
+                  "Minimum of density coefficient.");
+   args.AddOption(&glvis_visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview_output, "-pv", "--paraview", "-no-pv",
+                  "--no-paraview",
+                  "Enable or disable ParaView output.");
+   args.Parse();
+   if (!args.Good())
+   {
+      args.PrintUsage(mfem::out);
+      return 1;
+   }
+   args.PrintOptions(mfem::out);
+
+   Mesh mesh = Mesh::MakeCartesian2D(3, 1, mfem::Element::Type::QUADRILATERAL,
+                                     true, 3.0, 1.0);
+   int dim = mesh.Dimension();
+
+   // 2. Set BCs.
+   for (int i = 0; i<mesh.GetNBE(); i++)
+   {
+      Element * be = mesh.GetBdrElement(i);
+      Array<int> vertices;
+      be->GetVertices(vertices);
+
+      double * coords1 = mesh.GetVertex(vertices[0]);
+      double * coords2 = mesh.GetVertex(vertices[1]);
+
+      Vector center(2);
+      center(0) = 0.5*(coords1[0] + coords2[0]);
+      center(1) = 0.5*(coords1[1] + coords2[1]);
+
+      if (abs(center(0) - 0.0) < 1e-10)
+      {
+         // the left edge
+         be->SetAttribute(1);
+      }
+      else
+      {
+         // all other boundaries
+         be->SetAttribute(2);
+      }
+   }
+   mesh.SetAttributes();
+
+   // 3. Refine the mesh.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection state_fec(order, dim); // space for u
+   H1_FECollection filter_fec(order, dim); // space for ρ̃
+   L2_FECollection control_fec(order-1, dim,
+                               BasisType::GaussLobatto); // space for ψ
+   FiniteElementSpace state_fes(&mesh, &state_fec,dim);
+   FiniteElementSpace filter_fes(&mesh, &filter_fec);
+   FiniteElementSpace control_fes(&mesh, &control_fec);
+
+   int state_size = state_fes.GetTrueVSize();
+   int control_size = control_fes.GetTrueVSize();
+   int filter_size = filter_fes.GetTrueVSize();
+   mfem::out << "Number of state unknowns: " << state_size << std::endl;
+   mfem::out << "Number of filter unknowns: " << filter_size << std::endl;
+   mfem::out << "Number of control unknowns: " << control_size << std::endl;
+
+   // 5. Set the initial guess for ρ.
+   GridFunction u(&state_fes);
+   GridFunction psi(&control_fes);
+   GridFunction psi_old(&control_fes);
+   GridFunction rho_filter(&filter_fes);
+   u = 0.0;
+   rho_filter = vol_fraction;
+   psi = inv_sigmoid(vol_fraction);
+   psi_old = inv_sigmoid(vol_fraction);
+
+   // ρ = sigmoid(ψ)
+   MappedGridFunctionCoefficient rho(&psi, sigmoid);
+   // Interpolation of ρ = sigmoid(ψ) in control fes (for ParaView output)
+   GridFunction rho_gf(&control_fes);
+   // ρ - ρ_old = sigmoid(ψ) - sigmoid(ψ_old)
+   DiffMappedGridFunctionCoefficient succ_diff_rho(&psi, &psi_old, sigmoid);
+
+   // 6. Set-up the physics solver.
+   int maxat = mesh.bdr_attributes.Max();
+   Array<int> ess_bdr(maxat);
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient lambda_cf(lambda);
+   ConstantCoefficient mu_cf(mu);
+   LinearElasticitySolver * ElasticitySolver = new LinearElasticitySolver();
+   ElasticitySolver->SetMesh(&mesh);
+   ElasticitySolver->SetOrder(state_fec.GetOrder());
+   ElasticitySolver->SetupFEM();
+   Vector center(2); center(0) = 2.9; center(1) = 0.5;
+   Vector force(2); force(0) = 0.0; force(1) = -1.0;
+   double r = 0.05;
+   VolumeForceCoefficient vforce_cf(r,center,force);
+   ElasticitySolver->SetRHSCoefficient(&vforce_cf);
+   ElasticitySolver->SetEssentialBoundary(ess_bdr);
+
+   // 7. Set-up the filter solver.
+   ConstantCoefficient eps2_cf(epsilon*epsilon);
+   DiffusionSolver * FilterSolver = new DiffusionSolver();
+   FilterSolver->SetMesh(&mesh);
+   FilterSolver->SetOrder(filter_fec.GetOrder());
+   FilterSolver->SetDiffusionCoefficient(&eps2_cf);
+   FilterSolver->SetMassCoefficient(&one);
+   Array<int> ess_bdr_filter;
+   if (mesh.bdr_attributes.Size())
+   {
+      ess_bdr_filter.SetSize(mesh.bdr_attributes.Max());
+      ess_bdr_filter = 0;
+   }
+   FilterSolver->SetEssentialBoundary(ess_bdr_filter);
+   FilterSolver->SetupFEM();
+
+   BilinearForm mass(&control_fes);
+   mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
+   mass.Assemble();
+   SparseMatrix M;
+   Array<int> empty;
+   mass.FormSystemMatrix(empty,M);
+
+   // 8. Define the Lagrange multiplier and gradient functions.
+   GridFunction grad(&control_fes);
+   GridFunction w_filter(&filter_fes);
+
+   // 9. Define some tools for later.
+   ConstantCoefficient zero(0.0);
+   GridFunction onegf(&control_fes);
+   onegf = 1.0;
+   GridFunction zerogf(&control_fes);
+   zerogf = 0.0;
+   LinearForm vol_form(&control_fes);
+   vol_form.AddDomainIntegrator(new DomainLFIntegrator(one));
+   vol_form.Assemble();
+   double domain_volume = vol_form(onegf);
+   const double target_volume = domain_volume * vol_fraction;
+
+   // 10. Connect to GLVis. Prepare for VisIt output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sout_r;
+   if (glvis_visualization)
+   {
+      sout_r.open(vishost, visport);
+      sout_r.precision(8);
+   }
+
+   mfem::ParaViewDataCollection paraview_dc("ex37", &mesh);
+   if (paraview_output)
+   {
+      rho_gf.ProjectCoefficient(rho);
+      paraview_dc.SetPrefixPath("ParaView");
+      paraview_dc.SetLevelsOfDetail(order);
+      paraview_dc.SetDataFormat(VTKFormat::BINARY);
+      paraview_dc.SetHighOrderOutput(true);
+      paraview_dc.SetCycle(0);
+      paraview_dc.SetTime(0.0);
+      paraview_dc.RegisterField("displacement",&u);
+      paraview_dc.RegisterField("density",&rho_gf);
+      paraview_dc.RegisterField("filtered_density",&rho_filter);
+      paraview_dc.Save();
+   }
+
+   // 11. Iterate:
+   for (int k = 1; k <= max_it; k++)
+   {
+      if (k > 1) { alpha *= ((double) k) / ((double) k-1); }
+
+      mfem::out << "\nStep = " << k << std::endl;
+
+      // Step 1 - Filter solve
+      // Solve (ϵ^2 ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)
+      FilterSolver->SetRHSCoefficient(&rho);
+      FilterSolver->Solve();
+      rho_filter = *FilterSolver->GetFEMSolution();
+
+      // Step 2 - State solve
+      // Solve (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)
+      SIMPInterpolationCoefficient SIMP_cf(&rho_filter,rho_min, 1.0);
+      ProductCoefficient lambda_SIMP_cf(lambda_cf,SIMP_cf);
+      ProductCoefficient mu_SIMP_cf(mu_cf,SIMP_cf);
+      ElasticitySolver->SetLameCoefficients(&lambda_SIMP_cf,&mu_SIMP_cf);
+      ElasticitySolver->Solve();
+      u = *ElasticitySolver->GetFEMSolution();
+
+      // Step 3 - Adjoint filter solve
+      // Solve (ϵ² ∇ w̃, ∇ v) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)
+      StrainEnergyDensityCoefficient rhs_cf(&lambda_cf,&mu_cf,&u, &rho_filter,
+                                            rho_min);
+      FilterSolver->SetRHSCoefficient(&rhs_cf);
+      FilterSolver->Solve();
+      w_filter = *FilterSolver->GetFEMSolution();
+
+      // Step 4 - Compute gradient
+      // Solve G = M⁻¹w̃
+      GridFunctionCoefficient w_cf(&w_filter);
+      LinearForm w_rhs(&control_fes);
+      w_rhs.AddDomainIntegrator(new DomainLFIntegrator(w_cf));
+      w_rhs.Assemble();
+      M.Mult(w_rhs,grad);
+
+      // Step 5 - Update design variable ψ ← proj(ψ - αG)
+      psi.Add(-alpha, grad);
+      const double material_volume = proj(psi, target_volume);
+
+      // Compute ||ρ - ρ_old|| in control fes.
+      double norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
+      double norm_reduced_gradient = norm_increment/alpha;
+      psi_old = psi;
+
+      double compliance = (*(ElasticitySolver->GetLinearForm()))(u);
+      mfem::out << "norm of the reduced gradient = " << norm_reduced_gradient <<
+                std::endl;
+      mfem::out << "norm of the increment = " << norm_increment << endl;
+      mfem::out << "compliance = " << compliance << std::endl;
+      mfem::out << "volume fraction = " << material_volume / domain_volume <<
+                std::endl;
+
+      if (glvis_visualization)
+      {
+         GridFunction r_gf(&filter_fes);
+         r_gf.ProjectCoefficient(SIMP_cf);
+         sout_r << "solution\n" << mesh << r_gf
+                << "window_title 'Design density r(ρ̃)'" << flush;
+      }
+
+      if (paraview_output)
+      {
+         rho_gf.ProjectCoefficient(rho);
+         paraview_dc.SetCycle(k);
+         paraview_dc.SetTime((double)k);
+         paraview_dc.Save();
+      }
+
+      if (norm_reduced_gradient < ntol && norm_increment < itol)
+      {
+         break;
+      }
+   }
+
+   delete ElasticitySolver;
+   delete FilterSolver;
+
+   return 0;
+}
@@ -0,0 +1,748 @@
+//                  MFEM Example 37 - Serial/Parallel Shared Code
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <functional>
+
+namespace mfem
+{
+
+/// @brief Inverse sigmoid function
+double inv_sigmoid(double x)
+{
+   double tol = 1e-12;
+   x = std::min(std::max(tol,x),1.0-tol);
+   return std::log(x/(1.0-x));
+}
+
+/// @brief Sigmoid function
+double sigmoid(double x)
+{
+   if (x >= 0)
+   {
+      return 1.0/(1.0+std::exp(-x));
+   }
+   else
+   {
+      return std::exp(x)/(1.0+std::exp(x));
+   }
+}
+
+/// @brief Derivative of sigmoid function
+double der_sigmoid(double x)
+{
+   double tmp = sigmoid(-x);
+   return tmp - std::pow(tmp,2);
+}
+
+/// @brief Returns f(u(x)) where u is a scalar GridFunction and f:R → R
+class MappedGridFunctionCoefficient : public GridFunctionCoefficient
+{
+protected:
+   std::function<double(const double)> fun; // f:R → R
+public:
+   MappedGridFunctionCoefficient()
+      :GridFunctionCoefficient(),
+       fun([](double x) {return x;}) {}
+   MappedGridFunctionCoefficient(const GridFunction *gf,
+                                 std::function<double(const double)> fun_,
+                                 int comp=1)
+      :GridFunctionCoefficient(gf, comp),
+       fun(fun_) {}
+
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
+   {
+      return fun(GridFunctionCoefficient::Eval(T, ip));
+   }
+   void SetFunction(std::function<double(const double)> fun_) { fun = fun_; }
+};
+
+
+/// @brief Returns f(u(x)) - f(v(x)) where u, v are scalar GridFunctions and f:R → R
+class DiffMappedGridFunctionCoefficient : public GridFunctionCoefficient
+{
+protected:
+   const GridFunction *OtherGridF;
+   GridFunctionCoefficient OtherGridF_cf;
+   std::function<double(const double)> fun; // f:R → R
+public:
+   DiffMappedGridFunctionCoefficient()
+      :GridFunctionCoefficient(),
+       OtherGridF(nullptr),
+       OtherGridF_cf(),
+       fun([](double x) {return x;}) {}
+   DiffMappedGridFunctionCoefficient(const GridFunction *gf,
+                                     const GridFunction *other_gf,
+                                     std::function<double(const double)> fun_,
+                                     int comp=1)
+      :GridFunctionCoefficient(gf, comp),
+       OtherGridF(other_gf),
+       OtherGridF_cf(OtherGridF),
+       fun(fun_) {}
+
+   virtual double Eval(ElementTransformation &T,
+                       const IntegrationPoint &ip)
+   {
+      const double value1 = fun(GridFunctionCoefficient::Eval(T, ip));
+      const double value2 = fun(OtherGridF_cf.Eval(T, ip));
+      return value1 - value2;
+   }
+   void SetFunction(std::function<double(const double)> fun_) { fun = fun_; }
+};
+
+/// @brief Solid isotropic material penalization (SIMP) coefficient
+class SIMPInterpolationCoefficient : public Coefficient
+{
+protected:
+   GridFunction *rho_filter;
+   double min_val;
+   double max_val;
+   double exponent;
+
+public:
+   SIMPInterpolationCoefficient(GridFunction *rho_filter_, double min_val_= 1e-6,
+                                double max_val_ = 1.0, double exponent_ = 3)
+      : rho_filter(rho_filter_), min_val(min_val_), max_val(max_val_),
+        exponent(exponent_) { }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double val = rho_filter->GetValue(T, ip);
+      double coeff = min_val + pow(val,exponent)*(max_val-min_val);
+      return coeff;
+   }
+};
+
+
+/// @brief Strain energy density coefficient
+class StrainEnergyDensityCoefficient : public Coefficient
+{
+protected:
+   Coefficient * lambda=nullptr;
+   Coefficient * mu=nullptr;
+   GridFunction *u = nullptr; // displacement
+   GridFunction *rho_filter = nullptr; // filter density
+   DenseMatrix grad; // auxiliary matrix, used in Eval
+   double exponent;
+   double rho_min;
+
+public:
+   StrainEnergyDensityCoefficient(Coefficient *lambda_, Coefficient *mu_,
+                                  GridFunction * u_, GridFunction * rho_filter_, double rho_min_=1e-6,
+                                  double exponent_ = 3.0)
+      : lambda(lambda_), mu(mu_),  u(u_), rho_filter(rho_filter_),
+        exponent(exponent_), rho_min(rho_min_)
+   {
+      MFEM_ASSERT(rho_min_ >= 0.0, "rho_min must be >= 0");
+      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be > 1");
+      MFEM_ASSERT(u, "displacement field is not set");
+      MFEM_ASSERT(rho_filter, "density field is not set");
+   }
+
+   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
+   {
+      double L = lambda->Eval(T, ip);
+      double M = mu->Eval(T, ip);
+      u->GetVectorGradient(T, grad);
+      double div_u = grad.Trace();
+      double density = L*div_u*div_u;
+      int dim = T.GetSpaceDim();
+      for (int i=0; i<dim; i++)
+      {
+         for (int j=0; j<dim; j++)
+         {
+            density += M*grad(i,j)*(grad(i,j)+grad(j,i));
+         }
+      }
+      double val = rho_filter->GetValue(T,ip);
+
+      return -exponent * pow(val, exponent-1.0) * (1-rho_min) * density;
+   }
+};
+
+/// @brief Volumetric force for linear elasticity
+class VolumeForceCoefficient : public VectorCoefficient
+{
+private:
+   double r;
+   Vector center;
+   Vector force;
+public:
+   VolumeForceCoefficient(double r_,Vector &  center_, Vector & force_) :
+      VectorCoefficient(center_.Size()), r(r_), center(center_), force(force_) { }
+
+   using VectorCoefficient::Eval;
+
+   virtual void Eval(Vector &V, ElementTransformation &T,
+                     const IntegrationPoint &ip)
+   {
+      Vector xx; xx.SetSize(T.GetDimension());
+      T.Transform(ip,xx);
+      for (int i=0; i<xx.Size(); i++)
+      {
+         xx[i]=xx[i]-center[i];
+      }
+
+      double cr=xx.Norml2();
+      V.SetSize(T.GetDimension());
+      if (cr <= r)
+      {
+         V = force;
+      }
+      else
+      {
+         V = 0.0;
+      }
+   }
+
+   void Set(double r_,Vector & center_, Vector & force_)
+   {
+      r=r_;
+      center = center_;
+      force = force_;
+   }
+};
+
+/**
+ * @brief Class for solving Poisson's equation:
+ *
+ *       - ∇ ⋅(κ ∇ u) = f  in Ω
+ *
+ */
+class DiffusionSolver
+{
+private:
+   Mesh * mesh = nullptr;
+   int order = 1;
+   // diffusion coefficient
+   Coefficient * diffcf = nullptr;
+   // mass coefficient
+   Coefficient * masscf = nullptr;
+   Coefficient * rhscf = nullptr;
+   Coefficient * essbdr_cf = nullptr;
+   Coefficient * neumann_cf = nullptr;
+   VectorCoefficient * gradient_cf = nullptr;
+
+   // FEM solver
+   int dim;
+   FiniteElementCollection * fec = nullptr;
+   FiniteElementSpace * fes = nullptr;
+   Array<int> ess_bdr;
+   Array<int> neumann_bdr;
+   GridFunction * u = nullptr;
+   LinearForm * b = nullptr;
+   bool parallel;
+#ifdef MFEM_USE_MPI
+   ParMesh * pmesh = nullptr;
+   ParFiniteElementSpace * pfes = nullptr;
+#endif
+
+public:
+   DiffusionSolver() { }
+   DiffusionSolver(Mesh * mesh_, int order_, Coefficient * diffcf_,
+                   Coefficient * cf_);
+
+   void SetMesh(Mesh * mesh_)
+   {
+      mesh = mesh_;
+      parallel = false;
+#ifdef MFEM_USE_MPI
+      pmesh = dynamic_cast<ParMesh *>(mesh);
+      if (pmesh) { parallel = true; }
+#endif
+   }
+   void SetOrder(int order_) { order = order_ ; }
+   void SetDiffusionCoefficient(Coefficient * diffcf_) { diffcf = diffcf_; }
+   void SetMassCoefficient(Coefficient * masscf_) { masscf = masscf_; }
+   void SetRHSCoefficient(Coefficient * rhscf_) { rhscf = rhscf_; }
+   void SetEssentialBoundary(const Array<int> & ess_bdr_) { ess_bdr = ess_bdr_;};
+   void SetNeumannBoundary(const Array<int> & neumann_bdr_) { neumann_bdr = neumann_bdr_;};
+   void SetNeumannData(Coefficient * neumann_cf_) {neumann_cf = neumann_cf_;}
+   void SetEssBdrData(Coefficient * essbdr_cf_) {essbdr_cf = essbdr_cf_;}
+   void SetGradientData(VectorCoefficient * gradient_cf_) {gradient_cf = gradient_cf_;}
+
+   void ResetFEM();
+   void SetupFEM();
+
+   void Solve();
+   GridFunction * GetFEMSolution();
+   LinearForm * GetLinearForm() {return b;}
+#ifdef MFEM_USE_MPI
+   ParGridFunction * GetParFEMSolution();
+   ParLinearForm * GetParLinearForm()
+   {
+      if (parallel)
+      {
+         return dynamic_cast<ParLinearForm *>(b);
+      }
+      else
+      {
+         MFEM_ABORT("Wrong code path. Call GetLinearForm");
+         return nullptr;
+      }
+   }
+#endif
+
+   ~DiffusionSolver();
+
+};
+
+/**
+ * @brief Class for solving linear elasticity:
+ *
+ *        -∇ ⋅ σ(u) = f  in Ω  + BCs
+ *
+ *  where
+ *
+ *        σ(u) = λ ∇⋅u I + μ (∇ u + ∇uᵀ)
+ *
+ */
+class LinearElasticitySolver
+{
+private:
+   Mesh * mesh = nullptr;
+   int order = 1;
+   Coefficient * lambda_cf = nullptr;
+   Coefficient * mu_cf = nullptr;
+   VectorCoefficient * essbdr_cf = nullptr;
+   VectorCoefficient * rhs_cf = nullptr;
+
+   // FEM solver
+   int dim;
+   FiniteElementCollection * fec = nullptr;
+   FiniteElementSpace * fes = nullptr;
+   Array<int> ess_bdr;
+   Array<int> neumann_bdr;
+   GridFunction * u = nullptr;
+   LinearForm * b = nullptr;
+   bool parallel;
+#ifdef MFEM_USE_MPI
+   ParMesh * pmesh = nullptr;
+   ParFiniteElementSpace * pfes = nullptr;
+#endif
+
+public:
+   LinearElasticitySolver() { }
+   LinearElasticitySolver(Mesh * mesh_, int order_,
+                          Coefficient * lambda_cf_, Coefficient * mu_cf_);
+
+   void SetMesh(Mesh * mesh_)
+   {
+      mesh = mesh_;
+      parallel = false;
+#ifdef MFEM_USE_MPI
+      pmesh = dynamic_cast<ParMesh *>(mesh);
+      if (pmesh) { parallel = true; }
+#endif
+   }
+   void SetOrder(int order_) { order = order_ ; }
+   void SetLameCoefficients(Coefficient * lambda_cf_, Coefficient * mu_cf_) { lambda_cf = lambda_cf_; mu_cf = mu_cf_;  }
+   void SetRHSCoefficient(VectorCoefficient * rhs_cf_) { rhs_cf = rhs_cf_; }
+   void SetEssentialBoundary(const Array<int> & ess_bdr_) { ess_bdr = ess_bdr_;};
+   void SetNeumannBoundary(const Array<int> & neumann_bdr_) { neumann_bdr = neumann_bdr_;};
+   void SetEssBdrData(VectorCoefficient * essbdr_cf_) {essbdr_cf = essbdr_cf_;}
+
+   void ResetFEM();
+   void SetupFEM();
+
+   void Solve();
+   GridFunction * GetFEMSolution();
+   LinearForm * GetLinearForm() {return b;}
+#ifdef MFEM_USE_MPI
+   ParGridFunction * GetParFEMSolution();
+   ParLinearForm * GetParLinearForm()
+   {
+      if (parallel)
+      {
+         return dynamic_cast<ParLinearForm *>(b);
+      }
+      else
+      {
+         MFEM_ABORT("Wrong code path. Call GetLinearForm");
+         return nullptr;
+      }
+   }
+#endif
+
+   ~LinearElasticitySolver();
+
+};
+
+
+// Poisson solver
+
+DiffusionSolver::DiffusionSolver(Mesh * mesh_, int order_,
+                                 Coefficient * diffcf_, Coefficient * rhscf_)
+   : mesh(mesh_), order(order_), diffcf(diffcf_), rhscf(rhscf_)
+{
+
+#ifdef MFEM_USE_MPI
+   pmesh = dynamic_cast<ParMesh *>(mesh);
+   if (pmesh) { parallel = true; }
+#endif
+
+   SetupFEM();
+}
+
+void DiffusionSolver::SetupFEM()
+{
+   dim = mesh->Dimension();
+   fec = new H1_FECollection(order, dim);
+
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      pfes = new ParFiniteElementSpace(pmesh, fec);
+      u = new ParGridFunction(pfes);
+      b = new ParLinearForm(pfes);
+   }
+   else
+   {
+      fes = new FiniteElementSpace(mesh, fec);
+      u = new GridFunction(fes);
+      b = new LinearForm(fes);
+   }
+#else
+   fes = new FiniteElementSpace(mesh, fec);
+   u = new GridFunction(fes);
+   b = new LinearForm(fes);
+#endif
+   *u=0.0;
+
+   if (!ess_bdr.Size())
+   {
+      if (mesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh->bdr_attributes.Max());
+         ess_bdr = 1;
+      }
+   }
+}
+
+void DiffusionSolver::Solve()
+{
+   OperatorPtr A;
+   Vector B, X;
+   Array<int> ess_tdof_list;
+
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      pfes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+   }
+   else
+   {
+      fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+   }
+#else
+   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+#endif
+   *u=0.0;
+   if (b)
+   {
+      delete b;
+#ifdef MFEM_USE_MPI
+      if (parallel)
+      {
+         b = new ParLinearForm(pfes);
+      }
+      else
+      {
+         b = new LinearForm(fes);
+      }
+#else
+      b = new LinearForm(fes);
+#endif
+   }
+   if (rhscf)
+   {
+      b->AddDomainIntegrator(new DomainLFIntegrator(*rhscf));
+   }
+   if (neumann_cf)
+   {
+      MFEM_VERIFY(neumann_bdr.Size(), "neumann_bdr attributes not provided");
+      b->AddBoundaryIntegrator(new BoundaryLFIntegrator(*neumann_cf),neumann_bdr);
+   }
+   else if (gradient_cf)
+   {
+      MFEM_VERIFY(neumann_bdr.Size(), "neumann_bdr attributes not provided");
+      b->AddBoundaryIntegrator(new BoundaryNormalLFIntegrator(*gradient_cf),
+                               neumann_bdr);
+   }
+
+   b->Assemble();
+
+   BilinearForm * a = nullptr;
+
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      a = new ParBilinearForm(pfes);
+   }
+   else
+   {
+      a = new BilinearForm(fes);
+   }
+#else
+   a = new BilinearForm(fes);
+#endif
+   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
+   if (masscf)
+   {
+      a->AddDomainIntegrator(new MassIntegrator(*masscf));
+   }
+   a->Assemble();
+   if (essbdr_cf)
+   {
+      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
+   }
+   a->FormLinearSystem(ess_tdof_list, *u, *b, A, X, B);
+
+   CGSolver * cg = nullptr;
+   Solver * M = nullptr;
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      M = new HypreBoomerAMG;
+      dynamic_cast<HypreBoomerAMG*>(M)->SetPrintLevel(0);
+      cg = new CGSolver(pmesh->GetComm());
+   }
+   else
+   {
+      M = new GSSmoother((SparseMatrix&)(*A));
+      cg = new CGSolver;
+   }
+#else
+   M = new GSSmoother((SparseMatrix&)(*A));
+   cg = new CGSolver;
+#endif
+   cg->SetRelTol(1e-12);
+   cg->SetMaxIter(10000);
+   cg->SetPrintLevel(0);
+   cg->SetPreconditioner(*M);
+   cg->SetOperator(*A);
+   cg->Mult(B, X);
+   delete M;
+   delete cg;
+   a->RecoverFEMSolution(X, *b, *u);
+   delete a;
+}
+
+GridFunction * DiffusionSolver::GetFEMSolution()
+{
+   return u;
+}
+
+#ifdef MFEM_USE_MPI
+ParGridFunction * DiffusionSolver::GetParFEMSolution()
+{
+   if (parallel)
+   {
+      return dynamic_cast<ParGridFunction*>(u);
+   }
+   else
+   {
+      MFEM_ABORT("Wrong code path. Call GetFEMSolution");
+      return nullptr;
+   }
+}
+#endif
+
+DiffusionSolver::~DiffusionSolver()
+{
+   delete u; u = nullptr;
+   delete fes; fes = nullptr;
+#ifdef MFEM_USE_MPI
+   delete pfes; pfes=nullptr;
+#endif
+   delete fec; fec = nullptr;
+   delete b;
+}
+
+
+// Elasticity solver
+
+LinearElasticitySolver::LinearElasticitySolver(Mesh * mesh_, int order_,
+                                               Coefficient * lambda_cf_, Coefficient * mu_cf_)
+   : mesh(mesh_), order(order_), lambda_cf(lambda_cf_), mu_cf(mu_cf_)
+{
+#ifdef MFEM_USE_MPI
+   pmesh = dynamic_cast<ParMesh *>(mesh);
+   if (pmesh) { parallel = true; }
+#endif
+   SetupFEM();
+}
+
+void LinearElasticitySolver::SetupFEM()
+{
+   dim = mesh->Dimension();
+   fec = new H1_FECollection(order, dim,BasisType::Positive);
+
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      pfes = new ParFiniteElementSpace(pmesh, fec, dim);
+      u = new ParGridFunction(pfes);
+      b = new ParLinearForm(pfes);
+   }
+   else
+   {
+      fes = new FiniteElementSpace(mesh, fec,dim);
+      u = new GridFunction(fes);
+      b = new LinearForm(fes);
+   }
+#else
+   fes = new FiniteElementSpace(mesh, fec, dim);
+   u = new GridFunction(fes);
+   b = new LinearForm(fes);
+#endif
+   *u=0.0;
+
+   if (!ess_bdr.Size())
+   {
+      if (mesh->bdr_attributes.Size())
+      {
+         ess_bdr.SetSize(mesh->bdr_attributes.Max());
+         ess_bdr = 1;
+      }
+   }
+}
+
+void LinearElasticitySolver::Solve()
+{
+   GridFunction * x = nullptr;
+   OperatorPtr A;
+   Vector B, X;
+   Array<int> ess_tdof_list;
+
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      x = new ParGridFunction(pfes);
+      pfes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+   }
+   else
+   {
+      x = new GridFunction(fes);
+      fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+   }
+#else
+   x = new GridFunction(fes);
+   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
+#endif
+   *u=0.0;
+   if (b)
+   {
+      delete b;
+#ifdef MFEM_USE_MPI
+      if (parallel)
+      {
+         b = new ParLinearForm(pfes);
+      }
+      else
+      {
+         b = new LinearForm(fes);
+      }
+#else
+      b = new LinearForm(fes);
+#endif
+   }
+   if (rhs_cf)
+   {
+      b->AddDomainIntegrator(new VectorDomainLFIntegrator(*rhs_cf));
+   }
+
+   b->Assemble();
+
+   *x = 0.0;
+
+   BilinearForm * a = nullptr;
+
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      a = new ParBilinearForm(pfes);
+   }
+   else
+   {
+      a = new BilinearForm(fes);
+   }
+#else
+   a = new BilinearForm(fes);
+#endif
+   a->AddDomainIntegrator(new ElasticityIntegrator(*lambda_cf, *mu_cf));
+   a->Assemble();
+   if (essbdr_cf)
+   {
+      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
+   }
+   a->FormLinearSystem(ess_tdof_list, *x, *b, A, X, B);
+
+   CGSolver * cg = nullptr;
+   Solver * M = nullptr;
+#ifdef MFEM_USE_MPI
+   if (parallel)
+   {
+      M = new HypreBoomerAMG;
+      dynamic_cast<HypreBoomerAMG*>(M)->SetPrintLevel(0);
+      cg = new CGSolver(pmesh->GetComm());
+   }
+   else
+   {
+      M = new GSSmoother((SparseMatrix&)(*A));
+      cg = new CGSolver;
+   }
+#else
+   M = new GSSmoother((SparseMatrix&)(*A));
+   cg = new CGSolver;
+#endif
+   cg->SetRelTol(1e-10);
+   cg->SetMaxIter(10000);
+   cg->SetPrintLevel(0);
+   cg->SetPreconditioner(*M);
+   cg->SetOperator(*A);
+   cg->Mult(B, X);
+   delete M;
+   delete cg;
+   a->RecoverFEMSolution(X, *b, *x);
+   *u+=*x;
+   delete a;
+   delete x;
+}
+
+GridFunction * LinearElasticitySolver::GetFEMSolution()
+{
+   return u;
+}
+
+#ifdef MFEM_USE_MPI
+ParGridFunction * LinearElasticitySolver::GetParFEMSolution()
+{
+   if (parallel)
+   {
+      return dynamic_cast<ParGridFunction*>(u);
+   }
+   else
+   {
+      MFEM_ABORT("Wrong code path. Call GetFEMSolution");
+      return nullptr;
+   }
+}
+#endif
+
+LinearElasticitySolver::~LinearElasticitySolver()
+{
+   delete u; u = nullptr;
+   delete fes; fes = nullptr;
+#ifdef MFEM_USE_MPI
+   delete pfes; pfes=nullptr;
+#endif
+   delete fec; fec = nullptr;
+   delete b;
+}
+
+} // namespace mfem
+
@@ -0,0 +1,497 @@
+//                       MFEM Example 37 - Parallel Version
+//
+// Compile with: make ex37p
+//
+// Sample runs:
+//    mpirun -np 4 ex37p -alpha 10 -pv
+//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1
+//    mpirun -np 4 ex37p -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
+//    mpirun -np 4 ex37p -r 6 -o 2 -alpha 10.0 -epsilon 0.02 -mi 50 -ntol 1e-5
+//
+// Description: This example code demonstrates the use of MFEM to solve a
+//              density-filtered [3] topology optimization problem. The
+//              objective is to minimize the compliance
+//
+//                  minimize ∫_Ω f⋅u dx over u ∈ [H¹(Ω)]² and ρ ∈ L¹(Ω)
+//
+//                  subject to
+//
+//                    -Div(r(ρ̃)Cε(u)) = f       in Ω + BCs
+//                    -ϵ²Δρ̃ + ρ̃ = ρ             in Ω + Neumann BCs
+//                    0 ≤ ρ ≤ 1                 in Ω
+//                    ∫_Ω ρ dx = θ vol(Ω)
+//
+//              Here, r(ρ̃) = ρ₀ + ρ̃³ (1-ρ₀) is the solid isotropic material
+//              penalization (SIMP) law, C is the elasticity tensor for an
+//              isotropic linearly elastic material, ϵ > 0 is the design
+//              length scale, and 0 < θ < 1 is the volume fraction.
+//
+//              The problem is discretized and gradients are computing using
+//              finite elements [1]. The design is optimized using an entropic
+//              mirror descent algorithm introduced by Keith and Surowiec [2]
+//              that is tailored to the bound constraint 0 ≤ ρ ≤ 1.
+//
+//              This example highlights the ability of MFEM to deliver high-
+//              order solutions to inverse design problems and showcases how
+//              to set up and solve PDE-constrained optimization problems
+//              using the so-called reduced space approach.
+//
+// [1] Andreassen, E., Clausen, A., Schevenels, M., Lazarov, B. S., & Sigmund, O.
+//     (2011). Efficient topology optimization in MATLAB using 88 lines of
+//     code. Structural and Multidisciplinary Optimization, 43(1), 1-16.
+// [2] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
+//     preserving finite element method for pointwise bound constraints.
+//     arXiv:2307.12444 [math.NA]
+// [3] Lazarov, B. S., & Sigmund, O. (2011). Filters in topology optimization
+//     based on Helmholtz‐type differential equations. International Journal
+//     for Numerical Methods in Engineering, 86(6), 765-781.
+
+#include "mfem.hpp"
+#include <iostream>
+#include <fstream>
+#include "ex37.hpp"
+
+using namespace std;
+using namespace mfem;
+
+/**
+ * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
+ *        ∫_Ω ρ dx = θ vol(Ω) as follows:
+ *
+ *        1. Compute the root of the R → R function
+ *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
+ *        2. Set ψ ← ψ + c.
+ *
+ * @param psi a GridFunction to be updated
+ * @param target_volume θ vol(Ω)
+ * @param tol Newton iteration tolerance
+ * @param max_its Newton maximum iteration number
+ * @return double Final volume, ∫_Ω sigmoid(ψ)
+ */
+double proj(ParGridFunction &psi, double target_volume, double tol=1e-12,
+            int max_its=10)
+{
+   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
+   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
+
+   ParLinearForm int_sigmoid_psi(psi.ParFESpace());
+   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
+   ParLinearForm int_der_sigmoid_psi(psi.ParFESpace());
+   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
+                                              der_sigmoid_psi));
+   bool done = false;
+   for (int k=0; k<max_its; k++) // Newton iteration
+   {
+      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
+      double f = int_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+      f -= target_volume;
+
+      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
+      double df = int_der_sigmoid_psi.Sum();
+      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+      const double dc = -f/df;
+      psi += dc;
+      if (abs(dc) < tol) { done = true; break; }
+   }
+   if (!done)
+   {
+      mfem_warning("Projection reached maximum iteration without converging. "
+                   "Result may not be accurate.");
+   }
+   int_sigmoid_psi.Assemble();
+   double material_volume = int_sigmoid_psi.Sum();
+   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1, MPI_DOUBLE, MPI_SUM,
+                 MPI_COMM_WORLD);
+   return material_volume;
+}
+
+/**
+ * ---------------------------------------------------------------
+ *                      ALGORITHM PREAMBLE
+ * ---------------------------------------------------------------
+ *
+ *  The Lagrangian for this problem is
+ *
+ *          L(u,ρ,ρ̃,w,w̃) = (f,u) - (r(ρ̃) C ε(u),ε(w)) + (f,w)
+ *                       - (ϵ² ∇ρ̃,∇w̃) - (ρ̃,w̃) + (ρ,w̃)
+ *
+ *  where
+ *
+ *    r(ρ̃) = ρ₀ + ρ̃³ (1 - ρ₀)       (SIMP rule)
+ *
+ *    ε(u) = (∇u + ∇uᵀ)/2           (symmetric gradient)
+ *
+ *    C e = λtr(e)I + 2μe           (isotropic material)
+ *
+ *  NOTE: The Lame parameters can be computed from Young's modulus E
+ *        and Poisson's ratio ν as follows:
+ *
+ *             λ = E ν/((1+ν)(1-2ν)),      μ = E/(2(1+ν))
+ *
+ * ---------------------------------------------------------------
+ *
+ *  Discretization choices:
+ *
+ *     u ∈ V ⊂ (H¹)ᵈ (order p)
+ *     ψ ∈ L² (order p - 1), ρ = sigmoid(ψ)
+ *     ρ̃ ∈ H¹ (order p)
+ *     w ∈ V  (order p)
+ *     w̃ ∈ H¹ (order p)
+ *
+ * ---------------------------------------------------------------
+ *                          ALGORITHM
+ * ---------------------------------------------------------------
+ *
+ *  Update ρ with projected mirror descent via the following algorithm.
+ *
+ *  1. Initialize ψ = inv_sigmoid(vol_fraction) so that ∫ sigmoid(ψ) = θ vol(Ω)
+ *
+ *  While not converged:
+ *
+ *     2. Solve filter equation ∂_w̃ L = 0; i.e.,
+ *
+ *           (ϵ² ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)   ∀ v ∈ H¹.
+ *
+ *     3. Solve primal problem ∂_w L = 0; i.e.,
+ *
+ *      (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)   ∀ v ∈ V.
+ *
+ *     NB. The dual problem ∂_u L = 0 is the negative of the primal problem due to symmetry.
+ *
+ *     4. Solve for filtered gradient ∂_ρ̃ L = 0; i.e.,
+ *
+ *      (ϵ² ∇ w̃ , ∇ v ) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)   ∀ v ∈ H¹.
+ *
+ *     5. Project the gradient onto the discrete latent space; i.e., solve
+ *
+ *                         (G,v) = (w̃,v)   ∀ v ∈ L².
+ *
+ *     6. Bregman proximal gradient update; i.e.,
+ *
+ *                            ψ ← ψ - αG + c,
+ *
+ *     where α > 0 is a step size parameter and c ∈ R is a constant ensuring
+ *
+ *                     ∫_Ω sigmoid(ψ - αG + c) dx = θ vol(Ω).
+ *
+ *  end
+ */
+
+int main(int argc, char *argv[])
+{
+   // 0. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 1. Parse command-line options.
+   int ref_levels = 5;
+   int order = 2;
+   double alpha = 1.0;
+   double epsilon = 0.01;
+   double vol_fraction = 0.5;
+   int max_it = 1e3;
+   double itol = 1e-1;
+   double ntol = 1e-4;
+   double rho_min = 1e-6;
+   double lambda = 1.0;
+   double mu = 1.0;
+   bool glvis_visualization = true;
+   bool paraview_output = false;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&ref_levels, "-r", "--refine",
+                  "Number of times to refine the mesh uniformly.");
+   args.AddOption(&order, "-o", "--order",
+                  "Order (degree) of the finite elements.");
+   args.AddOption(&alpha, "-alpha", "--alpha-step-length",
+                  "Step length for gradient descent.");
+   args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
+                  "Length scale for ρ.");
+   args.AddOption(&max_it, "-mi", "--max-it",
+                  "Maximum number of gradient descent iterations.");
+   args.AddOption(&ntol, "-ntol", "--rel-tol",
+                  "Normalized exit tolerance.");
+   args.AddOption(&itol, "-itol", "--abs-tol",
+                  "Increment exit tolerance.");
+   args.AddOption(&vol_fraction, "-vf", "--volume-fraction",
+                  "Volume fraction for the material density.");
+   args.AddOption(&lambda, "-lambda", "--lambda",
+                  "Lamé constant λ.");
+   args.AddOption(&mu, "-mu", "--mu",
+                  "Lamé constant μ.");
+   args.AddOption(&rho_min, "-rmin", "--psi-min",
+                  "Minimum of density coefficient.");
+   args.AddOption(&glvis_visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.AddOption(&paraview_output, "-pv", "--paraview", "-no-pv",
+                  "--no-paraview",
+                  "Enable or disable ParaView output.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      MPI_Finalize();
+      return 1;
+   }
+   if (myid == 0)
+   {
+      mfem::out << num_procs << " number of process created.\n";
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh = Mesh::MakeCartesian2D(3, 1, mfem::Element::Type::QUADRILATERAL,
+                                     true, 3.0, 1.0);
+   int dim = mesh.Dimension();
+
+   // 2. Set BCs.
+   for (int i = 0; i<mesh.GetNBE(); i++)
+   {
+      Element * be = mesh.GetBdrElement(i);
+      Array<int> vertices;
+      be->GetVertices(vertices);
+
+      double * coords1 = mesh.GetVertex(vertices[0]);
+      double * coords2 = mesh.GetVertex(vertices[1]);
+
+      Vector center(2);
+      center(0) = 0.5*(coords1[0] + coords2[0]);
+      center(1) = 0.5*(coords1[1] + coords2[1]);
+
+      if (abs(center(0) - 0.0) < 1e-10)
+      {
+         // the left edge
+         be->SetAttribute(1);
+      }
+      else
+      {
+         // all other boundaries
+         be->SetAttribute(2);
+      }
+   }
+   mesh.SetAttributes();
+
+   // 3. Refine the mesh.
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   // 4. Define the necessary finite element spaces on the mesh.
+   H1_FECollection state_fec(order, dim); // space for u
+   H1_FECollection filter_fec(order, dim); // space for ρ̃
+   L2_FECollection control_fec(order-1, dim,
+                               BasisType::GaussLobatto); // space for ψ
+   ParFiniteElementSpace state_fes(&pmesh, &state_fec,dim);
+   ParFiniteElementSpace filter_fes(&pmesh, &filter_fec);
+   ParFiniteElementSpace control_fes(&pmesh, &control_fec);
+
+   HYPRE_BigInt state_size = state_fes.GlobalTrueVSize();
+   HYPRE_BigInt control_size = control_fes.GlobalTrueVSize();
+   HYPRE_BigInt filter_size = filter_fes.GlobalTrueVSize();
+   if (myid==0)
+   {
+      cout << "Number of state unknowns: " << state_size << endl;
+      cout << "Number of filter unknowns: " << filter_size << endl;
+      cout << "Number of control unknowns: " << control_size << endl;
+   }
+
+   // 5. Set the initial guess for ρ.
+   ParGridFunction u(&state_fes);
+   ParGridFunction psi(&control_fes);
+   ParGridFunction psi_old(&control_fes);
+   ParGridFunction rho_filter(&filter_fes);
+   u = 0.0;
+   rho_filter = vol_fraction;
+   psi = inv_sigmoid(vol_fraction);
+   psi_old = inv_sigmoid(vol_fraction);
+
+   // ρ = sigmoid(ψ)
+   MappedGridFunctionCoefficient rho(&psi, sigmoid);
+   // Interpolation of ρ = sigmoid(ψ) in control fes (for ParaView output)
+   ParGridFunction rho_gf(&control_fes);
+   // ρ - ρ_old = sigmoid(ψ) - sigmoid(ψ_old)
+   DiffMappedGridFunctionCoefficient succ_diff_rho(&psi, &psi_old, sigmoid);
+
+   // 6. Set-up the physics solver.
+   int maxat = pmesh.bdr_attributes.Max();
+   Array<int> ess_bdr(maxat);
+   ess_bdr = 0;
+   ess_bdr[0] = 1;
+   ConstantCoefficient one(1.0);
+   ConstantCoefficient lambda_cf(lambda);
+   ConstantCoefficient mu_cf(mu);
+   LinearElasticitySolver * ElasticitySolver = new LinearElasticitySolver();
+   ElasticitySolver->SetMesh(&pmesh);
+   ElasticitySolver->SetOrder(state_fec.GetOrder());
+   ElasticitySolver->SetupFEM();
+   Vector center(2); center(0) = 2.9; center(1) = 0.5;
+   Vector force(2); force(0) = 0.0; force(1) = -1.0;
+   double r = 0.05;
+   VolumeForceCoefficient vforce_cf(r,center,force);
+   ElasticitySolver->SetRHSCoefficient(&vforce_cf);
+   ElasticitySolver->SetEssentialBoundary(ess_bdr);
+
+   // 7. Set-up the filter solver.
+   ConstantCoefficient eps2_cf(epsilon*epsilon);
+   DiffusionSolver * FilterSolver = new DiffusionSolver();
+   FilterSolver->SetMesh(&pmesh);
+   FilterSolver->SetOrder(filter_fec.GetOrder());
+   FilterSolver->SetDiffusionCoefficient(&eps2_cf);
+   FilterSolver->SetMassCoefficient(&one);
+   Array<int> ess_bdr_filter;
+   if (pmesh.bdr_attributes.Size())
+   {
+      ess_bdr_filter.SetSize(pmesh.bdr_attributes.Max());
+      ess_bdr_filter = 0;
+   }
+   FilterSolver->SetEssentialBoundary(ess_bdr_filter);
+   FilterSolver->SetupFEM();
+
+   ParBilinearForm mass(&control_fes);
+   mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
+   mass.Assemble();
+   HypreParMatrix M;
+   Array<int> empty;
+   mass.FormSystemMatrix(empty,M);
+
+   // 8. Define the Lagrange multiplier and gradient functions.
+   ParGridFunction grad(&control_fes);
+   ParGridFunction w_filter(&filter_fes);
+
+   // 9. Define some tools for later.
+   ConstantCoefficient zero(0.0);
+   ParGridFunction onegf(&control_fes);
+   onegf = 1.0;
+   ParGridFunction zerogf(&control_fes);
+   zerogf = 0.0;
+   ParLinearForm vol_form(&control_fes);
+   vol_form.AddDomainIntegrator(new DomainLFIntegrator(one));
+   vol_form.Assemble();
+   double domain_volume = vol_form(onegf);
+   const double target_volume = domain_volume * vol_fraction;
+
+   // 10. Connect to GLVis. Prepare for VisIt output.
+   char vishost[] = "localhost";
+   int  visport   = 19916;
+   socketstream sout_r;
+   if (glvis_visualization)
+   {
+      sout_r.open(vishost, visport);
+      sout_r.precision(8);
+   }
+
+   mfem::ParaViewDataCollection paraview_dc("ex37p", &pmesh);
+   if (paraview_output)
+   {
+      rho_gf.ProjectCoefficient(rho);
+      paraview_dc.SetPrefixPath("ParaView");
+      paraview_dc.SetLevelsOfDetail(order);
+      paraview_dc.SetDataFormat(VTKFormat::BINARY);
+      paraview_dc.SetHighOrderOutput(true);
+      paraview_dc.SetCycle(0);
+      paraview_dc.SetTime(0.0);
+      paraview_dc.RegisterField("displacement",&u);
+      paraview_dc.RegisterField("density",&rho_gf);
+      paraview_dc.RegisterField("filtered_density",&rho_filter);
+      paraview_dc.Save();
+   }
+
+   // 11. Iterate:
+   for (int k = 1; k <= max_it; k++)
+   {
+      if (k > 1) { alpha *= ((double) k) / ((double) k-1); }
+
+      if (myid == 0)
+      {
+         cout << "\nStep = " << k << endl;
+      }
+
+      // Step 1 - Filter solve
+      // Solve (ϵ^2 ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)
+      FilterSolver->SetRHSCoefficient(&rho);
+      FilterSolver->Solve();
+      rho_filter = *FilterSolver->GetFEMSolution();
+
+      // Step 2 - State solve
+      // Solve (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)
+      SIMPInterpolationCoefficient SIMP_cf(&rho_filter,rho_min, 1.0);
+      ProductCoefficient lambda_SIMP_cf(lambda_cf,SIMP_cf);
+      ProductCoefficient mu_SIMP_cf(mu_cf,SIMP_cf);
+      ElasticitySolver->SetLameCoefficients(&lambda_SIMP_cf,&mu_SIMP_cf);
+      ElasticitySolver->Solve();
+      u = *ElasticitySolver->GetFEMSolution();
+
+      // Step 3 - Adjoint filter solve
+      // Solve (ϵ² ∇ w̃, ∇ v) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)
+      StrainEnergyDensityCoefficient rhs_cf(&lambda_cf,&mu_cf,&u, &rho_filter,
+                                            rho_min);
+      FilterSolver->SetRHSCoefficient(&rhs_cf);
+      FilterSolver->Solve();
+      w_filter = *FilterSolver->GetFEMSolution();
+
+      // Step 4 - Compute gradient
+      // Solve G = M⁻¹w̃
+      GridFunctionCoefficient w_cf(&w_filter);
+      ParLinearForm w_rhs(&control_fes);
+      w_rhs.AddDomainIntegrator(new DomainLFIntegrator(w_cf));
+      w_rhs.Assemble();
+      M.Mult(w_rhs,grad);
+
+      // Step 5 - Update design variable ψ ← proj(ψ - αG)
+      psi.Add(-alpha, grad);
+      const double material_volume = proj(psi, target_volume);
+
+      // Compute ||ρ - ρ_old|| in control fes.
+      double norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
+      double norm_reduced_gradient = norm_increment/alpha;
+      psi_old = psi;
+
+      double compliance = (*(ElasticitySolver->GetLinearForm()))(u);
+      MPI_Allreduce(MPI_IN_PLACE,&compliance,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+      if (myid == 0)
+      {
+         mfem::out << "norm of the reduced gradient = " << norm_reduced_gradient << endl;
+         mfem::out << "norm of the increment = " << norm_increment << endl;
+         mfem::out << "compliance = " << compliance << endl;
+         mfem::out << "volume fraction = " << material_volume / domain_volume << endl;
+      }
+
+      if (glvis_visualization)
+      {
+         ParGridFunction r_gf(&filter_fes);
+         r_gf.ProjectCoefficient(SIMP_cf);
+         sout_r << "parallel " << num_procs << " " << myid << "\n";
+         sout_r << "solution\n" << pmesh << r_gf
+                << "window_title 'Design density r(ρ̃)'" << flush;
+      }
+
+      if (paraview_output)
+      {
+         rho_gf.ProjectCoefficient(rho);
+         paraview_dc.SetCycle(k);
+         paraview_dc.SetTime((double)k);
+         paraview_dc.Save();
+      }
+
+      if (norm_reduced_gradient < ntol && norm_increment < itol)
+      {
+         break;
+      }
+   }
+
+   delete ElasticitySolver;
+   delete FilterSolver;
+
+   return 0;
+}
@@ -0,0 +1,696 @@
+//                                MFEM Example 38
+//
+// Compile with: make ex38
+//
+// Sample runs:
+// (since all sample runs require LAPACK, the * symbol is used to exclude them
+// from the automatically generated internal MFEM tests).
+//              * ex38
+//              * ex38 -i volumetric1d
+//              * ex38 -i surface2d
+//              * ex38 -i surface2d -o 4 -r 5
+//              * ex38 -i volumetric2d
+//              * ex38 -i volumetric2d -o 4 -r 5
+//              * ex38 -i surface3d
+//              * ex38 -i surface3d -o 4 -r 5
+//              * ex38 -i volumetric3d
+//              * ex38 -i volumetric3d -o 4 -r 5
+//
+// Description: This example code demonstrates the use of MFEM to integrate
+//              functions over implicit interfaces and subdomains bounded by
+//              implicit interfaces.
+//
+//              The quadrature rules are constructed by means of moment-fitting.
+//              The interface is given by the zero isoline of a level-set
+//              function ϕ and the subdomain is given as the domain where ϕ>0
+//              holds. The algorithm for construction of the quadrature rules
+//              was introduced by Mueller, Kummer and Oberlack [1].
+//
+//              This example also showcases how to set up integrators using the
+//              integration rules on implicit surfaces and subdomains.
+//
+// [1] Mueller, B., Kummer, F. and Oberlack, M. (2013) Highly accurate surface
+//     and volume integration on implicit domains by means of moment-fitting.
+//     Int. J. Numer. Meth. Engr. (96) 512-528. DOI:10.1002/nme.4569
+
+#include "mfem.hpp"
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+/// @brief Integration rule the example should demonstrate
+enum class IntegrationType { Volumetric1D, Surface2D, Volumetric2D,
+                             Surface3D, Volumetric3D
+                           };
+IntegrationType itype;
+
+/// @brief Level-set function defining the implicit interface
+double lvlset(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55 - X(0);
+      case IntegrationType::Surface2D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.));
+      case IntegrationType::Volumetric2D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.));
+      case IntegrationType::Surface3D:
+         return 1. - (pow(X(0), 2.) + pow(X(1), 2.) + pow(X(2), 2.));
+      case IntegrationType::Volumetric3D:
+         return 1. - (pow(X(0) / 1.5, 2.) + pow(X(1) / .75, 2.) + pow(X(2) / .5, 2.));
+      default:
+         return 1.;
+   }
+}
+
+/// @brief Function that should be integrated
+double integrand(const Vector& X)
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 3. * pow(X(0), 2.) - pow(X(1), 2.);
+      case IntegrationType::Volumetric2D:
+         return 1.;
+      case IntegrationType::Surface3D:
+         return 4. - 3. * pow(X(0), 2.) + 2. * pow(X(1), 2.) - pow(X(2), 2.);
+      case IntegrationType::Volumetric3D:
+         return 1.;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic surface integral
+double Surface()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return 1.;
+      case IntegrationType::Surface2D:
+         return 2. * M_PI;
+      case IntegrationType::Volumetric2D:
+         return 7.26633616541076;
+      case IntegrationType::Surface3D:
+         return 40. / 3. * M_PI;
+      case IntegrationType::Volumetric3D:
+         return 9.90182151329315;
+      default:
+         return 0.;
+   }
+}
+
+/// @brief Analytic volume integral over subdomain with positive level-set
+double Volume()
+{
+   switch (itype)
+   {
+      case IntegrationType::Volumetric1D:
+         return .55;
+      case IntegrationType::Surface2D:
+         return NAN;
+      case IntegrationType::Volumetric2D:
+         return 9. / 8. * M_PI;
+      case IntegrationType::Surface3D:
+         return NAN;
+      case IntegrationType::Volumetric3D:
+         return 3. / 4. * M_PI;
+      default:
+         return 0.;
+   }
+}
+
+#ifdef MFEM_USE_LAPACK
+/**
+ @brief Class for surface IntegrationRule
+
+ This class demonstrates how IntegrationRules computed as CutIntegrationRules
+ can be saved to reduce the impact by computing them from scratch each time.
+ */
+class SIntegrationRule : public IntegrationRule
+{
+protected:
+   /// @brief Space Dimension of the IntegrationRule
+   int dim;
+   /// @brief Column-wise matrix of the quadtrature weights
+   DenseMatrix Weights;
+   /// @brief Column-wise matrix of the transformation weights of the normal
+   DenseMatrix SurfaceWeights;
+
+public:
+   /**
+    @brief Constructor of SIntegrationRule
+
+    The surface integrationRules are computed and saved in the constructor.
+
+    @param [in] Order Order of the IntegrationRule
+    @param [in] LvlSet Level-set defining the implicit interface
+    @param [in] lsOrder Polynomial degree for approx of level-set function
+    @param [in] mesh Pointer to the mesh that is used
+   */
+   SIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   {
+      dim = mesh->Dimension();
+
+      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
+      mesh->GetElementTransformation(0, &Tr);
+      IntegrationRule ir;
+      MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+      if (dim >1)
+      {
+         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      }
+      else
+      {
+         Weights.SetSize(2, mesh->GetNE());
+      }
+      SurfaceWeights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      Vector w;
+      MFIRs.GetSurfaceWeights(Tr, ir, w);
+      SurfaceWeights.SetCol(0, w);
+      SetSize(ir.GetNPoints());
+
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntPoint(ip).index = ip;
+         IntegrationPoint &intp = IntPoint(ip);
+         intp.x = ir.IntPoint(ip).x;
+         intp.y = ir.IntPoint(ip).y;
+         intp.z = ir.IntPoint(ip).z;
+         if (dim > 1)
+         {
+            Weights(ip, 0) = ir.IntPoint(ip).weight;
+         }
+         else
+         {
+            Weights(0, 0) = ir.IntPoint(ip).x;
+            Weights(1, 0) = ir.IntPoint(ip).weight;
+         }
+      }
+
+
+      for (int elem = 1; elem < mesh->GetNE(); elem++)
+      {
+         mesh->GetElementTransformation(elem, &Tr);
+         MFIRs.GetSurfaceIntegrationRule(Tr, ir);
+         Vector w;
+         MFIRs.GetSurfaceWeights(Tr, ir, w);
+         SurfaceWeights.SetCol(elem, w);
+
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            if (dim > 1)
+            {
+               Weights(ip, elem) = ir.IntPoint(ip).weight;
+            }
+            else
+            {
+               Weights(0, elem) = ir.IntPoint(ip).x;
+               Weights(1, elem) = ir.IntPoint(ip).weight;
+            }
+         }
+      }
+   }
+
+   /**
+    @brief Set the weights for the given element and multiply them with the
+    transformation of the interface
+    */
+   void SetElementinclSurfaceWeight(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+         cout << intp.x << " " << Element << endl;
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element) * SurfaceWeights(ip, Element);
+         }
+   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+      {
+         IntegrationPoint &intp = IntPoint(0);
+         intp.x = Weights(0, Element);
+         intp.weight = Weights(1, Element);
+      }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of SIntegrationRule
+   ~SIntegrationRule() {}
+};
+
+/**
+ @brief Class for volume IntegrationRule
+
+ This class demonstrates how IntegrationRules computed as CutIntegrationRules
+ can be saved to reduce the impact by computing them from scratch each time.
+ */
+class CIntegrationRule : public IntegrationRule
+{
+protected:
+   /// @brief Space Dimension of the IntegrationRule
+   int dim;
+   /// @brief Column-wise matrix of the quadtrature weights
+   DenseMatrix Weights;
+
+public:
+   /**
+    @brief Constructor of CIntegrationRule
+
+    The volume integrationRules are computed and saved in the constructor.
+
+    @param [in] Order Order of the IntegrationRule
+    @param [in] LvlSet Level-set defining the implicit interface
+    @param [in] lsOrder Polynomial degree for approx of level-set function
+    @param [in] mesh Pointer to the mesh that is used
+   */
+   CIntegrationRule(int Order, Coefficient& LvlSet, int lsOrder, Mesh* mesh)
+   {
+      dim = mesh->Dimension();
+
+      IsoparametricTransformation Tr;
+      MomentFittingIntRules MFIRs(Order, LvlSet, lsOrder);
+      mesh->GetElementTransformation(0, &Tr);
+      IntegrationRule ir;
+      MFIRs.GetVolumeIntegrationRule(Tr, ir);
+      if (dim > 1)
+      {
+         Weights.SetSize(ir.GetNPoints(), mesh->GetNE());
+      }
+      else
+      {
+         Weights.SetSize(2 * ir.GetNPoints(), mesh->GetNE());
+      }
+
+      SetSize(ir.GetNPoints());
+      for (int ip = 0; ip < GetNPoints(); ip++)
+      {
+         IntPoint(ip).index = ip;
+         IntegrationPoint &intp = IntPoint(ip);
+         intp.x = ir.IntPoint(ip).x;
+         intp.y = ir.IntPoint(ip).y;
+         intp.z = ir.IntPoint(ip).z;
+         if (dim > 1)
+         {
+            Weights(ip, 0) = ir.IntPoint(ip).weight;
+         }
+         else
+         {
+            Weights(2 * ip, 0) = ir.IntPoint(ip).x;
+            Weights(2 * ip + 1, 0) = ir.IntPoint(ip).weight;
+         }
+      }
+
+      for (int elem = 1; elem < mesh->GetNE(); elem++)
+      {
+         mesh->GetElementTransformation(elem, &Tr);
+         MFIRs.GetVolumeIntegrationRule(Tr, ir);
+
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            if (dim > 1)
+            {
+               Weights(ip, elem) = ir.IntPoint(ip).weight;
+            }
+            else
+            {
+               Weights(2 * ip, elem) = ir.IntPoint(ip).x;
+               Weights(2 * ip + 1, elem) = ir.IntPoint(ip).weight;
+            }
+         }
+      }
+   }
+
+   /// @brief Set the weights for the given element
+   void SetElement(int Element)
+   {
+      if (dim == 1)
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.x = Weights(2 * ip, Element);
+            intp.weight = Weights(2 * ip + 1, Element);
+         }
+      else
+         for (int ip = 0; ip < GetNPoints(); ip++)
+         {
+            IntegrationPoint &intp = IntPoint(ip);
+            intp.weight = Weights(ip, Element);
+         }
+   }
+
+   /// @brief Destructor of CIntegrationRule
+   ~CIntegrationRule() {}
+};
+/**
+ @brief Class for surface linearform integrator
+
+ Integrator to demonstrate the use of the surface integration rule on an
+ implicit surface defined by a level-set.
+ */
+class SurfaceLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   SIntegrationRule* SIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the surface linear form integrator
+
+    Constructor for the surface linear form integrator to demonstrate the use
+    of the surface integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir surface integrtion rule to be used
+    */
+   SurfaceLFIntegrator(Coefficient &q, Coefficient &levelset,
+                       SIntegrationRule* ir)
+      : LinearFormIntegrator(), SIntRule(ir), LevelSet(levelset), Q(q) { }
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the surface integration rule for the current element
+      SIntRule->SetElementinclSurfaceWeight(Tr.ElementNo);
+
+      for (int ip = 0; ip < SIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(SIntRule->IntPoint(ip))));
+         double val = Tr.Weight() * Q.Eval(Tr, SIntRule->IntPoint(ip));
+         el.CalcShape(SIntRule->IntPoint(ip), shape);
+         add(elvect, SIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+};
+
+/**
+ @brief Class for subdomain linearform integrator
+
+ Integrator to demonstrate the use of the subdomain integration rule within
+ an area defined by an implicit surface defined by a level-set.
+ */
+class SubdomainLFIntegrator : public LinearFormIntegrator
+{
+protected:
+   /// @brief vector to evaluate the basis functions
+   Vector shape;
+
+   /// @brief surface integration rule
+   CIntegrationRule* CIntRule;
+
+   /// @brief coefficient representing the level-set defining the interface
+   Coefficient &LevelSet;
+
+   /// @brief coefficient representing the integrand
+   Coefficient &Q;
+
+public:
+   /**
+    @brief Constructor for the volumetric subdomain linear form integrator
+
+    Constructor for the subdomain linear form integrator to demonstrate the use
+    of the volumetric subdomain integration rule by means of moment-fitting.
+
+    @param [in] q coefficient representing the inegrand
+    @param [in] levelset level-set defining the implicit interfac
+    @param [in] ir subdomain integrtion rule to be used
+    */
+   SubdomainLFIntegrator(Coefficient &q, Coefficient &levelset,
+                         CIntegrationRule* ir)
+      : LinearFormIntegrator(), CIntRule(ir), LevelSet(levelset), Q(q) { }
+
+   /**
+    @brief Assembly of the element vector
+
+    Assemble the element vector of for the right hand side on the element given
+    by the FiniteElement and ElementTransformation.
+
+    @param [in] el finite Element the vector belongs to
+    @param [in] Tr transformation of finite element
+    @param [out] elvect vector containing the
+   */
+   virtual void AssembleRHSElementVect(const FiniteElement &el,
+                                       ElementTransformation &Tr,
+                                       Vector &elvect) override
+   {
+      int dof = el.GetDof();
+      shape.SetSize(dof);
+      elvect.SetSize(dof);
+      elvect = 0.;
+
+      // Update the subdomain integration rule
+      CIntRule->SetElement(Tr.ElementNo);
+
+      for (int ip = 0; ip < CIntRule->GetNPoints(); ip++)
+      {
+         Tr.SetIntPoint((&(CIntRule->IntPoint(ip))));
+         double val = Tr.Weight()
+                      * Q.Eval(Tr, CIntRule->IntPoint(ip));
+         el.CalcPhysShape(Tr, shape);
+         add(elvect, CIntRule->IntPoint(ip).weight * val, shape, elvect);
+      }
+   }
+};
+#endif // MFEM_USE_LAPACK
+
+int main(int argc, char *argv[])
+{
+#ifndef MFEM_USE_LAPACK
+   cout << "MFEM must be build with LAPACK for this example." << endl;
+   return EXIT_FAILURE;
+#else
+   // 1. Parse he command-line options.
+   int ref_levels = 3;
+   int order = 2;
+   const char *inttype = "surface2d";
+   bool visualization = true;
+   itype = IntegrationType::Surface2D;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&order, "-o", "--order", "Order of quadrature rule");
+   args.AddOption(&ref_levels, "-r", "--refine", "Number of meh refinements");
+   args.AddOption(&inttype, "-i", "--integrationtype",
+                  "IntegrationType to demonstrate");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.ParseCheck();
+
+   if (strcmp(inttype, "volumetric1d") == 0
+       || strcmp(inttype, "Volumetric1D") == 0)
+   {
+      itype = IntegrationType::Volumetric1D;
+   }
+   else if (strcmp(inttype, "surface2d") == 0
+            || strcmp(inttype, "Surface2D") == 0)
+   {
+      itype = IntegrationType::Surface2D;
+   }
+   else if (strcmp(inttype, "volumetric2d") == 0
+            || strcmp(inttype, "Volumetric2D") == 0)
+   {
+      itype = IntegrationType::Volumetric2D;
+   }
+   else if (strcmp(inttype, "surface3d") == 0
+            || strcmp(inttype, "Surface3d") == 0)
+   {
+      itype = IntegrationType::Surface3D;
+   }
+   else if (strcmp(inttype, "volumetric3d") == 0
+            || strcmp(inttype, "Volumetric3d") == 0)
+   {
+      itype = IntegrationType::Volumetric3D;
+   }
+
+   // 2. Construct and refine the mesh.
+   Mesh *mesh;
+   if (itype == IntegrationType::Volumetric1D)
+   {
+      mesh = new Mesh("../data/inline-segment.mesh");
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      mesh = new Mesh(2, 4, 1, 0, 2);
+      mesh->AddVertex(-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6);
+      mesh->AddVertex(1.6,1.6);
+      mesh->AddVertex(-1.6,1.6);
+      mesh->AddQuad(0,1,2,3);
+      mesh->FinalizeQuadMesh(1, 0, 1);
+   }
+   else if (itype == IntegrationType::Surface3D
+            || itype == IntegrationType::Volumetric3D)
+   {
+      mesh = new Mesh(3, 8, 1, 0, 3);
+      mesh->AddVertex(-1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,-1.6,-1.6);
+      mesh->AddVertex(1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,1.6,-1.6);
+      mesh->AddVertex(-1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,-1.6,1.6);
+      mesh->AddVertex(1.6,1.6,1.6);
+      mesh->AddVertex(-1.6,1.6,1.6);
+      mesh->AddHex(0,1,2,3,4,5,6,7);
+      mesh->FinalizeHexMesh(1, 0, 1);
+   }
+
+   for (int lev = 0; lev < ref_levels; lev++)
+   {
+      mesh->UniformRefinement();
+   }
+
+   // 3. Define the necessary finite element space on the mesh.
+   H1_FECollection fe_coll(1, mesh->Dimension());
+   FiniteElementSpace *fespace = new FiniteElementSpace(mesh, &fe_coll);
+
+   // 4. Construction Coefficients for the level set and the integrand.
+   FunctionCoefficient levelset(lvlset);
+   FunctionCoefficient u(integrand);
+
+   // 5. Define the necessary Integration rules on element 0.
+   IsoparametricTransformation Tr;
+   mesh->GetElementTransformation(0, &Tr);
+   SIntegrationRule* sir = new SIntegrationRule(order, levelset, 2, mesh);
+   CIntegrationRule* cir = NULL;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cir = new CIntegrationRule(order, levelset, 2, mesh);
+   }
+
+   // 6. Define and assemble the linear forms on the finite element space.
+   LinearForm surface(fespace);
+   LinearForm volume(fespace);
+
+   surface.AddDomainIntegrator(new SurfaceLFIntegrator(u, levelset, sir));
+   surface.Assemble();
+
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      volume.AddDomainIntegrator(new SubdomainLFIntegrator(u, levelset, cir));
+      volume.Assemble();
+   }
+
+   // 7. Print information, computed values and errors to the console.
+   int qorder = 0;
+   int nbasis = 2 * (order + 1) + (int)(order * (order + 1) / 2);
+   IntegrationRules irs(0, Quadrature1D::GaussLegendre);
+   IntegrationRule ir = irs.Get(Geometry::SQUARE, qorder);
+   for (; ir.GetNPoints() <= nbasis; qorder++)
+   {
+      ir = irs.Get(Geometry::SQUARE, qorder);
+   }
+   cout << "============================================" << endl;
+   cout << "Mesh size dx:                       ";
+   if (itype != IntegrationType::Volumetric1D)
+   {
+      cout << 3.2 / pow(2., (double)ref_levels) << endl;
+   }
+   else
+   {
+      cout << .25 / pow(2., (double)ref_levels) << endl;
+   }
+   if (itype == IntegrationType::Surface2D
+       || itype == IntegrationType::Volumetric2D)
+   {
+      cout << "Number of div free basis functions: " << nbasis << endl;
+      cout << "Number of quadrature points:        " << ir.GetNPoints() << endl;
+   }
+   cout << scientific << setprecision(2);
+   cout << "============================================" << endl;
+   cout << "Computed value of surface integral: " << surface.Sum() << endl;
+   cout << "True value of surface integral:     " << Surface() << endl;
+   cout << "Absolut Error (Surface):            ";
+   cout << abs(surface.Sum() - Surface()) << endl;
+   cout << "Relative Error (Surface):           ";
+   cout << abs(surface.Sum() - Surface()) / Surface() << endl;
+   if (itype == IntegrationType::Volumetric1D
+       || itype == IntegrationType::Volumetric2D
+       || itype == IntegrationType::Volumetric3D)
+   {
+      cout << "--------------------------------------------" << endl;
+      cout << "Computed value of volume integral:  " << volume.Sum() << endl;
+      cout << "True value of volume integral:      " << Volume() << endl;
+      cout << "Absolut Error (Volume):             ";
+      cout << abs(volume.Sum() - Volume()) << endl;
+      cout << "Relative Error (Volume):            ";
+      cout << abs(volume.Sum() - Volume()) / Volume() << endl;
+   }
+   cout << "============================================" << endl;
+
+   // 8. Plot the level-set function on a high order finite element space.
+   if (visualization)
+   {
+      H1_FECollection fe_coll2(5, mesh->Dimension());
+      FiniteElementSpace fespace2(mesh, &fe_coll2);
+      FunctionCoefficient levelset_coeff(levelset);
+      GridFunction lgf(&fespace2);
+      lgf.ProjectCoefficient(levelset_coeff);
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << *mesh << lgf << flush;
+      sol_sock << "keys pppppppppppppppppppppppppppcmmlRj\n";
+      sol_sock << "levellines " << 0. << " " << 0. << " " << 1 << "\n" << flush;
+   }
+
+   delete sir;
+   delete cir;
+   delete fespace;
+   delete mesh;
+   return EXIT_SUCCESS;
+#endif //MFEM_USE_LAPACK
+}
@@ -5,6 +5,7 @@
 // Sample runs:  mpirun -np 4 ex3p -m ../data/star.mesh
 //               mpirun -np 4 ex3p -m ../data/square-disc.mesh -o 2
 //               mpirun -np 4 ex3p -m ../data/beam-tet.mesh
+//               mpirun -np 4 ex3p -m ../data/beam-tet.mesh -nc -o 2
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh
 //               mpirun -np 4 ex3p -m ../data/beam-hex.mesh -o 2 -pa
 //               mpirun -np 4 ex3p -m ../data/escher.mesh
@@ -70,6 +71,7 @@ int main(int argc, char *argv[])
   int order = 1;
   bool static_cond = false;
   bool pa = false;
+   bool nc = false;
   const char *device_config = "cpu";
   bool visualization = true;
 #ifdef MFEM_USE_AMGX
@@ -87,6 +89,9 @@ int main(int argc, char *argv[])
                  "--no-static-condensation", "Enable static condensation.");
   args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                  "--no-partial-assembly", "Enable Partial Assembly.");
+   args.AddOption(&nc, "-nc", "--non-conforming", "-c",
+                  "--conforming",
+                  "Mark the mesh as nonconforming before partitioning.");
   args.AddOption(&device_config, "-d", "--device",
                  "Device configuration string, see Device::Configure().");
   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
@@ -124,6 +129,11 @@ int main(int argc, char *argv[])
   Mesh *mesh = new Mesh(mesh_file, 1, 1);
   dim = mesh->Dimension();
   int sdim = mesh->SpaceDimension();
+   if (nc)
+   {
+      // Can set to false to use conformal refinement for simplices.
+      mesh->EnsureNCMesh(true);
+   }

   // 5. Refine the serial mesh on all processors to increase the resolution. In
   //    this example we do 'ref_levels' of uniform refinement. We choose
@@ -450,7 +450,8 @@ int main(int argc, char *argv[])

 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
+   : TimeDependentOperator(M_.FESpace()->GetTrueVSize()),
+     M(M_), K(K_), b(b_), z(height)
 {
   Array<int> ess_tdof_list;
   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
@@ -659,9 +659,9 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
                           const Vector &b_, PrecType prec_type)
-   : TimeDependentOperator(M_.Height()), b(b_),
+   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize()), b(b_),
     M_solver(M_.ParFESpace()->GetComm()),
-     z(M_.Height())
+     z(height)
 {
   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
   {
@@ -3,28 +3,28 @@
 //
 // Compile with: make ex1
 //
-// Sample runs:  ex1 -m ../data/square-disc.mesh
-//               ex1 -m ../data/star.mesh
-//               ex1 -m ../data/star-mixed.mesh
-//               ex1 -m ../data/escher.mesh
-//               ex1 -m ../data/fichera.mesh
-//               ex1 -m ../data/fichera-mixed.mesh
-//               ex1 -m ../data/toroid-wedge.mesh
-//               ex1 -m ../data/square-disc-p2.vtk -o 2
-//               ex1 -m ../data/square-disc-p3.mesh -o 3
-//               ex1 -m ../data/square-disc-nurbs.mesh -o -1
-//               ex1 -m ../data/star-mixed-p2.mesh -o 2
-//               ex1 -m ../data/disc-nurbs.mesh -o -1
-//               ex1 -m ../data/pipe-nurbs.mesh -o -1
-//               ex1 -m ../data/fichera-mixed-p2.mesh -o 2
-//               ex1 -m ../data/star-surf.mesh
-//               ex1 -m ../data/square-disc-surf.mesh
-//               ex1 -m ../data/inline-segment.mesh
-//               ex1 -m ../data/amr-quad.mesh
-//               ex1 -m ../data/amr-hex.mesh
-//               ex1 -m ../data/fichera-amr.mesh
-//               ex1 -m ../data/mobius-strip.mesh
-//               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
+// Sample runs:  ex1 -m ../../data/square-disc.mesh
+//               ex1 -m ../../data/star.mesh
+//               ex1 -m ../../data/star-mixed.mesh
+//               ex1 -m ../../data/escher.mesh
+//               ex1 -m ../../data/fichera.mesh
+//               ex1 -m ../../data/fichera-mixed.mesh
+//               ex1 -m ../../data/toroid-wedge.mesh
+//               ex1 -m ../../data/square-disc-p2.vtk -o 2
+//               ex1 -m ../../data/square-disc-p3.mesh -o 3
+//               ex1 -m ../../data/square-disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/star-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/disc-nurbs.mesh -o -1
+//               ex1 -m ../../data/pipe-nurbs.mesh -o -1
+//               ex1 -m ../../data/fichera-mixed-p2.mesh -o 2
+//               ex1 -m ../../data/star-surf.mesh
+//               ex1 -m ../../data/square-disc-surf.mesh
+//               ex1 -m ../../data/inline-segment.mesh
+//               ex1 -m ../../data/amr-quad.mesh
+//               ex1 -m ../../data/amr-hex.mesh
+//               ex1 -m ../../data/fichera-amr.mesh
+//               ex1 -m ../../data/mobius-strip.mesh
+//               ex1 -m ../../data/mobius-strip.mesh -o -1 -sc
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
@@ -32,7 +32,7 @@
 //               ex1 -pa -d occa-cuda
 //               ex1 -pa -d raja-omp
 //               ex1 -pa -d occa-omp
-//               ex1 -m ../data/beam-hex.mesh -pa -d cuda
+//               ex1 -m ../../data/beam-hex.mesh -pa -d cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -22,14 +22,19 @@ MFEM_LIB_FILE = mfem_is_not_built
 -include $(CONFIG_MK)

 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
-  ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33 ex34 ex36
+   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
+   ex31 ex33 ex34 ex36 ex37
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
-  ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p
+   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
+   ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
+   ex37p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
-  ex24p ex25p ex26p ex34p ex35p
+   ex24p ex25p ex26p ex34p ex35p
+
+ifeq ($(MFEM_USE_LAPACK),YES)
+   SEQ_EXAMPLES += ex38
+endif

 ifeq ($(MFEM_USE_MPI),NO)
   EXAMPLES = $(SEQ_EXAMPLES)
@@ -92,10 +97,12 @@ $(SUBDIRS_TPRINT):
 # Additional dependencies
 ex18: $(SRC)ex18.hpp
 ex33: $(SRC)ex33.hpp
+ex37: $(SRC)ex37.hpp

 ifeq ($(MFEM_USE_MPI),YES)
 ex18p: $(SRC)ex18.hpp
 ex33p: $(SRC)ex33.hpp
+ex37p: $(SRC)ex37.hpp
 endif

 MFEM_TESTS = EXAMPLES
@@ -139,6 +146,10 @@ ex27-test-seq: ex27
 	@$(call mfem-test,$<,, Serial example,-dg)
 ex27p-test-par: ex27p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-dg)
+ex37-test-seq: ex37
+	@$(call mfem-test,$<,, Serial example,-mi 3)
+ex37p-test-par: ex37p
+	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-mi 3)
 # Testing: optional tests
 ifeq ($(MFEM_USE_STRUMPACK),YES)
 ex11p-test-strumpack: ex11p
@@ -24,13 +24,13 @@ if (MFEM_USE_MPI)
    ex10p.cpp
    )
  list(APPEND PETSC_RC_FILES
-    rc_ex1p
-    rc_ex2p
+    rc_ex1p rc_ex1p_device rc_ex1p_deviceamg
+    rc_ex2p rc_ex2p_bddc rc_ex2p_asm
    rc_ex3p rc_ex3p_bddc
    rc_ex4p rc_ex4p_bddc
    rc_ex5p_bddc rc_ex5p_fieldsplit
-    rc_ex9p_expl rc_ex9p_impl
-    rc_ex10p
+    rc_ex9p_expl rc_ex9p_expl_device rc_ex9p_impl
+    rc_ex10p rc_ex10p_mf rc_ex10p_mfop rc_ex10p_jfnk
    )
 endif()

@@ -39,7 +39,7 @@ if (MFEM_USE_SLEPC)
    ex11p.cpp
    )
  list(APPEND PETSC_RC_FILES
-    rc_ex11p_lobpcg rc_ex11p_gd
+    rc_ex11p_lobpcg rc_ex11p_lobpcg_device rc_ex11p_gd
    )
 endif()

@@ -74,7 +74,13 @@ add_mfem_examples(PETSC_EXAMPLES_SRCS ${PFX} copy_petsc_rc_files test_petsc)
 # Command line options for the tests.
 set(EX1_ARGS_W       -m ../../data/amr-quad.mesh --usepetsc)
 set(EX1_ARGS_P       -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
+set(EX1_ARGS_CUDA    -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_device)
+set(EX1_ARGS_CUDAAMG -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_deviceamg)
+set(EX1_ARGS_HIP     -m ../../data/star.mesh --usepetsc --partial-assembly --device hip --petscopts rc_ex1p_device)
+set(EX1_ARGS_HIPAMG  -m ../../data/star.mesh --usepetsc --device hip --petscopts rc_ex1p_deviceamg)
 set(EX2_ARGS         -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
+set(EX2_ARGS_BDDC    -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc)
+set(EX2_ARGS_ASM     -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm)
 set(EX3_ARGS         -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
 set(EX4_ARGS         -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
 set(EX4_HYB_ARGS     -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization)
@@ -85,22 +91,46 @@ set(EX6_ARGS         -m ../../data/amr-quad.mesh --usepetsc)
 set(EX6_NONOVL_ARGS  -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping)
 set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
+set(EX9_ES_ARGS_CUDA -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device cuda)
+set(EX9_ES_ARGS_HIP  -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device hip)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
+set(EX10_MF_ARGS     -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3)
+set(EX10_MFOP_ARGS   -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3)
+set(EX10_JFNK_ARGS   -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3)
 if (MFEM_USE_SLEPC)
  set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
  set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
+  set(EX11_ARGS_LOBPCG_CUDA -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device cuda)
+  set(EX11_ARGS_LOBPCG_HIP  -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device hip)
  set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
 endif()

 # Add the tests: one test per command-line-variable.
 if (MFEM_ENABLE_TESTING)
  set(TEST_OPTIONS_VARS
-      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
-      EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
-      EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
+      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX2_ARGS_BDDC EX2_ARGS_ASM EX3_ARGS
+      EX4_ARGS EX4_HYB_ARGS EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS
+      EX6_ARGS EX6_NONOVL_ARGS EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS
+      EX10_MF_ARGS EX10_MFOP_ARGS EX10_JFNK_ARGS)
  if (MFEM_USE_SLEPC)
-      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
+    list(APPEND TEST_OPTIONS_VARS
+      EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
+  endif()
+  # CUDA/HIP tests
+  if (MFEM_USE_CUDA)
+    list(APPEND TEST_OPTIONS_VARS
+      EX1_ARGS_CUDA EX1_ARGS_CUDAAMG EX9_ES_ARGS_CUDA)
+    if (MFEM_USE_SLEPC)
+      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_LOBPCG_CUDA)
+    endif()
+  elseif (MFEM_USE_HIP)
+    list(APPEND TEST_OPTIONS_VARS
+      EX1_ARGS_HIP EX1_ARGS_HIPAMG EX9_ES_ARGS_HIP)
+    if (MFEM_USE_SLEPC)
+      # SLEPc does not support BVSVEC with HIP
+      # list(APPEND TEST_OPTIONS_VARS EX11_ARGS_LOBPCG_HIP)
+    endif()
  endif()

  foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
@@ -115,7 +145,7 @@ if (MFEM_ENABLE_TESTING)

    # All PETSC tests are parallel.
    if (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME_FULL}_np=4
+      add_test(NAME ${TEST_NAME_FULL}_np=${MFEM_MPI_NP}
        COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
        ${MPIEXEC_PREFLAGS}
        $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}
@@ -273,12 +273,13 @@ int main(int argc, char *argv[])
 #ifdef MFEM_USE_STRUMPACK
         if (sp_solver)
         {
-            STRUMPACKSolver * strumpack = new STRUMPACKSolver(argc, argv, MPI_COMM_WORLD);
+            STRUMPACKSolver * strumpack = new STRUMPACKSolver(MPI_COMM_WORLD, argc, argv);
            strumpack->SetPrintFactorStatistics(true);
            strumpack->SetPrintSolveStatistics(false);
            strumpack->SetKrylovSolver(strumpack::KrylovSolver::DIRECT);
            strumpack->SetReorderingStrategy(strumpack::ReorderingStrategy::METIS);
-            strumpack->DisableMatching();
+            strumpack->SetMatching(strumpack::MatchingJob::NONE);
+            strumpack->SetCompression(strumpack::CompressionType::NONE);
            strumpack->SetOperator(*Arow);
            strumpack->SetFromCommandLine();
            precond = strumpack;
@@ -7,7 +7,7 @@
 //               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh --petscopts rc_ex1p
 //
 // Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda --petscopts rc_ex1p_cuda
+//               mpirun -np 4 ex1p -pa -d cuda --petscopts rc_ex1p_device
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem
@@ -17,7 +17,7 @@
 //               finite elements (velocity u) and piecewise discontinuous
 //               polynomials (pressure p).
 //
-//               The example demonstrates the use of the BlockMatrix class, as
+//               The example demonstrates the use of the BlockOperator class, as
 //               well as the collective saving of several grid functions in a
 //               VisIt (visit.llnl.gov) visualization format.
 //
@@ -520,10 +520,10 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
                           const Vector &b_,bool M_in_lhs)
-   : TimeDependentOperator(M_.Height(), 0.0,
+   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize(), 0.0,
                           M_in_lhs ? TimeDependentOperator::IMPLICIT
                           : TimeDependentOperator::EXPLICIT),
-     b(b_), comm(M_.ParFESpace()->GetComm()), M_solver(comm), z(M_.Height()),
+     b(b_), comm(M_.ParFESpace()->GetComm()), M_solver(comm), z(height),
     iJacobian(NULL), rJacobian(NULL)
 {
   MAlev = M_.GetAssemblyLevel();
@@ -66,7 +66,9 @@ include $(MFEM_TEST_MK)

 # Testing: Parallel runs
 RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-TESTNAME = Parallel PETSc example
+TESTNAME      = Parallel PETSc example
+TESTNAME_CUDA = Parallel CUDA PETSc example
+TESTNAME_HIP  = Parallel HIP PETSc example
 %-test-par: %
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME))

@@ -74,8 +76,10 @@ TESTNAME = Parallel PETSc example
 # Testing PETSc execution options.
 EX1_ARGS_W            := -m ../../data/amr-quad.mesh --usepetsc
 EX1_ARGS_P            := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
-EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_cuda
-EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
+EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_device
+EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_deviceamg
+EX1_ARGS_HIP          := -m ../../data/star.mesh --usepetsc --partial-assembly --device hip --petscopts rc_ex1p_device
+EX1_ARGS_HIPAMG       := -m ../../data/star.mesh --usepetsc --device hip --petscopts rc_ex1p_deviceamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
 EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
@@ -89,7 +93,8 @@ EX6_ARGS              := -m ../../data/amr-quad.mesh --usepetsc
 EX6_NONOVL_ARGS       := -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping
 EX9_E_ARGS            := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
 EX9_ES_ARGS           := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
-EX9_ES_ARGS_CUDA      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_cuda --no-step --partial-assembly --device cuda
+EX9_ES_ARGS_CUDA      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device cuda
+EX9_ES_ARGS_HIP       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device hip
 EX9_IS_ARGS           := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
 EX10_ARGS             := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
 EX10_MF_ARGS          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
@@ -97,15 +102,20 @@ EX10_MFOP_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_
 EX10_JFNK_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
 EX11_ARGS_SINV        := -m ../../data/star.mesh --useslepc
 EX11_ARGS_LOBPCG      := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
-EX11_ARGS_LOBPCG_CUDA := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_cuda --device cuda
+EX11_ARGS_LOBPCG_CUDA := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device cuda
+EX11_ARGS_LOBPCG_HIP  := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device hip
 EX11_ARGS_GD          := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd

 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
 ifeq ($(MFEM_USE_CUDA),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_CUDA))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_CUDAAMG))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX1_ARGS_CUDA))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX1_ARGS_CUDAAMG))
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX1_ARGS_HIP))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX1_ARGS_HIPAMG))
 endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
@@ -128,7 +138,10 @@ ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_IS_ARGS))
 ifeq ($(MFEM_USE_CUDA),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS_CUDA))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX9_ES_ARGS_CUDA))
+endif
+ifeq ($(MFEM_USE_HIP),YES)
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX9_ES_ARGS_HIP))
 endif
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_ARGS))
@@ -140,8 +153,12 @@ ex11p-test-par: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
 ifeq ($(MFEM_USE_CUDA),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG_CUDA))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX11_ARGS_LOBPCG_CUDA))
 endif
+# SLEPc does not support BVSVEC with HIP
+#ifeq ($(MFEM_USE_HIP),YES)
+#	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX11_ARGS_LOBPCG_HIP))
+#endif
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
 endif

@@ -156,6 +173,9 @@ clean: clean-build clean-exec
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
+ifneq ($(SRC),)
+	rm -f $(RC_FILES)
+endif

 clean-exec:
 	@rm -rf mesh.* sol.* sol_p.* sol_u.* Example5*
@@ -0,0 +1,246 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make stokes
+//
+// Sample runs:
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void rhs_func(const Vector & x, Vector &f)
+{
+   f.SetSize(x.Size());
+   f = 0.0;
+}
+
+void u0_func(const Vector & x, Vector &u0)
+{
+   u0.SetSize(x.Size());
+   u0 = 0.0;
+   // u0[0] = -sin(M_PI * x[1]);
+   u0[0] = -x[1]*(1-x[1])*x[2]*(1-x[2]);
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "../data/inline-hex.mesh";
+   int sref = 0;
+   int pref = 0;
+   bool visualization = true;
+
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sref, "-sr", "--refinements-serial",
+                  "Number of serial refinements");
+   args.AddOption(&pref, "-pr", "--refinements-parallel",
+                  "Number of parallel refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   // Mesh mesh(mesh_file, 1, 1);
+
+   Mesh mesh = Mesh::MakeCartesian3D(8,1,1,mfem::Element::HEXAHEDRON,8.0,1.0,1.0);
+   int dim = mesh.Dimension();
+
+   for (int l = 0; l < sref; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int l = 0; l < pref; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+   FiniteElementCollection *ufec = new H1_FECollection(2,dim);
+   FiniteElementCollection *pfec = new H1_FECollection(1,dim);
+
+   ParFiniteElementSpace ufes(&pmesh, ufec,dim,mfem::Ordering::byVDIM);
+   ParFiniteElementSpace pfes(&pmesh, pfec);
+   Array<ParFiniteElementSpace *> pfespaces;
+   pfespaces.Append(&ufes);
+   pfespaces.Append(&pfes);
+   HYPRE_BigInt usize = ufes.GlobalTrueVSize();
+   HYPRE_BigInt psize = pfes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of velocity dofs: " << usize << endl;
+      cout << "Number of pressure dofs: " << psize << endl;
+   }
+
+   ParGridFunction u_gf(&ufes); u_gf = 0.0;
+   ParGridFunction p_gf(&pfes); p_gf = 0.0;
+   VectorFunctionCoefficient u0_cf(dim, u0_func);
+   Array<int> ess_bdr_in(pmesh.bdr_attributes.Max()); ess_bdr_in = 0;
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max()); ess_bdr = 1;
+   ess_bdr_in[2] = 1; // x = 1 plane
+   ess_bdr[2] = 0; // x = 1 plane
+   ess_bdr[4] = 0; // x = 0 plane
+   Vector vzero(dim); vzero = 0.0;
+   VectorConstantCoefficient zero_cf(vzero);
+   u_gf.ProjectBdrCoefficient(zero_cf,ess_bdr);
+   u_gf.ProjectBdrCoefficient(u0_cf,ess_bdr_in);
+
+   // essential dofs;
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listp;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> essbdr(pmesh.bdr_attributes.Max());
+      essbdr = 1; essbdr[4] = 0;
+      ufes.GetEssentialTrueDofs(essbdr, ess_tdof_list);
+      essbdr = 0; essbdr[4] = 1;
+      pfes.GetEssentialTrueDofs(essbdr, ess_tdof_listp);
+   }
+
+   ParBilinearForm prec_pq(&pfes);
+   ConstantCoefficient one(1.0);
+   prec_pq.AddDomainIntegrator(new MassIntegrator(one));
+   prec_pq.Assemble();
+   HypreParMatrix Mpq;
+   prec_pq.FormSystemMatrix(ess_tdof_listp,Mpq);
+
+   for (int i = 0; i < ess_tdof_listp.Size(); i++)
+   {
+      ess_tdof_listp[i] += ufes.GetTrueVSize();
+   }
+   ess_tdof_list.Append(ess_tdof_listp);
+
+   // linear and bilinear forms
+   ParLinearForm b_v(&ufes);
+   VectorFunctionCoefficient rhs_cf(dim, rhs_func);
+   b_v.AddDomainIntegrator(new VectorDomainLFIntegrator(rhs_cf));
+   b_v.Assemble();
+
+   ParLinearForm b_q(&pfes);
+   b_q.Assemble();
+
+
+   ConstantCoefficient negone(-1.0);
+   // (∇ u, ∇ v)
+   ParBilinearForm a_uv(&ufes);
+   a_uv.AddDomainIntegrator(new VectorDiffusionIntegrator(one));
+
+   ParMixedBilinearForm a_pv(&pfes,&ufes);
+   a_pv.AddDomainIntegrator(new TransposeIntegrator(new VectorDivergenceIntegrator(
+                                                       negone)));
+
+   ParMixedBilinearForm a_uq(&ufes,&pfes);
+   a_uq.AddDomainIntegrator(new VectorDivergenceIntegrator(negone));
+
+   ParBilinearForm a_pq(&pfes);
+   ConstantCoefficient zero(0.0);
+   a_pq.AddDomainIntegrator(new MassIntegrator(zero));
+
+   ParBlockForm bform(pfespaces);
+   bform.SetBlock(&a_uv,0,0);
+   bform.SetBlock(&a_pv,0,1);
+   bform.SetBlock(&a_uq,1,0);
+   bform.SetBlock(&a_pq,1,1);
+
+   Array<int> tdof_offsets(3);
+   tdof_offsets[0] = 0;
+   tdof_offsets[1] = ufes.TrueVSize();
+   tdof_offsets[2] = pfes.TrueVSize();
+   tdof_offsets.PartialSum();
+   Array<int> dof_offsets(3);
+   dof_offsets[0] = 0;
+   dof_offsets[1] = ufes.GetVSize();
+   dof_offsets[2] = pfes.GetVSize();
+   dof_offsets.PartialSum();
+
+   Vector b(dof_offsets.Last());
+   b.SetVector(b_v,0);
+   b.SetVector(b_q,dof_offsets[1]);
+   Vector x(dof_offsets.Last());
+   x.SetVector(u_gf,0);
+   x.SetVector(p_gf,dof_offsets[1]);
+
+   OperatorPtr Ah;
+   Vector B,X;
+   bform.Assemble();
+   bform.FormLinearSystem(ess_tdof_list,x, b, Ah, X, B);
+
+   BlockOperator * A = Ah.As<BlockOperator>();
+
+   BlockDiagonalPreconditioner prec(tdof_offsets);
+   HypreBoomerAMG amg_v((HypreParMatrix&)A->GetBlock(0,0));
+   amg_v.SetSystemsOptions(dim);
+
+   HypreBoomerAMG amg_p(Mpq);
+
+   prec.SetDiagonalBlock(0,&amg_v);
+   prec.SetDiagonalBlock(1,&amg_p);
+
+
+   // ((HypreParMatrix*)&A->GetBlock(0,0))->Print("A00.mat");
+   // // ((HypreParMatrix*)&A->GetBlock(0,1))->Print("A01.mat");
+   // ((HypreParMatrix*)&A->GetBlock(1,0))->Print("A10.mat");
+   // // ((HypreParMatrix*)&A->GetBlock(1,1))->Print("A11.mat");
+   // return 0;
+
+
+   MINRESSolver solver(MPI_COMM_WORLD);
+   solver.SetRelTol(1e-12);
+   solver.SetMaxIter(60);
+   solver.SetPrintLevel(1);
+   solver.SetPreconditioner(prec);
+   solver.SetOperator(*A);
+   solver.Mult(B, X);
+
+
+   BlockVector Xb(X.GetData(),tdof_offsets);
+   u_gf.SetFromTrueDofs(Xb.GetBlock(0));
+   p_gf.SetFromTrueDofs(Xb.GetBlock(1));
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << u_gf << flush;
+
+      socketstream solp_sock(vishost, visport);
+      solp_sock << "parallel " << num_procs << " " << myid << "\n";
+      solp_sock.precision(8);
+      solp_sock << "solution\n" << pmesh << p_gf << flush;
+
+   }
+
+   delete ufec;
+   delete pfec;
+
+   return 0;
+}
@@ -0,0 +1,260 @@
+//                       MFEM Example 1 - Parallel Version
+//
+// Compile with: make stokes
+//
+// Sample runs:
+
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+void rhs_func(const Vector & x, Vector &f)
+{
+   f.SetSize(x.Size());
+   f = 0.0;
+}
+
+void u0_func(const Vector & x, Vector &u0)
+{
+   u0.SetSize(x.Size());
+   u0 = 0.0;
+   double r0 = 2.393;
+   u0[0] = -(r0*r0 - x[1]*x[1] - x[2]*x[2]);
+   // u0[0] = +1;
+}
+
+int main(int argc, char *argv[])
+{
+   // 1. Initialize MPI and HYPRE.
+   Mpi::Init();
+   int num_procs = Mpi::WorldSize();
+   int myid = Mpi::WorldRank();
+   Hypre::Init();
+
+   // 2. Parse command-line options.
+   const char *mesh_file = "anu.msh";
+   int sref = 0;
+   int pref = 0;
+   bool visualization = true;
+
+   OptionsParser args(argc, argv);
+   args.AddOption(&mesh_file, "-m", "--mesh",
+                  "Mesh file to use.");
+   args.AddOption(&sref, "-sr", "--refinements-serial",
+                  "Number of serial refinements");
+   args.AddOption(&pref, "-pr", "--refinements-parallel",
+                  "Number of parallel refinements");
+   args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
+                  "--no-visualization",
+                  "Enable or disable GLVis visualization.");
+   args.Parse();
+   if (!args.Good())
+   {
+      if (myid == 0)
+      {
+         args.PrintUsage(cout);
+      }
+      return 1;
+   }
+   if (myid == 0)
+   {
+      args.PrintOptions(cout);
+   }
+
+   Mesh mesh(mesh_file, 1, 1);
+
+   int dim = mesh.Dimension();
+
+   for (int l = 0; l < sref; l++)
+   {
+      mesh.UniformRefinement();
+   }
+
+   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+   mesh.Clear();
+
+   for (int l = 0; l < pref; l++)
+   {
+      pmesh.UniformRefinement();
+   }
+
+
+   FiniteElementCollection *ufec = new H1_FECollection(2,dim);
+   FiniteElementCollection *pfec = new H1_FECollection(1,dim);
+
+   ParFiniteElementSpace ufes(&pmesh, ufec,dim,mfem::Ordering::byVDIM);
+   ParFiniteElementSpace pfes(&pmesh, pfec);
+   Array<ParFiniteElementSpace *> pfespaces;
+   pfespaces.Append(&ufes);
+   pfespaces.Append(&pfes);
+
+   HYPRE_BigInt usize = ufes.GlobalTrueVSize();
+   HYPRE_BigInt psize = pfes.GlobalTrueVSize();
+   if (myid == 0)
+   {
+      cout << "Number of velocity dofs: " << usize << endl;
+      cout << "Number of pressure dofs: " << psize << endl;
+   }
+
+   ParGridFunction u_gf(&ufes); u_gf = 0.0;
+   ParGridFunction p_gf(&pfes); p_gf = 0.0;
+   VectorFunctionCoefficient u0_cf(dim, u0_func);
+   Array<int> ess_bdr_in(pmesh.bdr_attributes.Max()); ess_bdr_in = 0;
+   Array<int> ess_bdr(pmesh.bdr_attributes.Max()); ess_bdr = 1;
+   ess_bdr_in[1] = 1; // inflow
+   ess_bdr[0] = 0; // rest of bdr (except the outflow)
+   Vector vzero(dim); vzero = 0.0;
+   VectorConstantCoefficient zero_cf(vzero);
+   u_gf.ProjectBdrCoefficient(zero_cf,ess_bdr);
+   u_gf.ProjectBdrCoefficient(u0_cf,ess_bdr_in);
+
+   // essential dofs;
+   Array<int> ess_tdof_list;
+   Array<int> ess_tdof_listp;
+   if (pmesh.bdr_attributes.Size())
+   {
+      Array<int> essbdr(pmesh.bdr_attributes.Max());
+      essbdr = 1; essbdr[2] = 0;
+      ufes.GetEssentialTrueDofs(essbdr, ess_tdof_list);
+      essbdr = 0; essbdr[2] = 1;
+      // pfes.GetEssentialTrueDofs(essbdr, ess_tdof_listp);
+   }
+   ConstantCoefficient one(1.0);
+
+   ParBilinearForm prec_pq(&pfes);
+   prec_pq.AddDomainIntegrator(new MassIntegrator(one));
+   prec_pq.Assemble();
+   HypreParMatrix Mpq;
+   prec_pq.FormSystemMatrix(ess_tdof_listp,Mpq);
+
+   for (int i = 0; i < ess_tdof_listp.Size(); i++)
+   {
+      ess_tdof_listp[i] += ufes.GetTrueVSize();
+   }
+   ess_tdof_list.Append(ess_tdof_listp);
+
+   // linear and bilinear forms
+   ParLinearForm b_v(&ufes);
+   VectorFunctionCoefficient rhs_cf(dim, rhs_func);
+   b_v.AddDomainIntegrator(new VectorDomainLFIntegrator(rhs_cf));
+   b_v.Assemble();
+
+   ParLinearForm b_q(&pfes);
+   b_q.Assemble();
+
+
+   ConstantCoefficient negone(-1.0);
+
+   // (∇ u, ∇ v)
+   ParBilinearForm a_uv(&ufes);
+   a_uv.AddDomainIntegrator(new VectorDiffusionIntegrator(one));
+   ParMixedBilinearForm a_pv(&pfes,&ufes);
+   a_pv.AddDomainIntegrator(new TransposeIntegrator(new VectorDivergenceIntegrator(
+                                                       negone)));
+
+   ParMixedBilinearForm a_uq(&ufes,&pfes);
+   a_uq.AddDomainIntegrator(new VectorDivergenceIntegrator(negone));
+
+   ParBilinearForm a_pq(&pfes);
+
+   ParBlockForm bform(pfespaces);
+   bform.SetBlock(&a_uv,0,0);
+   bform.SetBlock(&a_pv,0,1);
+   bform.SetBlock(&a_uq,1,0);
+   bform.SetBlock(&a_pq,1,1);
+
+
+   Array<int> tdof_offsets(3);
+   tdof_offsets[0] = 0;
+   tdof_offsets[1] = ufes.TrueVSize();
+   tdof_offsets[2] = pfes.TrueVSize();
+   tdof_offsets.PartialSum();
+   Array<int> dof_offsets(3);
+   dof_offsets[0] = 0;
+   dof_offsets[1] = ufes.GetVSize();
+   dof_offsets[2] = pfes.GetVSize();
+   dof_offsets.PartialSum();
+
+   Vector b(dof_offsets.Last());
+   b.SetVector(b_v,0);
+   b.SetVector(b_q,dof_offsets[1]);
+   Vector x(dof_offsets.Last());
+   x.SetVector(u_gf,0);
+   x.SetVector(p_gf,dof_offsets[1]);
+
+   OperatorPtr Ah;
+   Vector B,X;
+   bform.Assemble();
+   bform.FormLinearSystem(ess_tdof_list,x, b, Ah, X, B);
+
+   BlockOperator * A = Ah.As<BlockOperator>();
+
+   BlockDiagonalPreconditioner prec(tdof_offsets);
+   HypreParMatrix A00 = (HypreParMatrix&)A->GetBlock(0,0);
+   HypreParMatrix A01 = (HypreParMatrix&)A->GetBlock(0,1);
+   HypreBoomerAMG amg_v(A00);
+   amg_v.SetSystemsOptions(dim);
+   prec.SetDiagonalBlock(0,&amg_v);
+
+   HypreParVector A00_diag(MPI_COMM_WORLD, A00.GetGlobalNumRows(),
+                                 A00.GetRowStarts());
+   A00.GetDiag(A00_diag);
+   HypreParMatrix S_tmp(A01);
+   S_tmp.InvScaleRows(A00_diag);
+   HypreParMatrix *S = ParMult(A01.Transpose(), &S_tmp, true);
+   HypreBoomerAMG amg_p(*S);
+   // HypreBoomerAMG amg_p(Mpq);
+   prec.SetDiagonalBlock(1,&amg_p);
+
+   MINRESSolver solver(MPI_COMM_WORLD);
+   solver.SetRelTol(1e-12);
+   solver.SetMaxIter(20000);
+   solver.SetPrintLevel(1);
+   solver.SetPreconditioner(prec);
+   solver.SetOperator(*A);
+   solver.Mult(B, X);
+
+   BlockVector Xb(X.GetData(),tdof_offsets);
+   u_gf.SetFromTrueDofs(Xb.GetBlock(0));
+   p_gf.SetFromTrueDofs(Xb.GetBlock(1));
+
+   ParaViewDataCollection * paraview_dc = nullptr;
+   bool paraview = true;
+   if (paraview)
+   {
+      paraview_dc = new ParaViewDataCollection(mesh_file, &pmesh);
+      paraview_dc->SetPrefixPath("ParaView");
+      paraview_dc->SetLevelsOfDetail(2);
+      paraview_dc->SetCycle(0);
+      paraview_dc->SetDataFormat(VTKFormat::BINARY);
+      paraview_dc->SetHighOrderOutput(true);
+      paraview_dc->SetTime(0.0); // set the time
+      paraview_dc->RegisterField("velocity",&u_gf);
+      paraview_dc->RegisterField("pressure",&p_gf);
+      paraview_dc->Save();
+   }
+
+   if (visualization)
+   {
+      char vishost[] = "localhost";
+      int  visport   = 19916;
+      socketstream sol_sock(vishost, visport);
+      sol_sock << "parallel " << num_procs << " " << myid << "\n";
+      sol_sock.precision(8);
+      sol_sock << "solution\n" << pmesh << u_gf << flush;
+
+      socketstream solp_sock(vishost, visport);
+      solp_sock << "parallel " << num_procs << " " << myid << "\n";
+      solp_sock.precision(8);
+      solp_sock << "solution\n" << pmesh << p_gf << flush;
+
+   }
+
+   delete ufec;
+   delete pfec;
+
+   return 0;
+}
@@ -476,7 +476,8 @@ int main(int argc, char *argv[])

 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(BilinearForm &M_, BilinearForm &K_, const Vector &b_)
-   : TimeDependentOperator(M_.Height()), M(M_), K(K_), b(b_), z(M_.Height())
+   : TimeDependentOperator(M_.FESpace()->GetTrueVSize()),
+     M(M_), K(K_), b(b_), z(height)
 {
   Array<int> ess_tdof_list;
   if (M.GetAssemblyLevel() == AssemblyLevel::LEGACY)
@@ -679,10 +679,10 @@ int main(int argc, char *argv[])
 // Implementation of class FE_Evolution
 FE_Evolution::FE_Evolution(ParBilinearForm &M_, ParBilinearForm &K_,
                           const Vector &b_, PrecType prec_type)
-   : TimeDependentOperator(M_.Height()),
+   : TimeDependentOperator(M_.ParFESpace()->GetTrueVSize()),
     b(b_),
     M_solver(M_.ParFESpace()->GetComm()),
-     z(M_.Height())
+     z(height)
 {
   if (M_.GetAssemblyLevel()==AssemblyLevel::LEGACY)
   {
@@ -77,6 +77,7 @@ set(SRCS
  gridfunc.cpp
  hybridization.cpp
  intrules.cpp
+  intrules_cut.cpp
  ceed/interface/basis.cpp
  ceed/interface/restriction.cpp
  ceed/interface/operator.cpp
@@ -88,6 +89,7 @@ set(SRCS
  ceed/solvers/algebraic.cpp
  ceed/solvers/full-assembly.cpp
  ceed/solvers/solvers-atpmg.cpp
+  kdtree.cpp
  linearform.cpp
  linearform_ext.cpp
  lininteg.cpp
@@ -95,9 +97,6 @@ set(SRCS
  lor/lor_ads.cpp
  lor/lor_ams.cpp
  lor/lor_batched.cpp
-  lor/lor_h1.cpp
-  lor/lor_nd.cpp
-  lor/lor_rt.cpp
  multigrid.cpp
  nonlinearform.cpp
  nonlinearform_ext.cpp
@@ -185,6 +184,7 @@ set(HDRS
  gridfunc.hpp
  hybridization.hpp
  intrules.hpp
+  intrules_cut.hpp
  ceed/interface/basis.hpp
  ceed/interface/integrator.hpp
  ceed/interface/interface.hpp
@@ -199,6 +199,7 @@ set(HDRS
  ceed/solvers/algebraic.hpp
  ceed/solvers/full-assembly.hpp
  ceed/solvers/solvers-atpmg.hpp
+  kdtree.hpp
  linearform.hpp
  linearform_ext.hpp
  lininteg.hpp
@@ -267,7 +268,8 @@ if (MFEM_USE_MPI)
    pgridfunc.cpp
    plinearform.cpp
    pnonlinearform.cpp
-    prestriction.cpp)
+    prestriction.cpp
+    pblockform.cpp)
  # If this list (HDRS -> HEADERS) is used for install, we probably want the
  # headers added all the time.
  list(APPEND HDRS
@@ -276,7 +278,8 @@ if (MFEM_USE_MPI)
    pgridfunc.hpp
    plinearform.hpp
    pnonlinearform.hpp
-    prestriction.hpp)
+    prestriction.hpp
+    pblockform.hpp)
 endif()

 convert_filenames_to_full_paths(SRCS)
@@ -101,6 +101,7 @@ BilinearForm::BilinearForm (FiniteElementSpace * f, BilinearForm * bf, int ps)

   // Copy the pointers to the integrators
   domain_integs = bf->domain_integs;
+   domain_integs_marker = bf->domain_integs_marker;

   boundary_integs = bf->boundary_integs;
   boundary_integs_marker = bf->boundary_integs_marker;
@@ -433,6 +434,10 @@ void BilinearForm::Assemble(int skip_zeros)
      // Element-wise integration
      for (int i = 0; i < fes -> GetNE(); i++)
      {
+         // Set both doftrans (potentially needed to assemble the element
+         // matrix) and vdofs, which is also needed when the element matrices
+         // are pre-assembled.
+         doftrans = fes->GetElementVDofs(i, vdofs);
         if (element_matrices)
         {
            elmat_p = &(*element_matrices)(i);
@@ -440,6 +445,8 @@ void BilinearForm::Assemble(int skip_zeros)
         else
         {
            const int elem_attr = fes->GetMesh()->GetAttribute(i);
+            eltrans = fes->GetElementTransformation(i);
+
            elmat.SetSize(0);
            for (int k = 0; k < domain_integs.Size(); k++)
            {
@@ -447,9 +454,8 @@ void BilinearForm::Assemble(int skip_zeros)
                    (*(domain_integs_marker[k]))[elem_attr-1] == 1)
                   && !domain_integs[k]->Patchwise())
               {
-                  const FiniteElement &fe = *fes->GetFE(i);
-                  eltrans = fes->GetElementTransformation(i);
-                  domain_integs[k]->AssembleElementMatrix(fe, *eltrans, elemmat);
+                  domain_integs[k]->AssembleElementMatrix(*fes->GetFE(i),
+                                                          *eltrans, elemmat);
                  if (elmat.Size() == 0)
                  {
                     elmat = elemmat;
@@ -468,7 +474,6 @@ void BilinearForm::Assemble(int skip_zeros)
            {
               elmat_p = &elmat;
            }
-            doftrans = fes->GetElementVDofs(i, vdofs);
            if (doftrans)
            {
               doftrans->TransformDual(elmat);
@@ -1072,7 +1077,8 @@ void BilinearForm::EliminateEssentialBCFromDofs(
 void BilinearForm::EliminateEssentialBCFromDofs (const Array<int> &ess_dofs,
                                                 DiagonalPolicy dpolicy)
 {
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(ess_dofs.Size() == height,
+               "incorrect dof Array size: " << ess_dofs.Size() << ' ' << height);

   for (int i = 0; i < ess_dofs.Size(); i++)
      if (ess_dofs[i] < 0)
@@ -1084,7 +1090,8 @@ void BilinearForm::EliminateEssentialBCFromDofs (const Array<int> &ess_dofs,
 void BilinearForm::EliminateEssentialBCFromDofsDiag (const Array<int> &ess_dofs,
                                                     double value)
 {
-   MFEM_ASSERT(ess_dofs.Size() == height, "incorrect dof Array size");
+   MFEM_ASSERT(ess_dofs.Size() == height,
+               "incorrect dof Array size: " << ess_dofs.Size() << ' ' << height);

   for (int i = 0; i < ess_dofs.Size(); i++)
      if (ess_dofs[i] < 0)
@@ -1222,11 +1229,14 @@ MixedBilinearForm::MixedBilinearForm (FiniteElementSpace *tr_fes,

   // Copy the pointers to the integrators
   domain_integs = mbf->domain_integs;
-   boundary_integs = mbf->boundary_integs;
-   trace_face_integs = mbf->trace_face_integs;
-   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
+   domain_integs_marker = mbf->domain_integs_marker;

+   boundary_integs = mbf->boundary_integs;
   boundary_integs_marker = mbf->boundary_integs_marker;
+
+   trace_face_integs = mbf->trace_face_integs;
+
+   boundary_trace_face_integs = mbf->boundary_trace_face_integs;
   boundary_trace_face_integs_marker = mbf->boundary_trace_face_integs_marker;

   assembly = AssemblyLevel::LEGACY;
@@ -1349,6 +1359,14 @@ void MixedBilinearForm::GetBlocks(Array2D<SparseMatrix *> &blocks) const
 void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi)
 {
   domain_integs.Append (bfi);
+   domain_integs_marker.Append(NULL); // NULL marker means apply everywhere
+}
+
+void MixedBilinearForm::AddDomainIntegrator (BilinearFormIntegrator * bfi,
+                                             Array<int> &elem_marker)
+{
+   domain_integs.Append (bfi);
+   domain_integs_marker.Append(&elem_marker);
 }

 void MixedBilinearForm::AddBoundaryIntegrator (BilinearFormIntegrator * bfi)
@@ -1383,7 +1401,7 @@ void MixedBilinearForm::AddBdrTraceFaceIntegrator(BilinearFormIntegrator *bfi,
   boundary_trace_face_integs_marker.Append(&bdr_marker);
 }

-void MixedBilinearForm::Assemble (int skip_zeros)
+void MixedBilinearForm::Assemble(int skip_zeros)
 {
   if (ext)
   {
@@ -1405,8 +1423,20 @@ void MixedBilinearForm::Assemble (int skip_zeros)

   if (domain_integs.Size())
   {
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         if (domain_integs_marker[k] != NULL)
+         {
+            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
+                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
+                        "invalid element marker for domain integrator #"
+                        << k << ", counting from zero");
+         }
+      }
+
      for (int i = 0; i < test_fes -> GetNE(); i++)
      {
+         const int elem_attr = mesh->GetAttribute(i);
         dom_dof_trans = trial_fes -> GetElementVDofs (i, trial_vdofs);
         ran_dof_trans = test_fes  -> GetElementVDofs (i, test_vdofs);
         eltrans = test_fes -> GetElementTransformation (i);
@@ -1415,10 +1445,14 @@ void MixedBilinearForm::Assemble (int skip_zeros)
         elmat = 0.0;
         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
-                                                        *test_fes  -> GetFE(i),
-                                                        *eltrans, elemmat);
-            elmat += elemmat;
+            if (domain_integs_marker[k] == NULL ||
+                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+            {
+               domain_integs[k] -> AssembleElementMatrix2 (*trial_fes -> GetFE(i),
+                                                           *test_fes  -> GetFE(i),
+                                                           *eltrans, elemmat);
+               elmat += elemmat;
+            }
         }
         if (ran_dof_trans || dom_dof_trans)
         {
@@ -1941,41 +1975,56 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      return;
   }

-   Array<int> dom_vdofs, ran_vdofs;
-   ElementTransformation *T;
+   ElementTransformation *eltrans;
   DofTransformation * dom_dof_trans;
   DofTransformation * ran_dof_trans;
-   const FiniteElement *dom_fe, *ran_fe;
-   DenseMatrix totelmat, elmat;
+   DenseMatrix elmat;
+
+   Mesh *mesh = test_fes->GetMesh();

   if (mat == NULL)
   {
      mat = new SparseMatrix(height, width);
   }

-   if (domain_integs.Size() > 0)
+   if (domain_integs.Size())
   {
+      for (int k = 0; k < domain_integs.Size(); k++)
+      {
+         if (domain_integs_marker[k] != NULL)
+         {
+            MFEM_VERIFY(domain_integs_marker[k]->Size() ==
+                        (mesh->attributes.Size() ? mesh->attributes.Max() : 0),
+                        "invalid element marker for domain integrator #"
+                        << k << ", counting from zero");
+         }
+      }
+
      for (int i = 0; i < test_fes->GetNE(); i++)
      {
-         dom_dof_trans = trial_fes->GetElementVDofs(i, dom_vdofs);
-         ran_dof_trans = test_fes->GetElementVDofs(i, ran_vdofs);
-         T = test_fes->GetElementTransformation(i);
-         dom_fe = trial_fes->GetFE(i);
-         ran_fe = test_fes->GetFE(i);
+         const int elem_attr = mesh->GetAttribute(i);
+         dom_dof_trans = trial_fes->GetElementVDofs(i, trial_vdofs);
+         ran_dof_trans = test_fes->GetElementVDofs(i, test_vdofs);
+         eltrans = test_fes->GetElementTransformation(i);

-         domain_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                  totelmat);
-         for (int j = 1; j < domain_integs.Size(); j++)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < domain_integs.Size(); k++)
         {
-            domain_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                     elmat);
-            totelmat += elmat;
+            if (domain_integs_marker[k] == NULL ||
+                (*(domain_integs_marker[k]))[elem_attr-1] == 1)
+            {
+               domain_integs[k]->AssembleElementMatrix2(*trial_fes->GetFE(i),
+                                                        *test_fes->GetFE(i),
+                                                        *eltrans, elemmat);
+               elmat += elemmat;
+            }
         }
         if (ran_dof_trans || dom_dof_trans)
         {
-            TransformPrimal(ran_dof_trans, dom_dof_trans, totelmat);
+            TransformPrimal(ran_dof_trans, dom_dof_trans, elemmat);
         }
-         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
+         mat->SetSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
      }
   }

@@ -1984,21 +2033,20 @@ void DiscreteLinearOperator::Assemble(int skip_zeros)
      const int nfaces = test_fes->GetMesh()->GetNumFaces();
      for (int i = 0; i < nfaces; i++)
      {
-         trial_fes->GetFaceVDofs(i, dom_vdofs);
-         test_fes->GetFaceVDofs(i, ran_vdofs);
-         T = test_fes->GetMesh()->GetFaceTransformation(i);
-         dom_fe = trial_fes->GetFaceElement(i);
-         ran_fe = test_fes->GetFaceElement(i);
+         trial_fes->GetFaceVDofs(i, trial_vdofs);
+         test_fes->GetFaceVDofs(i, test_vdofs);
+         eltrans = test_fes->GetMesh()->GetFaceTransformation(i);

-         trace_face_integs[0]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                      totelmat);
-         for (int j = 1; j < trace_face_integs.Size(); j++)
+         elmat.SetSize(test_vdofs.Size(), trial_vdofs.Size());
+         elmat = 0.0;
+         for (int k = 0; k < trace_face_integs.Size(); k++)
         {
-            trace_face_integs[j]->AssembleElementMatrix2(*dom_fe, *ran_fe, *T,
-                                                         elmat);
-            totelmat += elmat;
+            trace_face_integs[k]->AssembleElementMatrix2(*trial_fes->GetFaceElement(i),
+                                                         *test_fes->GetFaceElement(i),
+                                                         *eltrans, elemmat);
+            elmat += elemmat;
         }
-         mat->SetSubMatrix(ran_vdofs, dom_vdofs, totelmat, skip_zeros);
+         mat->SetSubMatrix(test_vdofs, trial_vdofs, elmat, skip_zeros);
      }
   }
 }
@@ -100,7 +100,7 @@ protected:
   /// Includes all by default.
   /// 0 - ignore attribute
   /// 1 - include attribute
-   Array<Array<int>*>             domain_integs_marker;
+   Array<Array<int>*> domain_integs_marker; ///< Entries are not owned.

   /// Set of Boundary Integrators to be applied.
   Array<BilinearFormIntegrator*> boundary_integs;
@@ -254,6 +254,12 @@ public:
   /// Access all the integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }

+   /// @brief Access all boundary markers added with AddDomainIntegrator().
+   ///
+   /// If no marker was specified when the integrator was added, the
+   /// corresponding pointer (to Array<int>) will be NULL. */
+   Array<Array<int>*> *GetDBFI_Marker() { return &domain_integs_marker; }
+
   /// Access all the integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
   /** @brief Access all boundary markers added with AddBoundaryIntegrator().
@@ -452,7 +458,7 @@ public:
       practice it is convenient to have it in transposed form for
       construction of RAP operators in matrix-free methods. */
   virtual const Operator *GetOutputRestrictionTranspose() const
-   { return GetOutputProlongation(); }
+   { return fes->GetRestrictionTransposeOperator(); }
   /// Get the output finite element space restriction matrix
   virtual const Operator *GetOutputRestriction() const
   { return GetRestriction(); }
@@ -716,10 +722,13 @@ protected:

   /// Domain integrators.
   Array<BilinearFormIntegrator*> domain_integs;
+   /// Entries are not owned.
+   Array<Array<int>*> domain_integs_marker;

   /// Boundary integrators.
   Array<BilinearFormIntegrator*> boundary_integs;
-   Array<Array<int>*> boundary_integs_marker; ///< Entries are not owned.
+   /// Entries are not owned.
+   Array<Array<int>*> boundary_integs_marker;

   /// Trace face (skeleton) integrators.
   Array<BilinearFormIntegrator*> trace_face_integs;
@@ -799,12 +808,16 @@ public:
   /// Adds a domain integrator. Assumes ownership of @a bfi.
   void AddDomainIntegrator(BilinearFormIntegrator *bfi);

+   /// Adds a domain integrator. Assumes ownership of @a bfi.
+   void AddDomainIntegrator(BilinearFormIntegrator *bfi,
+                            Array<int> &elem_marker);
+
   /// Adds a boundary integrator. Assumes ownership of @a bfi.
   void AddBoundaryIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary integrator. Assumes ownership of @a bfi.
-   void AddBoundaryIntegrator (BilinearFormIntegrator * bfi,
-                               Array<int> &bdr_marker);
+   void AddBoundaryIntegrator(BilinearFormIntegrator * bfi,
+                              Array<int> &bdr_marker);

   /** @brief Add a trace face integrator. Assumes ownership of @a bfi.

@@ -814,14 +827,18 @@ public:
   void AddTraceFaceIntegrator(BilinearFormIntegrator *bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi);
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi);

   /// Adds a boundary trace face integrator. Assumes ownership of @a bfi.
-   void AddBdrTraceFaceIntegrator (BilinearFormIntegrator * bfi,
-                                   Array<int> &bdr_marker);
+   void AddBdrTraceFaceIntegrator(BilinearFormIntegrator * bfi,
+                                  Array<int> &bdr_marker);

   /// Access all integrators added with AddDomainIntegrator().
   Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
+   /** @brief Access all domain markers added with AddDomainIntegrator().
+       If no marker was specified when the integrator was added, the
+       corresponding pointer (to Array<int>) will be NULL. */
+   Array<Array<int>*> *GetDBFI_Marker() { return &domain_integs_marker; }

   /// Access all integrators added with AddBoundaryIntegrator().
   Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
@@ -1059,6 +1076,9 @@ public:
   /// Adds a domain interpolator. Assumes ownership of @a di.
   void AddDomainInterpolator(DiscreteInterpolator *di)
   { AddDomainIntegrator(di); }
+   void AddDomainInterpolator(DiscreteInterpolator *di,
+                              Array<int> &elem_marker)
+   { AddDomainIntegrator(di, elem_marker); }

   /// Adds a trace face interpolator. Assumes ownership of @a di.
   void AddTraceFaceInterpolator(DiscreteInterpolator *di)
@@ -1066,6 +1086,7 @@ public:

   /// Access all interpolators added with AddDomainInterpolator().
   Array<BilinearFormIntegrator*> *GetDI() { return &domain_integs; }
+   Array<Array<int>*> *GetDI_Marker() { return &domain_integs_marker; }

   /// Set the desired assembly level. The default is AssemblyLevel::FULL.
   /** This method must be called before assembly. */
@@ -264,6 +264,14 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      localX.SetSize(elem_restrict->Height(), Device::GetDeviceMemoryType());
      localY.SetSize(elem_restrict->Height(), Device::GetDeviceMemoryType());
      localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
+
+      // Gather the attributes on the host from all the elements
+      const Mesh &mesh = *trial_fes->GetMesh();
+      elem_attributes.SetSize(mesh.GetNE());
+      for (int i = 0; i < mesh.GetNE(); ++i)
+      {
+         elem_attributes[i] = mesh.GetAttribute(i);
+      }
   }

   // Construct face restriction operators only if the bilinear form has
@@ -289,6 +297,46 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
      bdr_face_X.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      bdr_face_Y.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
      bdr_face_Y.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device
+
+      const Mesh &mesh = *trial_fes->GetMesh();
+      // See LinearFormExtension::Update for explanation of f_to_be logic.
+      std::unordered_map<int,int> f_to_be;
+      for (int i = 0; i < mesh.GetNBE(); ++i)
+      {
+         const int f = mesh.GetBdrElementFaceIndex(i);
+         f_to_be[f] = i;
+      }
+      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
+      bdr_attributes.SetSize(nf_bdr);
+      int f_ind = 0;
+      int missing_bdr_elems = 0;
+      for (int f = 0; f < mesh.GetNumFaces(); ++f)
+      {
+         if (!mesh.GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
+         {
+            continue;
+         }
+         int attribute = 1; // default value
+         if (f_to_be.find(f) != f_to_be.end())
+         {
+            const int be = f_to_be[f];
+            attribute = mesh.GetBdrAttribute(be);
+         }
+         else
+         {
+            // If a boundary face does not correspond to the a boundary element,
+            // we assign it the default attribute of 1. We also generate a
+            // warning at runtime with the number of such missing elements.
+            ++missing_bdr_elems;
+         }
+         bdr_attributes[f_ind] = attribute;
+         ++f_ind;
+      }
+      if (missing_bdr_elems)
+      {
+         MFEM_WARNING("Missing " << missing_bdr_elems << " boundary elements "
+                      "for boundary faces.");
+      }
   }
 }

@@ -458,11 +506,13 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   {
      if (iSz)
      {
+         Array<Array<int>*> &elem_markers = *a->GetDBFI_Marker();
         elem_restrict->Mult(x, localX);
         localY = 0.0;
         for (int i = 0; i < iSz; ++i)
         {
-            integrators[i]->AddMultPA(localX, localY);
+            AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
+                               false, localY);
         }
         elem_restrict->MultTranspose(localY, y);
      }
@@ -495,17 +545,21 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
   const bool has_bdr_integs = (n_bdr_face_integs > 0 || n_bdr_integs > 0);
   if (bdr_face_restrict_lex && has_bdr_integs)
   {
+      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
+      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
      bdr_face_restrict_lex->Mult(x, bdr_face_X);
      if (bdr_face_X.Size()>0)
      {
         bdr_face_Y = 0.0;
         for (int i = 0; i < n_bdr_integs; ++i)
         {
-            bdr_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
+                               false, bdr_face_Y);
         }
         for (int i = 0; i < n_bdr_face_integs; ++i)
         {
-            bdr_face_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
+            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                               bdr_attributes, false, bdr_face_Y);
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
@@ -518,11 +572,13 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
   const int iSz = integrators.Size();
   if (elem_restrict)
   {
+      Array<Array<int>*> &elem_markers = *a->GetDBFI_Marker();
      elem_restrict->Mult(x, localX);
      localY = 0.0;
      for (int i = 0; i < iSz; ++i)
      {
-         integrators[i]->AddMultTransposePA(localX, localY);
+         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
+                            true, localY);
      }
      elem_restrict->MultTranspose(localY, y);
   }
@@ -552,23 +608,85 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
      }
   }

-   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
-   const int bFISz = bdrFaceIntegrators.Size();
-   if (bdr_face_restrict_lex && bFISz>0)
+   Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
+   Array<BilinearFormIntegrator*> &bdr_face_integs = *a->GetBFBFI();
+   const int n_bdr_integs = bdr_integs.Size();
+   const int n_bdr_face_integs = bdr_face_integs.Size();
+   const bool has_bdr_integs = (n_bdr_face_integs > 0 || n_bdr_integs > 0);
+   if (bdr_face_restrict_lex && has_bdr_integs)
   {
+      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
+      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
+
      bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size()>0)
+      if (bdr_face_X.Size() > 0)
      {
         bdr_face_Y = 0.0;
-         for (int i = 0; i < bFISz; ++i)
+         for (int i = 0; i < n_bdr_integs; ++i)
         {
-            bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
+            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
+                               true, bdr_face_Y);
+         }
+         for (int i = 0; i < n_bdr_face_integs; ++i)
+         {
+            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
+                               bdr_attributes, true, bdr_face_Y);
         }
         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
      }
   }
 }

+// Compute kernels for PABilinearFormExtension::AddMultWithMarkers.
+// Cannot be in member function with non-public visibility.
+static void AddWithMarkers_(
+   const int ne,
+   const int nd,
+   const Vector &x,
+   const Array<int> &markers,
+   const Array<int> &attributes,
+   Vector &y)
+{
+   const auto d_x = Reshape(x.Read(), nd, ne);
+   const auto d_m = Reshape(markers.Read(), markers.Size());
+   const auto d_attr = Reshape(attributes.Read(), ne);
+   auto d_y = Reshape(y.ReadWrite(), nd, ne);
+   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int attr = d_attr[e];
+      if (d_m[attr - 1] == 0) { return; }
+      for (int i = 0; i < nd; ++i)
+      {
+         d_y(i, e) += d_x(i, e);
+      }
+   });
+}
+
+void PABilinearFormExtension::AddMultWithMarkers(
+   const BilinearFormIntegrator &integ,
+   const Vector &x,
+   const Array<int> *markers,
+   const Array<int> &attributes,
+   const bool transpose,
+   Vector &y) const
+{
+   if (markers)
+   {
+      tmp_evec.SetSize(y.Size());
+      tmp_evec = 0.0;
+      if (transpose) { integ.AddMultTransposePA(x, tmp_evec); }
+      else { integ.AddMultPA(x, tmp_evec); }
+      const int ne = attributes.Size();
+      const int nd = x.Size() / ne;
+      AddWithMarkers_(ne, nd, tmp_evec, *markers, attributes, y);
+   }
+   else
+   {
+      if (transpose) { integ.AddMultTransposePA(x, y); }
+      else { integ.AddMultPA(x, y); }
+   }
+}
+
 // Data and methods for element-assembled bilinear forms
 EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
   : PABilinearFormExtension(form),
@@ -68,6 +68,9 @@ class PABilinearFormExtension : public BilinearFormExtension
 {
 protected:
   const FiniteElementSpace *trial_fes, *test_fes; // Not owned
+   /// Attributes of all mesh elements.
+   Array<int> elem_attributes, bdr_attributes;
+   mutable Vector tmp_evec; // Work array
   mutable Vector localX, localY;
   mutable Vector int_face_X, int_face_Y;
   mutable Vector bdr_face_X, bdr_face_Y;
@@ -91,6 +94,25 @@ public:

 protected:
   void SetupRestrictionOperators(const L2FaceValues m);
+
+   /// @brief Accumulate the action (or transpose) of the integrator on @a x
+   /// into @a y, taking into account the (possibly null) @a markers array.
+   ///
+   /// If @a markers is non-null, then only those elements or boundary elements
+   /// whose attribute is marked in the markers array will be added to @a y.
+   ///
+   /// @param integ The integrator (domain, boundary, or boundary face).
+   /// @param x Input E-vector.
+   /// @param markers Marked attributes (possibly null, meaning all attributes).
+   /// @param attributes Array of element or boundary element attributes.
+   /// @param transpose Compute the action or transpose of the integrator .
+   /// @param y Output E-vector
+   void AddMultWithMarkers(const BilinearFormIntegrator &integ,
+                           const Vector &x,
+                           const Array<int> *markers,
+                           const Array<int> &attributes,
+                           const bool transpose,
+                           Vector &y) const;
 };

 /// Data and methods for element-assembled bilinear forms
@@ -1340,10 +1340,11 @@ void MassIntegrator::AssembleElementMatrix2(
   for (int i = 0; i < ir->GetNPoints(); i++)
   {
      const IntegrationPoint &ip = ir->IntPoint(i);
-      trial_fe.CalcShape(ip, shape);
-      test_fe.CalcShape(ip, te_shape);
-
      Trans.SetIntPoint (&ip);
+
+      trial_fe.CalcPhysShape(Trans, shape);
+      test_fe.CalcPhysShape(Trans, te_shape);
+
      w = Trans.Weight() * ip.weight;
      if (Q)
      {
@@ -2456,7 +2457,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix(
 {
   int dof = el.GetDof();
   int spaceDim = Trans.GetSpaceDim();
-   int vdim = std::max(spaceDim, el.GetVDim());
+   int vdim = std::max(spaceDim, el.GetRangeDim());

   double w;

@@ -2524,7 +2525,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
   {
      // assume test_fe is scalar FE and trial_fe is vector FE
      int spaceDim = Trans.GetSpaceDim();
-      int vdim = std::max(spaceDim, trial_fe.GetVDim());
+      int vdim = std::max(spaceDim, trial_fe.GetRangeDim());
      int trial_dof = trial_fe.GetDof();
      int test_dof = test_fe.GetDof();
      double w;
@@ -2622,8 +2623,8 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
   {
      // assume both test_fe and trial_fe are vector FE
      int spaceDim = Trans.GetSpaceDim();
-      int trial_vdim = std::max(spaceDim, trial_fe.GetVDim());
-      int test_vdim = std::max(spaceDim, test_fe.GetVDim());
+      int trial_vdim = std::max(spaceDim, trial_fe.GetRangeDim());
+      int test_vdim = std::max(spaceDim, test_fe.GetRangeDim());
      int trial_dof = trial_fe.GetDof();
      int test_dof = test_fe.GetDof();
      double w;
@@ -12,10 +12,14 @@
 #ifndef MFEM_LIBCEED_CEED
 #define MFEM_LIBCEED_CEED

+#include "../../../config/config.hpp"
+
 #ifdef MFEM_USE_CEED
+
 #include <ceed.h>
-#if !CEED_VERSION_GE(0,10,0)
-#error MFEM requires a libCEED version >= 0.10.0
+
+#if !CEED_VERSION_GE(0,12,0)
+#error MFEM requires a libCEED version >= 0.12.0
 #endif

 namespace mfem
@@ -294,14 +294,14 @@ public:
                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(build_oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
+                              CEED_BASIS_NONE, quadCoeff->coeffVector);
      }
      CeedOperatorSetField(build_oper, "dx", mesh_restr,
                           mesh_basis, CEED_VECTOR_ACTIVE);
      CeedOperatorSetField(build_oper, "weights", CEED_ELEMRESTRICTION_NONE,
                           mesh_basis, CEED_VECTOR_NONE);
      CeedOperatorSetField(build_oper, "qdata", restr_i,
-                           CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                           CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);

      // Compute the quadrature data for the operator.
      CeedOperatorApply(build_oper, node_coords, qdata, CEED_REQUEST_IMMEDIATE);
@@ -355,7 +355,7 @@ public:
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "u", trial_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "u", trial_restr, trial_basis, CEED_VECTOR_ACTIVE);
@@ -369,14 +369,14 @@ public:
            break;
      }
      // qdata
-      CeedOperatorSetField(oper, "qdata", restr_i, CEED_BASIS_COLLOCATED,
+      CeedOperatorSetField(oper, "qdata", restr_i, CEED_BASIS_NONE,
                           qdata);
      // output
      switch (op.test_op)
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "v", test_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "v", test_restr, test_basis, CEED_VECTOR_ACTIVE);
@@ -685,14 +685,14 @@ public:
                                nelem, nqpts, ncomp, strides,
                                &quadCoeff->restr);
         CeedOperatorSetField(oper, "coeff", quadCoeff->restr,
-                              CEED_BASIS_COLLOCATED, quadCoeff->coeffVector);
+                              CEED_BASIS_NONE, quadCoeff->coeffVector);
      }
      // input
      switch (op.trial_op)
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "u", trial_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "u", trial_restr, trial_basis,
@@ -718,7 +718,7 @@ public:
      {
         case EvalMode::None:
            CeedOperatorSetField(oper, "v", test_restr,
-                                 CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
+                                 CEED_BASIS_NONE, CEED_VECTOR_ACTIVE);
            break;
         case EvalMode::Interp:
            CeedOperatorSetField(oper, "v", test_restr, test_basis,
@@ -132,24 +132,24 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(oper, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(oper, &ceed); PCeedChk(ierr);

   CeedQFunction qf;
   bool isComposite;
-   ierr = CeedOperatorIsComposite(oper, &isComposite); CeedChk(ierr);
+   ierr = CeedOperatorIsComposite(oper, &isComposite); PCeedChk(ierr);
   CeedOperator *subops;
   if (isComposite)
   {
 #if CEED_VERSION_GE(0, 10, 2)
-      ierr = CeedCompositeOperatorGetSubList(oper, &subops); CeedChk(ierr);
+      ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr);
 #else
-      ierr = CeedOperatorGetSubList(oper, &subops); CeedChk(ierr);
+      ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr);
 #endif
-      ierr = CeedOperatorGetQFunction(subops[0], &qf); CeedChk(ierr);
+      ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr);
   }
   else
   {
-      ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+      ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   }
   CeedInt numinputfields, numoutputfields;
   ierr = CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
@@ -157,12 +157,12 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   if (isComposite)
   {
      ierr = CeedOperatorGetFields(subops[0], &numinputfields, &inputfields,
-                                   &numoutputfields, NULL); CeedChk(ierr);
+                                   &numoutputfields, NULL); PCeedChk(ierr);
   }
   else
   {
      ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
-                                   &numoutputfields, NULL); CeedChk(ierr);
+                                   &numoutputfields, NULL); PCeedChk(ierr);
   }

   CeedVector if_vector;
@@ -170,7 +170,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
   int found_index = -1;
   for (int i = 0; i < numinputfields; ++i)
   {
-      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr);
      if (if_vector == CEED_VECTOR_ACTIVE)
      {
         if (found)
@@ -13,12 +13,15 @@
 #define MFEM_LIBCEED_UTIL

 #include "../../../config/config.hpp"
+#include "../../../general/error.hpp"
+
 #include <functional>
 #include <string>
 #include <tuple>
 #include <unordered_map>

 #include "ceed.hpp"
+
 #ifdef MFEM_USE_CEED
 #include <ceed/backend.h>  // for CeedOperatorField
 #endif
@@ -170,10 +173,12 @@ namespace internal
 {

 #ifdef MFEM_USE_CEED
+
 /** @warning These maps have a tendency to create bugs when adding new "types"
    of CeedBasis and CeedElemRestriction. */
 extern ceed::BasisMap ceed_basis_map;
 extern ceed::RestrMap ceed_restr_map;
+
 #endif

 } // namespace internal
@@ -99,7 +99,7 @@ int CeedOperatorGetSize(CeedOperator oper, CeedInt * size)
 {
   CeedSize in_len, out_len;
   int ierr = CeedOperatorGetActiveVectorLengths(oper, &in_len, &out_len);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   *size = (CeedInt)in_len;
   MFEM_VERIFY(in_len == out_len, "not a square CeedOperator");
   MFEM_VERIFY(in_len == *size, "size overflow");
@@ -378,67 +378,68 @@ int AlgebraicInterpolation::Initialize(

   CeedSize height, width;
   ierr = CeedElemRestrictionGetLVectorSize(erestrictu_coarse, &width);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedElemRestrictionGetLVectorSize(erestrictu_fine, &height);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // interpolation qfunction
   const int bp3_ncompu = 1;
   CeedQFunction l_qf_restrict, l_qf_prolong;
   ierr = CeedQFunctionCreateIdentity(ceed, bp3_ncompu, CEED_EVAL_NONE,
-                                      CEED_EVAL_INTERP, &l_qf_restrict); CeedChk(ierr);
+                                      CEED_EVAL_INTERP, &l_qf_restrict); PCeedChk(ierr);
   ierr = CeedQFunctionCreateIdentity(ceed, bp3_ncompu, CEED_EVAL_INTERP,
-                                      CEED_EVAL_NONE, &l_qf_prolong); CeedChk(ierr);
+                                      CEED_EVAL_NONE, &l_qf_prolong); PCeedChk(ierr);

   qf_restrict = l_qf_restrict;
   qf_prolong = l_qf_prolong;

   CeedVector c_fine_multiplicity;
-   ierr = CeedVectorCreate(ceed, height, &c_fine_multiplicity); CeedChk(ierr);
-   ierr = CeedVectorSetValue(c_fine_multiplicity, 0.0); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &c_fine_multiplicity); PCeedChk(ierr);
+   ierr = CeedVectorSetValue(c_fine_multiplicity, 0.0); PCeedChk(ierr);

   // Create the restriction operator
   // Restriction - Fine to coarse
   ierr = CeedOperatorCreate(ceed, qf_restrict, CEED_QFUNCTION_NONE,
-                             CEED_QFUNCTION_NONE, &op_restrict); CeedChk(ierr);
+                             CEED_QFUNCTION_NONE, &op_restrict); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_restrict, "input", erestrictu_fine,
-                               CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               CEED_BASIS_NONE, CEED_VECTOR_ACTIVE); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_restrict, "output", erestrictu_coarse,
-                               basisctof, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               basisctof, CEED_VECTOR_ACTIVE); PCeedChk(ierr);

   // Interpolation - Coarse to fine
   // Create the prolongation operator
   ierr =  CeedOperatorCreate(ceed, qf_prolong, CEED_QFUNCTION_NONE,
-                              CEED_QFUNCTION_NONE, &op_interp); CeedChk(ierr);
+                              CEED_QFUNCTION_NONE, &op_interp); PCeedChk(ierr);
   ierr =  CeedOperatorSetField(op_interp, "input", erestrictu_coarse,
-                                basisctof, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                                basisctof, CEED_VECTOR_ACTIVE); PCeedChk(ierr);
   ierr = CeedOperatorSetField(op_interp, "output", erestrictu_fine,
-                               CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); CeedChk(ierr);
+                               CEED_BASIS_NONE, CEED_VECTOR_ACTIVE); PCeedChk(ierr);

   ierr = CeedElemRestrictionGetMultiplicity(erestrictu_fine,
-                                             c_fine_multiplicity); CeedChk(ierr);
-   ierr = CeedVectorCreate(ceed, height, &fine_multiplicity_r); CeedChk(ierr);
+                                             c_fine_multiplicity); PCeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &fine_multiplicity_r); PCeedChk(ierr);

   CeedScalar* fine_r_data;
   const CeedScalar* fine_data;
   ierr = CeedVectorGetArrayWrite(fine_multiplicity_r, CEED_MEM_HOST,
-                                  &fine_r_data); CeedChk(ierr);
+                                  &fine_r_data); PCeedChk(ierr);
   ierr = CeedVectorGetArrayRead(c_fine_multiplicity, CEED_MEM_HOST,
-                                 &fine_data); CeedChk(ierr);
+                                 &fine_data); PCeedChk(ierr);
   for (CeedSize i = 0; i < height; ++i)
   {
      fine_r_data[i] = 1.0 / fine_data[i];
   }

-   ierr = CeedVectorRestoreArray(fine_multiplicity_r, &fine_r_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(fine_multiplicity_r, &fine_r_data);
+   PCeedChk(ierr);
   ierr = CeedVectorRestoreArrayRead(c_fine_multiplicity, &fine_data);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&c_fine_multiplicity); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&c_fine_multiplicity); PCeedChk(ierr);

-   ierr = CeedVectorCreate(ceed, height, &fine_work); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &fine_work); PCeedChk(ierr);

-   ierr = CeedVectorCreate(ceed, height, &v_); CeedChk(ierr);
-   ierr = CeedVectorCreate(ceed, width, &u_); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, height, &v_); PCeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, width, &u_); PCeedChk(ierr);

   return 0;
 }
@@ -447,12 +448,12 @@ int AlgebraicInterpolation::Finalize()
 {
   int ierr;

-   ierr = CeedQFunctionDestroy(&qf_restrict); CeedChk(ierr);
-   ierr = CeedQFunctionDestroy(&qf_prolong); CeedChk(ierr);
-   ierr = CeedOperatorDestroy(&op_interp); CeedChk(ierr);
-   ierr = CeedOperatorDestroy(&op_restrict); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&fine_multiplicity_r); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&fine_work); CeedChk(ierr);
+   ierr = CeedQFunctionDestroy(&qf_restrict); PCeedChk(ierr);
+   ierr = CeedQFunctionDestroy(&qf_prolong); PCeedChk(ierr);
+   ierr = CeedOperatorDestroy(&op_interp); PCeedChk(ierr);
+   ierr = CeedOperatorDestroy(&op_restrict); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&fine_multiplicity_r); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&fine_work); PCeedChk(ierr);

   return 0;
 }
@@ -498,8 +499,8 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
   CeedVectorGetCeed(a, &ceed);

   CeedSize length, length2;
-   ierr = CeedVectorGetLength(a, &length); CeedChk(ierr);
-   ierr = CeedVectorGetLength(b, &length2); CeedChk(ierr);
+   ierr = CeedVectorGetLength(a, &length); PCeedChk(ierr);
+   ierr = CeedVectorGetLength(b, &length2); PCeedChk(ierr);
   if (length != length2)
   {
      return CeedError(ceed, 1, "Vector sizes don't match");
@@ -516,14 +517,14 @@ int CeedVectorPointwiseMult(CeedVector a, const CeedVector b)
   }
   CeedScalar *a_data;
   const CeedScalar *b_data;
-   ierr = CeedVectorGetArray(a, mem, &a_data); CeedChk(ierr);
-   ierr = CeedVectorGetArrayRead(b, mem, &b_data); CeedChk(ierr);
+   ierr = CeedVectorGetArray(a, mem, &a_data); PCeedChk(ierr);
+   ierr = CeedVectorGetArrayRead(b, mem, &b_data); PCeedChk(ierr);
   MFEM_VERIFY(int(length) == length, "length overflow");
   mfem::forall(length, [=] MFEM_HOST_DEVICE (int i)
   {a_data[i] *= b_data[i];});

-   ierr = CeedVectorRestoreArray(a, &a_data); CeedChk(ierr);
-   ierr = CeedVectorRestoreArrayRead(b, &b_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(a, &a_data); PCeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(b, &b_data); PCeedChk(ierr);

   return 0;
 }
@@ -11,12 +11,12 @@

 #include "full-assembly.hpp"

+#ifdef MFEM_USE_CEED
+
 #include "../../../linalg/sparsemat.hpp"
 #include "../interface/util.hpp"
 #include "../interface/ceed.hpp"

-#ifdef MFEM_USE_CEED
-
 namespace mfem
 {

@@ -45,32 +45,32 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(op, &ceed); PCeedChk(ierr);

   // Assemble QFunction
   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(op, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedVector assembledqf;
   CeedElemRestriction rstr_q;
   ierr = CeedOperatorLinearAssembleQFunction(
-             op, &assembledqf, &rstr_q, CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
+             op, &assembledqf, &rstr_q, CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);

   CeedSize qflength;
-   ierr = CeedVectorGetLength(assembledqf, &qflength); CeedChk(ierr);
+   ierr = CeedVectorGetLength(assembledqf, &qflength); PCeedChk(ierr);

   CeedOperatorField *input_fields;
   CeedOperatorField *output_fields;
   ierr = CeedOperatorGetFields(op, &numinputfields, &input_fields,
                                &numoutputfields, &output_fields);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Determine active input basis
   CeedQFunctionField *qffields;
   ierr = CeedQFunctionGetFields(qf, &numinputfields, &qffields,
                                 &numoutputfields, NULL);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedInt numemodein = 0, ncomp, dim = 1;
   CeedEvalMode *emodein = NULL;
   CeedBasis basisin = NULL;
@@ -78,28 +78,28 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
   for (CeedInt i=0; i<numinputfields; i++)
   {
      CeedVector vec;
-      ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); PCeedChk(ierr);
      if (vec == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basisin);
-         CeedChk(ierr);
-         ierr = CeedBasisGetNumComponents(basisin, &ncomp); CeedChk(ierr);
-         ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
+         PCeedChk(ierr);
+         ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
+         ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstrin);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         CeedEvalMode emode;
         ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         switch (emode)
         {
            case CEED_EVAL_NONE:
            case CEED_EVAL_INTERP:
-               ierr = CeedHackRealloc(numemodein + 1, &emodein); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodein + 1, &emodein); PCeedChk(ierr);
               emodein[numemodein] = emode;
               numemodein += 1;
               break;
            case CEED_EVAL_GRAD:
-               ierr = CeedHackRealloc(numemodein + dim, &emodein); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodein + dim, &emodein); PCeedChk(ierr);
               for (CeedInt d=0; d<dim; d++)
               {
                  emodein[numemodein+d] = emode;
@@ -116,7 +116,7 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)

   // Determine active output basis
   ierr = CeedQFunctionGetFields(qf, &numinputfields, NULL, &numoutputfields,
-                                 &qffields); CeedChk(ierr);
+                                 &qffields); PCeedChk(ierr);
   CeedInt numemodeout = 0;
   CeedEvalMode *emodeout = NULL;
   CeedBasis basisout = NULL;
@@ -124,27 +124,27 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
   for (CeedInt i=0; i<numoutputfields; i++)
   {
      CeedVector vec;
-      ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); CeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); PCeedChk(ierr);
      if (vec == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basisout);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstrout);
-         CeedChk(ierr);
-         CeedChk(ierr);
+         PCeedChk(ierr);
+         PCeedChk(ierr);
         CeedEvalMode emode;
         ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
-         CeedChk(ierr);
+         PCeedChk(ierr);
         switch (emode)
         {
            case CEED_EVAL_NONE:
            case CEED_EVAL_INTERP:
-               ierr = CeedHackRealloc(numemodeout + 1, &emodeout); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodeout + 1, &emodeout); PCeedChk(ierr);
               emodeout[numemodeout] = emode;
               numemodeout += 1;
               break;
            case CEED_EVAL_GRAD:
-               ierr = CeedHackRealloc(numemodeout + dim, &emodeout); CeedChk(ierr);
+               ierr = CeedHackRealloc(numemodeout + dim, &emodeout); PCeedChk(ierr);
               for (CeedInt d=0; d<dim; d++)
               {
                  emodeout[numemodeout+d] = emode;
@@ -161,47 +161,47 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)

   CeedInt nelem, elemsize, nqpts;
   CeedSize nnodes;
-   ierr = CeedElemRestrictionGetNumElements(rstrin, &nelem); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetElementSize(rstrin, &elemsize); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetLVectorSize(rstrin, &nnodes); CeedChk(ierr);
-   ierr = CeedBasisGetNumQuadraturePoints(basisin, &nqpts); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumElements(rstrin, &nelem); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetElementSize(rstrin, &elemsize); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetLVectorSize(rstrin, &nnodes); PCeedChk(ierr);
+   ierr = CeedBasisGetNumQuadraturePoints(basisin, &nqpts); PCeedChk(ierr);

   // Determine elem_dof relation
   CeedVector index_vec;
-   ierr = CeedVectorCreate(ceed, nnodes, &index_vec); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, nnodes, &index_vec); PCeedChk(ierr);
   CeedScalar *array;
   ierr = CeedVectorGetArrayWrite(index_vec, CEED_MEM_HOST, &array);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   for (CeedSize i = 0; i < nnodes; ++i)
   {
      array[i] = i;
   }
-   ierr = CeedVectorRestoreArray(index_vec, &array); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(index_vec, &array); PCeedChk(ierr);
   CeedVector elem_dof;
-   ierr = CeedVectorCreate(ceed, nelem * elemsize, &elem_dof); CeedChk(ierr);
-   ierr = CeedVectorSetValue(elem_dof, 0.0); CeedChk(ierr);
+   ierr = CeedVectorCreate(ceed, nelem * elemsize, &elem_dof); PCeedChk(ierr);
+   ierr = CeedVectorSetValue(elem_dof, 0.0); PCeedChk(ierr);
   CeedElemRestrictionApply(rstrin, CEED_NOTRANSPOSE, index_vec,
-                            elem_dof, CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
+                            elem_dof, CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);
   const CeedScalar * elem_dof_a;
   ierr = CeedVectorGetArrayRead(elem_dof, CEED_MEM_HOST, &elem_dof_a);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&index_vec); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&index_vec); PCeedChk(ierr);

   // loop over elements and put in SparseMatrix
   // SparseMatrix * out = new SparseMatrix(nnodes, nnodes);
   MFEM_ASSERT(out->Height() == nnodes, "Sizes don't match!");
   MFEM_ASSERT(out->Width() == nnodes, "Sizes don't match!");
   const CeedScalar *interpin, *gradin;
-   ierr = CeedBasisGetInterp(basisin, &interpin); CeedChk(ierr);
-   ierr = CeedBasisGetGrad(basisin, &gradin); CeedChk(ierr);
+   ierr = CeedBasisGetInterp(basisin, &interpin); PCeedChk(ierr);
+   ierr = CeedBasisGetGrad(basisin, &gradin); PCeedChk(ierr);

   const CeedScalar * assembledqfarray;
   ierr = CeedVectorGetArrayRead(assembledqf, CEED_MEM_HOST, &assembledqfarray);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedInt layout[3];
-   ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); CeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstr_q); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); PCeedChk(ierr);
+   ierr = CeedElemRestrictionDestroy(&rstr_q); PCeedChk(ierr);

   // enforce structurally symmetric for later elimination
   const int skip_zeros = 0;
@@ -280,13 +280,13 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
      out->AddSubMatrix(rows, rows, elem_mat, skip_zeros);
   }

-   ierr = CeedVectorRestoreArrayRead(elem_dof, &elem_dof_a); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&elem_dof); CeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(elem_dof, &elem_dof_a); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&elem_dof); PCeedChk(ierr);
   ierr = CeedVectorRestoreArrayRead(assembledqf, &assembledqfarray);
-   CeedChk(ierr);
-   ierr = CeedVectorDestroy(&assembledqf); CeedChk(ierr);
-   ierr = CeedHackFree(&emodein); CeedChk(ierr);
-   ierr = CeedHackFree(&emodeout); CeedChk(ierr);
+   PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&assembledqf); PCeedChk(ierr);
+   ierr = CeedHackFree(&emodein); PCeedChk(ierr);
+   ierr = CeedHackFree(&emodeout); PCeedChk(ierr);

   return 0;
 }
@@ -297,7 +297,7 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)

   CeedSize in_len, out_len;
   ierr = CeedOperatorGetActiveVectorLengths(op, &in_len, &out_len);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   const int nnodes = in_len;
   MFEM_VERIFY(in_len == out_len, "not a square CeedOperator");
   MFEM_VERIFY(in_len == nnodes, "size overflow");
@@ -305,26 +305,26 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
   SparseMatrix *out = new SparseMatrix(nnodes, nnodes);

   bool isComposite;
-   ierr = CeedOperatorIsComposite(op, &isComposite); CeedChk(ierr);
+   ierr = CeedOperatorIsComposite(op, &isComposite); PCeedChk(ierr);
   if (isComposite)
   {
      CeedInt numsub;
      CeedOperator *subops;
 #if CEED_VERSION_GE(0, 10, 2)
      CeedCompositeOperatorGetNumSub(op, &numsub);
-      ierr = CeedCompositeOperatorGetSubList(op, &subops); CeedChk(ierr);
+      ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
 #else
      CeedOperatorGetNumSub(op, &numsub);
-      ierr = CeedOperatorGetSubList(op, &subops); CeedChk(ierr);
+      ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
 #endif
      for (int i = 0; i < numsub; ++i)
      {
-         ierr = CeedSingleOperatorFullAssemble(subops[i], out); CeedChk(ierr);
+         ierr = CeedSingleOperatorFullAssemble(subops[i], out); PCeedChk(ierr);
      }
   }
   else
   {
-      ierr = CeedSingleOperatorFullAssemble(op, out); CeedChk(ierr);
+      ierr = CeedSingleOperatorFullAssemble(op, out); PCeedChk(ierr);
   }
   // enforce structurally symmetric for later elimination
   const int skip_zeros = 0;
@@ -338,4 +338,4 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)

 } // namespace mfem

-#endif
+#endif // MFEM_USE_CEED
@@ -19,6 +19,8 @@
 namespace mfem
 {

+class SparseMatrix;
+
 namespace ceed
 {

@@ -34,6 +36,6 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat);

 } // namespace mfem

-#endif
+#endif // MFEM_USE_CEED

-#endif
+#endif // MFEM_CEED_ASSEMBLE_HPP
@@ -15,8 +15,8 @@
 #include "../interface/util.hpp"

 #ifdef MFEM_USE_CEED
-#include <ceed/backend.h>

+#include <ceed/backend.h>
 #include <math.h>
 // todo: should probably use Ceed memory wrappers instead of calloc/free?
 #include <stdlib.h>
@@ -86,14 +86,14 @@ int CeedATPMGElemRestriction(int order,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedElemRestrictionGetCeed(er_in, &ceed); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetCeed(er_in, &ceed); PCeedChk(ierr);

   CeedInt numelem, numcomp, elemsize;
   CeedSize numnodes;
-   ierr = CeedElemRestrictionGetNumElements(er_in, &numelem); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetLVectorSize(er_in, &numnodes); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetElementSize(er_in, &elemsize); CeedChk(ierr);
-   ierr = CeedElemRestrictionGetNumComponents(er_in, &numcomp); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumElements(er_in, &numelem); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetLVectorSize(er_in, &numnodes); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetElementSize(er_in, &elemsize); PCeedChk(ierr);
+   ierr = CeedElemRestrictionGetNumComponents(er_in, &numcomp); PCeedChk(ierr);
   if (numcomp != 1)
   {
      // todo: multi-component will require more thought
@@ -107,31 +107,31 @@ int CeedATPMGElemRestriction(int order,

   CeedVector in_lvec, in_evec;
   ierr = CeedElemRestrictionCreateVector(er_in, &in_lvec, &in_evec);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Create the elem_dof array from the given high-order ElemRestriction
   // by using it to map the L-vector indices to an E-vector
   CeedScalar * lvec_data;
   ierr = CeedVectorGetArrayWrite(in_lvec, CEED_MEM_HOST, &lvec_data);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   for (CeedSize i = 0; i < numnodes; ++i)
   {
      lvec_data[i] = (CeedScalar) i;
   }
-   ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); CeedChk(ierr);
+   ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); PCeedChk(ierr);
   CeedInt in_layout[3];
-   ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); CeedChk(ierr);
+   ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); PCeedChk(ierr);
   if (in_layout[0] == 0 && in_layout[1] == 0 && in_layout[2] == 0)
   {
      return CeedError(ceed, 1, "Cannot interpret e-vector ordering of given"
                       "CeedElemRestriction!");
   }
   ierr = CeedElemRestrictionApply(er_in, CEED_NOTRANSPOSE, in_lvec, in_evec,
-                                   CEED_REQUEST_IMMEDIATE); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&in_lvec); CeedChk(ierr);
+                                   CEED_REQUEST_IMMEDIATE); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&in_lvec); PCeedChk(ierr);
   const CeedScalar * in_elem_dof;
   ierr = CeedVectorGetArrayRead(in_evec, CEED_MEM_HOST, &in_elem_dof);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   // Create a map (dof_map) that maps high-order ldof indices to
   // low-order ldof indices, with -1 indicating no correspondence
@@ -469,13 +469,13 @@ int CeedATPMGElemRestriction(int order,
                       "CeedATPMGElemRestriction does not yet support this dimension.");
   }

-   ierr = CeedVectorRestoreArrayRead(in_evec, &in_elem_dof); CeedChk(ierr);
-   ierr = CeedVectorDestroy(&in_evec); CeedChk(ierr);
+   ierr = CeedVectorRestoreArrayRead(in_evec, &in_elem_dof); PCeedChk(ierr);
+   ierr = CeedVectorDestroy(&in_evec); PCeedChk(ierr);

   ierr = CeedElemRestrictionCreate(ceed, numelem, coarse_elemsize, numcomp,
                                    0, running_out_ldof_count,
                                    CEED_MEM_HOST, CEED_COPY_VALUES, out_elem_dof,
-                                    er_out); CeedChk(ierr);
+                                    er_out); PCeedChk(ierr);

   delete [] out_elem_dof;

@@ -491,7 +491,7 @@ int CeedBasisATPMGCoarseToFine(Ceed ceed, int P1d, int dim, int order_reduction,
   // calling the following Ceed function)
   int ierr;
   ierr = CeedBasisCreateTensorH1Lagrange(ceed, dim, 1, P1d - order_reduction, P1d,
-                                          CEED_GAUSS_LOBATTO, basisc2f); CeedChk(ierr);
+                                          CEED_GAUSS_LOBATTO, basisc2f); PCeedChk(ierr);
   return 0;
 }

@@ -501,13 +501,13 @@ int CeedBasisATPMGCoarseToFine(CeedBasis basisin,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedBasisGetCeed(basisin, &ceed); CeedChk(ierr);
+   ierr = CeedBasisGetCeed(basisin, &ceed); PCeedChk(ierr);

   CeedInt dim, P1d;
-   ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
-   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); CeedChk(ierr);
+   ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarseToFine(ceed, P1d, dim, order_reduction,
-                                     basisc2f); CeedChk(ierr);
+                                     basisc2f); PCeedChk(ierr);
   return 0;
 }

@@ -518,20 +518,20 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
 {
   int ierr;
   Ceed ceed;
-   ierr = CeedBasisGetCeed(basisin, &ceed); CeedChk(ierr);
+   ierr = CeedBasisGetCeed(basisin, &ceed); PCeedChk(ierr);

   CeedInt dim, ncomp, P1d, Q1d;
-   ierr = CeedBasisGetDimension(basisin, &dim); CeedChk(ierr);
-   ierr = CeedBasisGetNumComponents(basisin, &ncomp); CeedChk(ierr);
-   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); CeedChk(ierr);
-   ierr = CeedBasisGetNumQuadraturePoints1D(basisin, &Q1d); CeedChk(ierr);
+   ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
+   ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basisin, &P1d); PCeedChk(ierr);
+   ierr = CeedBasisGetNumQuadraturePoints1D(basisin, &Q1d); PCeedChk(ierr);

   CeedInt coarse_P1d = P1d - order_reduction;

   const CeedScalar *interp1d;
-   ierr = CeedBasisGetInterp1D(basisin, &interp1d); CeedChk(ierr);
+   ierr = CeedBasisGetInterp1D(basisin, &interp1d); PCeedChk(ierr);
   const CeedScalar * grad1d;
-   ierr = CeedBasisGetGrad1D(basisin, &grad1d); CeedChk(ierr);
+   ierr = CeedBasisGetGrad1D(basisin, &grad1d); PCeedChk(ierr);

   CeedScalar * coarse_interp1d = new CeedScalar[coarse_P1d * Q1d];
   CeedScalar * coarse_grad1d = new CeedScalar[coarse_P1d * Q1d];
@@ -542,14 +542,14 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
   /* one way you might be able to tell is to just run this algorithm
      with coarse_P1d = 2 (i.e., linear) and look for symmetry in the coarse
      basis matrix? */
-   ierr = CeedLobattoQuadrature(P1d, fine_nodal_points, NULL); CeedChk(ierr);
+   ierr = CeedLobattoQuadrature(P1d, fine_nodal_points, NULL); PCeedChk(ierr);
   for (int i = 0; i < P1d; ++i)
   {
      fine_nodal_points[i] = 0.5 * fine_nodal_points[i] + 0.5; // cheating
   }

   const CeedScalar *interp_ctof;
-   ierr = CeedBasisGetInterp1D(basisc2f, &interp_ctof); CeedChk(ierr);
+   ierr = CeedBasisGetInterp1D(basisc2f, &interp_ctof); PCeedChk(ierr);

   for (int i = 0; i < Q1d; ++i)
   {
@@ -568,12 +568,12 @@ int CeedBasisATPMGCoarsen(CeedBasis basisin,
   }

   const CeedScalar * qref1d;
-   ierr = CeedBasisGetQRef(basisin, &qref1d); CeedChk(ierr);
+   ierr = CeedBasisGetQRef(basisin, &qref1d); PCeedChk(ierr);
   const CeedScalar * qweight1d;
-   ierr = CeedBasisGetQWeights(basisin, &qweight1d); CeedChk(ierr);
+   ierr = CeedBasisGetQWeights(basisin, &qweight1d); PCeedChk(ierr);
   ierr = CeedBasisCreateTensorH1(ceed, dim, ncomp,
                                  coarse_P1d, Q1d, coarse_interp1d, coarse_grad1d,
-                                  qref1d, qweight1d, basisout); CeedChk(ierr);
+                                  qref1d, qweight1d, basisout); PCeedChk(ierr);

   delete [] fine_nodal_points;
   delete [] coarse_interp1d;
@@ -593,19 +593,19 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,

   int ierr;
   Ceed ceed;
-   ierr = CeedOperatorGetCeed(oper, &ceed); CeedChk(ierr);
+   ierr = CeedOperatorGetCeed(oper, &ceed); PCeedChk(ierr);

   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
   CeedQFunctionField *inputqfields, *outputqfields;
   ierr = CeedQFunctionGetFields(qf, &numinputfields, &inputqfields,
                                 &numoutputfields, &outputqfields);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   CeedOperatorField *inputfields, *outputfields;
   ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
                                &numoutputfields, &outputfields);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedElemRestriction * er_input = new CeedElemRestriction[numinputfields];
   CeedElemRestriction * er_output = new CeedElemRestriction[numoutputfields];
@@ -619,10 +619,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   for (int i = 0; i < numinputfields; ++i)
   {
      ierr = CeedOperatorFieldGetElemRestriction(inputfields[i],
-                                                 &er_input[i]); CeedChk(ierr);
-      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector[i]); CeedChk(ierr);
+                                                 &er_input[i]); PCeedChk(ierr);
+      ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector[i]);
+      PCeedChk(ierr);
      ierr = CeedOperatorFieldGetBasis(inputfields[i], &basis_input[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      if (if_vector[i] == CEED_VECTOR_ACTIVE)
      {
         if (active_input_basis < 0)
@@ -638,11 +639,11 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   for (int i = 0; i < numoutputfields; ++i)
   {
      ierr = CeedOperatorFieldGetElemRestriction(outputfields[i],
-                                                 &er_output[i]); CeedChk(ierr);
+                                                 &er_output[i]); PCeedChk(ierr);
      ierr = CeedOperatorFieldGetVector(outputfields[i], &of_vector[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      ierr = CeedOperatorFieldGetBasis(outputfields[i], &basis_output[i]);
-      CeedChk(ierr);
+      PCeedChk(ierr);
      if (of_vector[i] == CEED_VECTOR_ACTIVE)
      {
         // should already be coarsened
@@ -659,36 +660,36 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,

   CeedOperator coper;
   ierr = CeedOperatorCreate(ceed, qf, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE,
-                             &coper); CeedChk(ierr);
+                             &coper); PCeedChk(ierr);

   for (int i = 0; i < numinputfields; ++i)
   {
      char * fieldname;
-      ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); CeedChk(ierr);
+      ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); PCeedChk(ierr);
      if (if_vector[i] == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorSetField(coper, fieldname, coarse_er, cbasis,
-                                     if_vector[i]); CeedChk(ierr);
+                                     if_vector[i]); PCeedChk(ierr);
      }
      else
      {
         ierr = CeedOperatorSetField(coper, fieldname, er_input[i], basis_input[i],
-                                     if_vector[i]); CeedChk(ierr);
+                                     if_vector[i]); PCeedChk(ierr);
      }
   }
   for (int i = 0; i < numoutputfields; ++i)
   {
      char * fieldname;
-      ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); CeedChk(ierr);
+      ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); PCeedChk(ierr);
      if (of_vector[i] == CEED_VECTOR_ACTIVE)
      {
         ierr = CeedOperatorSetField(coper, fieldname, coarse_er, cbasis,
-                                     of_vector[i]); CeedChk(ierr);
+                                     of_vector[i]); PCeedChk(ierr);
      }
      else
      {
         ierr = CeedOperatorSetField(coper, fieldname, er_output[i], basis_output[i],
-                                     of_vector[i]); CeedChk(ierr);
+                                     of_vector[i]); PCeedChk(ierr);
      }
   }
   delete [] er_input;
@@ -711,21 +712,21 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
   int ierr;

   CeedQFunction qf;
-   ierr = CeedOperatorGetQFunction(oper, &qf); CeedChk(ierr);
+   ierr = CeedOperatorGetQFunction(oper, &qf); PCeedChk(ierr);
   CeedInt numinputfields, numoutputfields;
   CeedOperatorField *inputfields;
   ierr = CeedOperatorGetFields(oper, &numinputfields, &inputfields,
                                &numoutputfields, NULL);
-   CeedChk(ierr);
+   PCeedChk(ierr);

   CeedBasis basis;
-   ierr = CeedOperatorGetActiveBasis(oper, &basis); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveBasis(oper, &basis); PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarseToFine(basis, basis_ctof_out, order_reduction);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedBasisATPMGCoarsen(basis, *basis_ctof_out, coarse_basis_out,
-                                order_reduction); CeedChk(ierr);
+                                order_reduction); PCeedChk(ierr);
   ierr = CeedATPMGOperator(oper, order_reduction, coarse_er, *coarse_basis_out,
-                            *basis_ctof_out, out); CeedChk(ierr);
+                            *basis_ctof_out, out); PCeedChk(ierr);
   return 0;
 }

@@ -734,11 +735,11 @@ int CeedOperatorGetOrder(CeedOperator oper, CeedInt * order)
   int ierr;

   CeedOperatorField active_field;
-   ierr = CeedOperatorGetActiveField(oper, &active_field); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveField(oper, &active_field); PCeedChk(ierr);
   CeedBasis basis;
-   ierr = CeedOperatorFieldGetBasis(active_field, &basis); CeedChk(ierr);
+   ierr = CeedOperatorFieldGetBasis(active_field, &basis); PCeedChk(ierr);
   int P1d;
-   ierr = CeedBasisGetNumNodes1D(basis, &P1d); CeedChk(ierr);
+   ierr = CeedBasisGetNumNodes1D(basis, &P1d); PCeedChk(ierr);
   *order = P1d - 1;

   return 0;
@@ -753,13 +754,13 @@ int CeedATPMGBundle(CeedOperator oper, int order_reduction,
 {
   int ierr;
   CeedInt order;
-   ierr = CeedOperatorGetOrder(oper, &order); CeedChk(ierr);
+   ierr = CeedOperatorGetOrder(oper, &order); PCeedChk(ierr);
   CeedElemRestriction ho_er;
-   ierr = CeedOperatorGetActiveElemRestriction(oper, &ho_er); CeedChk(ierr);
+   ierr = CeedOperatorGetActiveElemRestriction(oper, &ho_er); PCeedChk(ierr);
   ierr = CeedATPMGElemRestriction(order, order_reduction, ho_er, er_out, dof_map);
-   CeedChk(ierr);
+   PCeedChk(ierr);
   ierr = CeedATPMGOperator(oper, order_reduction, *er_out, coarse_basis_out,
-                            basis_ctof_out, coarse_oper); CeedChk(ierr);
+                            basis_ctof_out, coarse_oper); PCeedChk(ierr);
   return 0;
 }

@@ -220,12 +220,12 @@ double TransformedCoefficient::Eval(ElementTransformation &T,
 {
   if (Q2)
   {
-      return (*Transform2)(Q1->Eval(T, ip, GetTime()),
-                           Q2->Eval(T, ip, GetTime()));
+      return Transform2(Q1->Eval(T, ip, GetTime()),
+                        Q2->Eval(T, ip, GetTime()));
   }
   else
   {
-      return (*Transform1)(Q1->Eval(T, ip, GetTime()));
+      return Transform1(Q1->Eval(T, ip, GetTime()));
   }
 }

@@ -1591,14 +1591,21 @@ void VectorQuadratureFunctionCoefficient::Eval(Vector &V,
 {
   QuadF.HostRead();

+   const int el_idx = QuadF.GetSpace()->GetEntityIndex(T);
+   // Handle the case of "interior boundary elements" and FaceQuadratureSpace
+   // with FaceType::Boundary.
+   if (el_idx < 0) { V = 0.0; return; }
+
+   const int ip_idx = QuadF.GetSpace()->GetPermutedIndex(el_idx, ip.index);
+
   if (index == 0 && vdim == QuadF.GetVDim())
   {
-      QuadF.GetValues(T.ElementNo, ip.index, V);
+      QuadF.GetValues(el_idx, ip_idx, V);
   }
   else
   {
      Vector temp;
-      QuadF.GetValues(T.ElementNo, ip.index, temp);
+      QuadF.GetValues(el_idx, ip_idx, temp);
      V.SetSize(vdim);
      for (int i = 0; i < vdim; i++)
      {
@@ -1625,7 +1632,12 @@ double QuadratureFunctionCoefficient::Eval(ElementTransformation &T,
 {
   QuadF.HostRead();
   Vector temp(1);
-   QuadF.GetValues(T.ElementNo, ip.index, temp);
+   const int el_idx = QuadF.GetSpace()->GetEntityIndex(T);
+   // Handle the case of "interior boundary elements" and FaceQuadratureSpace
+   // with FaceType::Boundary.
+   if (el_idx < 0) { return 0.0; }
+   const int ip_idx = QuadF.GetSpace()->GetPermutedIndex(el_idx, ip.index);
+   QuadF.GetValues(el_idx, ip_idx, temp);
   return temp[0];
 }

@@ -422,15 +422,15 @@ class TransformedCoefficient : public Coefficient
 private:
   Coefficient * Q1;
   Coefficient * Q2;
-   double (*Transform1)(double);
-   double (*Transform2)(double,double);
+   std::function<double(double)> Transform1;
+   std::function<double(double, double)> Transform2;

 public:
-   TransformedCoefficient (Coefficient * q,double (*F)(double))
-      : Q1(q), Transform1(F) { Q2 = 0; Transform2 = 0; }
+   TransformedCoefficient (Coefficient * q, std::function<double(double)> F)
+      : Q1(q), Transform1(std::move(F)) { Q2 = 0; Transform2 = 0; }
   TransformedCoefficient (Coefficient * q1,Coefficient * q2,
-                           double (*F)(double,double))
-      : Q1(q1), Q2(q2), Transform2(F) { Transform1 = 0; }
+                           std::function<double(double, double)> F)
+      : Q1(q1), Q2(q2), Transform2(std::move(F)) { Transform1 = 0; }

   /// Set the time for internally stored coefficients
   void SetTime(double t);
@@ -166,21 +166,19 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
      b = b_.Read();
   }

-   constexpr int NB = Q1D ? Q1D : 1; // block size
+   static constexpr int NB = Q1D ? Q1D : 1; // block size

   mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
   {
-      constexpr int NB = Q1D ? Q1D : 1; // redefine here for some compilers
-
      // Perform change of basis if needed
      if (CHANGE_BASIS)
      {
         // Transform RHS
-         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
+         DGMassBasis<DIM,D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
         if (IT_MODE)
         {
            // Transform initial guess
-            DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, d2q_B, u, u, d1d);
+            DGMassBasis<DIM,D1D>(e, NE, d2q_B, u, u, d1d);
         }
      }

@@ -257,7 +255,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const

      if (CHANGE_BASIS)
      {
-         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_B, u, u, d1d);
+         DGMassBasis<DIM,D1D>(e, NE, q2d_B, u, u, d1d);
      }
   });
 }
@@ -172,7 +172,7 @@ double DGMassDot(const int e,
   return s_dot[0];
 }

-template<int T_D1D = 0, int MAX_D1D = 0>
+template<int T_D1D = 0>
 MFEM_HOST_DEVICE inline
 void DGMassBasis2D(const int e,
                   const int NE,
@@ -181,7 +181,7 @@ void DGMassBasis2D(const int e,
                   double *y_,
                   const int d1d = 0)
 {
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
   const int D1D = T_D1D ? T_D1D : d1d;

   const auto b = Reshape(b_, D1D, D1D);
@@ -213,7 +213,7 @@ void DGMassBasis2D(const int e,
   MFEM_SYNC_THREAD;
 }

-template<int T_D1D = 0, int MAX_D1D = 0>
+template<int T_D1D = 0>
 MFEM_HOST_DEVICE inline
 void DGMassBasis3D(const int e,
                   const int NE,
@@ -228,7 +228,7 @@ void DGMassBasis3D(const int e,
   const auto x = Reshape(x_, D1D, D1D, D1D, NE);
   auto y = Reshape(y_, D1D, D1D, D1D, NE);

-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;

   MFEM_SHARED double sB[MD1*MD1];
   MFEM_SHARED double sm0[MD1*MD1*MD1];
@@ -260,7 +260,7 @@ void DGMassBasis3D(const int e,
   MFEM_SYNC_THREAD;
 }

-template<int DIM, int T_D1D = 0, int MAX_D1D = 0>
+template<int DIM, int T_D1D = 0>
 MFEM_HOST_DEVICE inline
 void DGMassBasis(const int e,
                 const int NE,
@@ -271,11 +271,11 @@ void DGMassBasis(const int e,
 {
   if (DIM == 2)
   {
-      DGMassBasis2D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
+      DGMassBasis2D<T_D1D>(e, NE, b_, x_, y_, d1d);
   }
   else if (DIM == 3)
   {
-      DGMassBasis3D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
+      DGMassBasis3D<T_D1D>(e, NE, b_, x_, y_, d1d);
   }
   else
   {
@@ -14,175 +14,166 @@
 namespace mfem
 {

+void DofTransformation::TransformPrimal(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->TransformPrimal(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->TransformPrimal(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::InvTransformPrimal(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Height();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->InvTransformPrimal(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->InvTransformPrimal(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::TransformDual(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->TransformDual(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->TransformDual(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
+void DofTransformation::InvTransformDual(double *v) const
+{
+   MFEM_ASSERT(dof_trans_,
+               "DofTransformation has no local transformation, call "
+               "SetDofTransformation first!");
+   int size = dof_trans_->Size();
+
+   if (vdim_ == 1 || (Ordering::Type)ordering_ == Ordering::byNODES)
+   {
+      for (int i=0; i<vdim_; i++)
+      {
+         dof_trans_->InvTransformDual(Fo_, &v[i*size]);
+      }
+   }
+   else
+   {
+      Vector vec(size);
+      for (int i=0; i<vdim_; i++)
+      {
+         for (int j=0; j<size; j++)
+         {
+            vec(j) = v[j*vdim_+i];
+         }
+         dof_trans_->InvTransformDual(Fo_, vec);
+         for (int j=0; j<size; j++)
+         {
+            v[j*vdim_+i] = vec(j);
+         }
+      }
+   }
+}
+
 void TransformPrimal(const DofTransformation *ran_dof_trans,
                     const DofTransformation *dom_dof_trans,
                     DenseMatrix &elmat)
 {
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformPrimalCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
+   // No action if both transformations are NULL
+   if (ran_dof_trans)
   {
      ran_dof_trans->TransformPrimalCols(elmat);
   }
-   else if (dom_dof_trans)
+   if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
 }

 void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat)
 {
-   if (ran_dof_trans && dom_dof_trans)
-   {
-      ran_dof_trans->TransformDualCols(elmat);
-      dom_dof_trans->TransformDualRows(elmat);
-   }
-   else if (ran_dof_trans)
+   // No action if both transformations are NULL
+   if (ran_dof_trans)
   {
      ran_dof_trans->TransformDualCols(elmat);
   }
-   else if (dom_dof_trans)
+   if (dom_dof_trans)
   {
      dom_dof_trans->TransformDualRows(elmat);
   }
-   else
-   {
-      // If both transformations are NULL this function should not be called
-   }
-}
-
-void StatelessVDofTransformation::TransformPrimal(const Array<int> & face_ori,
-                                                  double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES || vdim_ == 1)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->TransformPrimal(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->TransformPrimal(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::InvTransformPrimal(
-   const Array<int> & face_ori,
-   double *v) const
-{
-   int size = sdoftrans_->Height();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->InvTransformPrimal(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->InvTransformPrimal(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::TransformDual(const Array<int> & face_ori,
-                                                double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->TransformDual(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->TransformDual(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
-}
-
-void StatelessVDofTransformation::InvTransformDual(const Array<int> & face_ori,
-                                                   double *v) const
-{
-   int size = sdoftrans_->Size();
-
-   if ((Ordering::Type)ordering_ == Ordering::byNODES)
-   {
-      for (int i=0; i<vdim_; i++)
-      {
-         sdoftrans_->InvTransformDual(face_ori, &v[i*size]);
-      }
-   }
-   else
-   {
-      Vector vec(size);
-      for (int i=0; i<vdim_; i++)
-      {
-         for (int j=0; j<size; j++)
-         {
-            vec(j) = v[j*vdim_+i];
-         }
-         sdoftrans_->InvTransformDual(face_ori, vec);
-         for (int j=0; j<size; j++)
-         {
-            v[j*vdim_+i] = vec(j);
-         }
-      }
-   }
 }

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_StatelessDofTransformation::T_data[24] =
+const double ND_DofTransformation::T_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -192,11 +183,11 @@ const double ND_StatelessDofTransformation::T_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_StatelessDofTransformation
-::T(const_cast<double*>(ND_StatelessDofTransformation::T_data), 2, 2, 6);
+const DenseTensor ND_DofTransformation
+::T(const_cast<double *>(ND_DofTransformation::T_data), 2, 2, 6);

 // ordering (i0j0, i1j0, i0j1, i1j1), each row is a column major matrix
-const double ND_StatelessDofTransformation::TInv_data[24] =
+const double ND_DofTransformation::TInv_data[24] =
 {
   1.0,  0.0,  0.0,  1.0,
   -1.0, -1.0,  0.0,  1.0,
@@ -206,12 +197,11 @@ const double ND_StatelessDofTransformation::TInv_data[24] =
   0.0,  1.0,  1.0,  0.0
 };

-const DenseTensor ND_StatelessDofTransformation
-::TInv(const_cast<double*>(TInv_data), 2, 2, 6);
+const DenseTensor ND_DofTransformation
+::TInv(const_cast<double *>(TInv_data), 2, 2, 6);

-ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
-                                                             int num_edges,
-                                                             int num_tri_faces)
+ND_DofTransformation::ND_DofTransformation(int size, int p, int num_edges,
+                                           int num_tri_faces)
   : StatelessDofTransformation(size)
   , order(p)
   , nedofs(p)
@@ -221,18 +211,19 @@ ND_StatelessDofTransformation::ND_StatelessDofTransformation(int size, int p,
 {
 }

-void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
-                                                    double *v) const
+void ND_DofTransformation::TransformPrimal(const Array<int> & Fo,
+                                           double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -240,23 +231,25 @@ void ND_StatelessDofTransformation::TransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
+         T2.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
-                                                       double *v) const
+void ND_DofTransformation::InvTransformPrimal(const Array<int> & Fo,
+                                              double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -264,23 +257,24 @@ void ND_StatelessDofTransformation::InvTransformPrimal(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
+         T2Inv.Mult(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
-                                                  double *v) const
+void ND_DofTransformation::TransformDual(const Array<int> & Fo, double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2Inv;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -288,23 +282,25 @@ void ND_StatelessDofTransformation::TransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         TInv(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2Inv.UseExternalData(const_cast<double *>(TInv.GetData(Fo[f])), 2, 2);
+         T2Inv.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }

-void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
-                                                     double *v) const
+void ND_DofTransformation::InvTransformDual(const Array<int> & Fo,
+                                            double *v) const
 {
   // Return immediately when no face DoFs are present
-   if (nfdofs < 2) { return; }
+   if (IsIdentity()) { return; }

   MFEM_VERIFY(Fo.Size() >= nfaces,
               "Face orientation array is shorter than the number of faces in "
-               "ND_StatelessDofTransformation");
+               "ND_DofTransformation");

   double data[2];
   Vector v2(data, 2);
+   DenseMatrix T2;

   // Transform face DoFs
   for (int f=0; f<nfaces; f++)
@@ -312,7 +308,8 @@ void ND_StatelessDofTransformation::InvTransformDual(const Array<int> & Fo,
      for (int i=0; i<nfdofs/2; i++)
      {
         v2 = &v[nedges*nedofs + f*nfdofs + 2*i];
-         T(Fo[f]).MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
+         T2.UseExternalData(const_cast<double *>(T.GetData(Fo[f])), 2, 2);
+         T2.MultTranspose(v2, &v[nedges*nedofs + f*nfdofs + 2*i]);
      }
   }
 }
@@ -80,6 +80,9 @@ public:
   inline int Width() const { return size_; }
   inline int NumCols() const { return size_; }

+   /// If the DofTransformation performs no transformation
+   virtual bool IsIdentity() const = 0;
+
   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
@@ -115,6 +118,8 @@ public:
   inline void InvTransformDual(const Array<int> & face_orientation,
                                Vector &v) const
   { InvTransformDual(face_orientation, v.GetData()); }
+
+   virtual ~StatelessDofTransformation() = default;
 };

 /** The DofTransformation class is an extension of the
@@ -125,7 +130,7 @@ public:
    DofTransformation objects are provided by the FiniteElementSpace which has
    access to the mesh and can therefore provide the face orientations. This is
    convenient when working with GridFunction, LinearForm, or BilinearForm
-    obejcts or their parallel counterparts.
+    objects or their parallel counterparts.

    StatelessDofTransformation objects are provided by FiniteElement or
    FiniteElementCollection objects which do not have access to face
@@ -133,35 +138,76 @@ public:
    transferring finite element degrees of freedom between different meshes.
    For examples of its use see the TransferMap used by the SubMesh class.
   */
-class DofTransformation : virtual public StatelessDofTransformation
+class DofTransformation
 {
 protected:
-   Array<int> Fo;
-
-   DofTransformation(int size)
-      : StatelessDofTransformation(size) {}
+   Array<int> Fo_;
+   const StatelessDofTransformation * dof_trans_;
+   int vdim_;
+   int ordering_;

 public:
+   /** @brief Default constructor which requires that SetDofTransformation be
+       called before use. */
+   DofTransformation(int vdim = 1, int ordering = 0)
+      : dof_trans_(NULL)
+      , vdim_(vdim)
+      , ordering_(ordering)
+   {}
+
+   /// Constructor with a known StatelessDofTransformation
+   DofTransformation(const StatelessDofTransformation & dof_trans,
+                     int vdim = 1, int ordering = 0)
+      : dof_trans_(&dof_trans)
+      , vdim_(vdim)
+      , ordering_(ordering)
+   {}

   /** @brief Configure the transformation using face orientations for the
       current element. */
   /// The face_orientation array can be obtained from Mesh::GetElementFaces.
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   { Fo = face_orientation; }
+   inline void SetFaceOrientations(const Array<int> & Fo)
+   { Fo_ = Fo; }

-   inline const Array<int> & GetFaceOrientations() const { return Fo; }
+   /// Return the face orientations for the current element
+   inline const Array<int> & GetFaceOrientations() const { return Fo_; }

-   using StatelessDofTransformation::TransformPrimal;
-   using StatelessDofTransformation::InvTransformPrimal;
-   using StatelessDofTransformation::TransformDual;
-   using StatelessDofTransformation::InvTransformDual;
+   /// Set or change the nested StatelessDofTransformation object
+   inline void SetDofTransformation(const StatelessDofTransformation & dof_trans)
+   {
+      dof_trans_ = &dof_trans;
+   }
+   inline void SetDofTransformation(const StatelessDofTransformation * dof_trans)
+   {
+      dof_trans_ = dof_trans;
+   }
+
+   /// Return the nested StatelessDofTransformation object
+   inline const StatelessDofTransformation * GetDofTransformation() const
+   { return dof_trans_; }
+
+   /// Set or change the vdim and ordering parameter
+   inline void SetVDim(int vdim = 1, int ordering = 0)
+   {
+      vdim_ = vdim;
+      ordering_ = ordering;
+   }
+
+   /// Return the current vdim value
+   inline int GetVDim() const { return vdim_; }
+
+   inline int Size() const { return dof_trans_->Size(); }
+   inline int Height() const { return dof_trans_->Height(); }
+   inline int NumRows() const { return dof_trans_->NumRows(); }
+   inline int Width() const { return dof_trans_->Width(); }
+   inline int NumCols() const { return dof_trans_->NumCols(); }
+   inline bool IsIdentity() const { return dof_trans_->IsIdentity(); }

   /** Transform local DoFs to align with the global DoFs. For example, this
       transformation can be used to map the local vector computed by
       FiniteElement::Project() to the transformed vector stored within a
       GridFunction object. */
-   inline void TransformPrimal(double *v) const
-   { TransformPrimal(Fo, v); }
+   void TransformPrimal(double *v) const;
   inline void TransformPrimal(Vector &v) const
   { TransformPrimal(v.GetData()); }

@@ -179,21 +225,18 @@ public:
       transform the vector obtained using GridFunction::GetSubVector before it
       can be used to compute a local interpolation.
   */
-   inline void InvTransformPrimal(double *v) const
-   { InvTransformPrimal(Fo, v); }
+   void InvTransformPrimal(double *v) const;
   inline void InvTransformPrimal(Vector &v) const
   { InvTransformPrimal(v.GetData()); }

   /** Transform dual DoFs as computed by a LinearFormIntegrator before summing
       into a LinearForm object. */
-   inline void TransformDual(double *v) const
-   { TransformDual(Fo, v); }
+   void TransformDual(double *v) const;
   inline void TransformDual(Vector &v) const
   { TransformDual(v.GetData()); }

   /** Inverse Transform dual DoFs */
-   inline void InvTransformDual(double *v) const
-   { InvTransformDual(Fo, v); }
+   void InvTransformDual(double *v) const;
   inline void InvTransformDual(Vector &v) const
   { InvTransformDual(v.GetData()); }

@@ -225,8 +268,6 @@ public:
         TransformDual(V.GetColumn(c));
      }
   }
-
-   virtual ~DofTransformation() = default;
 };

 /** Transform a matrix of DoFs entries from different finite element spaces as
@@ -245,145 +286,6 @@ void TransformDual(const DofTransformation *ran_dof_trans,
                   const DofTransformation *dom_dof_trans,
                   DenseMatrix &elmat);

-/** The StatelessVDofTransformation class implements a nested transformation
-    where an arbitrary StatelessDofTransformation is replicated with a
-    vdim >= 1.
-*/
-class StatelessVDofTransformation : virtual public StatelessDofTransformation
-{
-protected:
-   int vdim_;
-   int ordering_;
-   StatelessDofTransformation * sdoftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   StatelessVDofTransformation(int vdim = 1, int ordering = 0)
-      : StatelessDofTransformation(0)
-      , vdim_(vdim)
-      , ordering_(ordering)
-      , sdoftrans_(NULL)
-   {}
-
-   /// Constructor with a known StatelessDofTransformation
-   StatelessVDofTransformation(StatelessDofTransformation & doftrans,
-                               int vdim = 1,
-                               int ordering = 0)
-      : StatelessDofTransformation(vdim * doftrans.Size())
-      , vdim_(vdim)
-      , ordering_(ordering)
-      , sdoftrans_(&doftrans)
-   {}
-
-   /// Set or change the vdim parameter
-   inline void SetVDim(int vdim)
-   {
-      vdim_ = vdim;
-      if (sdoftrans_)
-      {
-         size_ = vdim_ * sdoftrans_->Size();
-      }
-   }
-
-   /// Return the current vdim value
-   inline int GetVDim() const { return vdim_; }
-
-   /// Set or change the nested StatelessDofTransformation object
-   inline void SetDofTransformation(StatelessDofTransformation & doftrans)
-   {
-      size_ = vdim_ * doftrans.Size();
-      sdoftrans_ = &doftrans;
-   }
-
-   /// Return the nested StatelessDofTransformation object
-   inline StatelessDofTransformation * GetDofTransformation() const
-   { return sdoftrans_; }
-
-   using StatelessDofTransformation::TransformPrimal;
-   using StatelessDofTransformation::InvTransformPrimal;
-   using StatelessDofTransformation::TransformDual;
-   using StatelessDofTransformation::InvTransformDual;
-
-   /** Specializations of these base class methods which account for the vdim
-       and ordering of the full set of DoFs.
-   */
-   void TransformPrimal(const Array<int> & face_ori, double *v) const;
-   void InvTransformPrimal(const Array<int> & face_ori, double *v) const;
-   void TransformDual(const Array<int> & face_ori, double *v) const;
-   void InvTransformDual(const Array<int> & face_ori, double *v) const;
-};
-
-/** The VDofTransformation class implements a nested transformation where an
-    arbitrary DofTransformation is replicated with a vdim >= 1.
-*/
-class VDofTransformation : public StatelessVDofTransformation,
-   public DofTransformation
-{
-protected:
-   DofTransformation * doftrans_;
-
-public:
-   /** @brief Default constructor which requires that SetDofTransformation be
-       called before use. */
-   VDofTransformation(int vdim = 1, int ordering = 0)
-      : StatelessDofTransformation(0)
-      , StatelessVDofTransformation(vdim, ordering)
-      , DofTransformation(0)
-      , doftrans_(NULL)
-   {}
-
-   /// Constructor with a known DofTransformation
-   /// @note The face orientations in @a doftrans will be copied into the
-   /// new VDofTransformation object.
-   VDofTransformation(DofTransformation & doftrans, int vdim = 1,
-                      int ordering = 0)
-      : StatelessDofTransformation(vdim * doftrans.Size())
-      , StatelessVDofTransformation(doftrans, vdim, ordering)
-      , DofTransformation(vdim * doftrans.Size())
-      , doftrans_(&doftrans)
-   {
-      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
-   }
-
-   using StatelessVDofTransformation::SetDofTransformation;
-
-   /// Set or change the nested DofTransformation object
-   /// @note The face orientations in @a doftrans will be copied into the
-   /// VDofTransformation object.
-   void SetDofTransformation(DofTransformation & doftrans)
-   {
-      doftrans_ = &doftrans;
-      StatelessVDofTransformation::SetDofTransformation(doftrans);
-      DofTransformation::SetFaceOrientations(doftrans.GetFaceOrientations());
-   }
-
-   /// Return the nested DofTransformation object
-   inline DofTransformation * GetDofTransformation() const { return doftrans_; }
-
-   /// Set new face orientations in both the VDofTransformation and the
-   /// DofTransformation contained within (if there is one).
-   inline void SetFaceOrientations(const Array<int> & face_orientation)
-   {
-      DofTransformation::SetFaceOrientations(face_orientation);
-      if (doftrans_) { doftrans_->SetFaceOrientations(face_orientation); }
-   }
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   inline void TransformPrimal(double *v) const
-   { TransformPrimal(Fo, v); }
-   inline void InvTransformPrimal(double *v) const
-   { InvTransformPrimal(Fo, v); }
-   inline void TransformDual(double *v) const
-   { TransformDual(Fo, v); }
-   inline void InvTransformDual(double *v) const
-   { InvTransformDual(Fo, v); }
-};
-
 /** Abstract base class for high-order Nedelec spaces on elements with
    triangular faces.

@@ -396,7 +298,7 @@ public:
    be accessed as DenseMatrices using the GetFaceTransform() and
    GetFaceInverseTransform() methods.
 */
-class ND_StatelessDofTransformation : virtual public StatelessDofTransformation
+class ND_DofTransformation : public StatelessDofTransformation
 {
 private:
   static const double T_data[24];
@@ -410,8 +312,7 @@ protected:
   const int nedges; // number of edges per element
   const int nfaces; // number of triangular faces per element

-   ND_StatelessDofTransformation(int size, int order,
-                                 int num_edges, int num_tri_faces);
+   ND_DofTransformation(int size, int order, int num_edges, int num_tri_faces);

 public:
   // Return the 2x2 transformation operator for the given face orientation
@@ -421,116 +322,41 @@ public:
   static const DenseMatrix & GetFaceInverseTransform(int ori)
   { return TInv(ori); }

-   void TransformPrimal(const Array<int> & face_orientation,
-                        double *v) const;
+   bool IsIdentity() const override { return nfdofs < 2; }

-   void InvTransformPrimal(const Array<int> & face_orientation,
-                           double *v) const;
-
-   void TransformDual(const Array<int> & face_orientation,
-                      double *v) const;
-
-   void InvTransformDual(const Array<int> & face_orientation,
-                         double *v) const;
+   void TransformPrimal(const Array<int> & Fo, double *v) const override;
+   void InvTransformPrimal(const Array<int> & Fo, double *v) const override;
+   void TransformDual(const Array<int> & Fo, double *v) const override;
+   void InvTransformDual(const Array<int> & Fo, double *v) const override;
 };

 /// Stateless DoF transformation implementation for the Nedelec basis on
 /// triangles
-class ND_TriStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_TriStatelessDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2))
-      , ND_StatelessDofTransformation(order*(order + 2), order, 3, 1)
-   {}
-};
-
-/// DoF transformation implementation for the Nedelec basis on triangles
-class ND_TriDofTransformation : public DofTransformation,
-   public ND_TriStatelessDofTransformation
+class ND_TriDofTransformation : public ND_DofTransformation
 {
 public:
   ND_TriDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2))
-      , DofTransformation(order*(order + 2))
-      , ND_TriStatelessDofTransformation(order)
-   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_TriStatelessDofTransformation::TransformPrimal;
-   using ND_TriStatelessDofTransformation::InvTransformPrimal;
-   using ND_TriStatelessDofTransformation::TransformDual;
-   using ND_TriStatelessDofTransformation::InvTransformDual;
-};
-
-/// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_TetStatelessDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
-      , ND_StatelessDofTransformation(order*(order + 2)*(order + 3)/2, order,
-                                      6, 4)
+      : ND_DofTransformation(order*(order + 2), order, 3, 1)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on tetrahedra
-class ND_TetDofTransformation : public DofTransformation,
-   public ND_TetStatelessDofTransformation
+class ND_TetDofTransformation : public ND_DofTransformation
 {
 public:
   ND_TetDofTransformation(int order)
-      : StatelessDofTransformation(order*(order + 2)*(order + 3)/2)
-      , DofTransformation(order*(order + 2)*(order + 3)/2)
-      , ND_TetStatelessDofTransformation(order)
-   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_TetStatelessDofTransformation::TransformPrimal;
-   using ND_TetStatelessDofTransformation::InvTransformPrimal;
-   using ND_TetStatelessDofTransformation::TransformDual;
-   using ND_TetStatelessDofTransformation::InvTransformDual;
-};
-
-/// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeStatelessDofTransformation : public ND_StatelessDofTransformation
-{
-public:
-   ND_WedgeStatelessDofTransformation(int order)
-      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , ND_StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2,
-                                      order, 9, 2)
+      : ND_DofTransformation(order*(order + 2)*(order + 3)/2, order, 6, 4)
   {}
 };

 /// DoF transformation implementation for the Nedelec basis on wedge elements
-class ND_WedgeDofTransformation : public DofTransformation,
-   public ND_WedgeStatelessDofTransformation
+class ND_WedgeDofTransformation : public ND_DofTransformation
 {
 public:
   ND_WedgeDofTransformation(int order)
-      : StatelessDofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , DofTransformation(3 * order * ((order + 1) * (order + 2))/2)
-      , ND_WedgeStatelessDofTransformation(order)
+      : ND_DofTransformation(3 * order * ((order + 1) * (order + 2))/2,
+                             order, 9, 2)
   {}
-
-   using DofTransformation::TransformPrimal;
-   using DofTransformation::InvTransformPrimal;
-   using DofTransformation::TransformDual;
-   using DofTransformation::InvTransformDual;
-
-   using ND_WedgeStatelessDofTransformation::TransformPrimal;
-   using ND_WedgeStatelessDofTransformation::InvTransformPrimal;
-   using ND_WedgeStatelessDofTransformation::TransformDual;
-   using ND_WedgeStatelessDofTransformation::InvTransformDual;
 };

 } // namespace mfem
@@ -355,15 +355,11 @@ int InverseElementTransformation::Transform(const Vector &pt,
         }
         else
         {
-            const int old_type = GlobGeometryRefiner.GetType();
-            GlobGeometryRefiner.SetType(qpts_type);
-            RefinedGeometry &RefG =
-               *GlobGeometryRefiner.Refine(T->GetGeometryType(), order);
+            RefinedGeometry &RefG = *refiner.Refine(T->GetGeometryType(), order);
            int closest_idx = (init_guess_type == ClosestPhysNode) ?
                              FindClosestPhysPoint(pt, RefG.RefPts) :
                              FindClosestRefPoint(pt, RefG.RefPts);
            ip0 = &RefG.RefPts.IntPoint(closest_idx);
-            GlobGeometryRefiner.SetType(old_type);
         }
         break;
      }
@@ -492,6 +488,7 @@ int IsoparametricTransformation::OrderGrad(const FiniteElement *fe) const
 void IsoparametricTransformation::Transform (const IntegrationPoint &ip,
                                             Vector &trans)
 {
+   MFEM_ASSERT(FElem != nullptr, "Must provide a valid FiniteElement object!");
   shape.SetSize(FElem->GetDof());
   trans.SetSize(PointMat.Height());

@@ -233,7 +233,7 @@ protected:
   // Parameters of the inversion algorithms:
   const IntegrationPoint *ip0;
   int init_guess_type; // algorithm to use
-   int qpts_type; // Quadrature1D type for the initial guess type
+   GeometryRefiner refiner; // geometry refiner for initial guess
   int rel_qpts_order; // num_1D_qpts = max(trans_order+rel_qpts_order,0)+1
   int solver_type; // solution strategy to use
   int max_iter; // max. number of Newton iterations
@@ -276,7 +276,7 @@ public:
      : T(Trans),
        ip0(NULL),
        init_guess_type(Center),
-        qpts_type(Quadrature1D::OpenHalfUniform),
+        refiner(Quadrature1D::OpenHalfUniform),
        rel_qpts_order(-1),
        solver_type(NewtonElementProject),
        max_iter(16),
@@ -301,7 +301,7 @@ public:
   { ip0 = &init_ip; SetInitialGuessType(GivenPoint); }

   /// Set the Quadrature1D type used for the `Closest*` initial guess types.
-   void SetInitGuessPointsType(int q_type) { qpts_type = q_type; }
+   void SetInitGuessPointsType(int q_type) { refiner.SetType(q_type); }

   /// Set the relative order used for the `Closest*` initial guess types.
   /** The number of points in each spatial direction is given by the formula
@@ -361,7 +361,7 @@ public:
 class IsoparametricTransformation : public ElementTransformation
 {
 private:
-   DenseMatrix dshape,d2shape;
+   DenseMatrix dshape, d2shape;
   Vector shape;

   const FiniteElement *FElem;
@@ -74,11 +74,11 @@ public:
 /** @brief The ZienkiewiczZhuEstimator class implements the Zienkiewicz-Zhu
    error estimation procedure.

-    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    [1] Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
    and a posteriori error estimates. Part 1: The recovery technique.
    Int. J. Num. Meth. Engng. 33, 1331-1364 (1992).

-    Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
+    [2] Zienkiewicz, O.C. and Zhu, J.Z., The superconvergent patch recovery
    and a posteriori error estimates. Part 2: Error estimates and adaptivity.
    Int. J. Num. Meth. Engng. 33, 1365-1382 (1992).

@@ -220,8 +220,8 @@ public:
    The required BilinearFormIntegrator must implement the method
    ComputeElementFlux().

-   COMMENTS:
-   *  The present implementation ignores all single-element patches corresponding
+   @note
+   -  The present implementation ignores all single-element patches corresponding
      to boundary faces. This is appropriate for Dirichlet boundaries, but
      suboptimal for Neumann boundaries. Reference 3 shows that a constrained
      least-squares problem, where the reconstructed flux is constrained by the
@@ -229,13 +229,13 @@ public:
      NOTE THAT THIS CONSTRAINED LS PROBLEM IS NOT YET IMPLEMENTED, so it is
      possible that the local error estimates for elements on a Neumann boundary
      are suboptimal.
-   *  The global polynomial basis used for the flux reconstruction is, by default,
+   -  The global polynomial basis used for the flux reconstruction is, by default,
      aligned with the physical Cartesian axis. For patches with 2D elements, this
      has been improved on so that the basis is aligned with the physical patch
      orientation. Reorientation of the flux reconstruction basis is helpful to
      maintain symmetry in the refinement pattern and could be extended to 3D.
-   *  This estimator is ONLY implemented IN SERIAL.
-   *  Anisotropic refinement is NOT YET SUPPORTED.
+   -  This estimator is ONLY implemented IN SERIAL.
+   -  Anisotropic refinement is NOT YET SUPPORTED.

 */
 class LSZienkiewiczZhuEstimator : public ErrorEstimator
@@ -359,135 +359,148 @@ void  FiniteElement::CalcPhysHessian(ElementTransformation &Trans,

   // Hessian in physical coords
   lhm.Invert();
-   Mult( hess, lhm, Hessian);
+   Mult(hess, lhm, Hessian);
 }

 const DofToQuad &FiniteElement::GetDofToQuad(const IntegrationRule &ir,
                                             DofToQuad::Mode mode) const
 {
+   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::FULL, "invalid mode requested");

-   for (int i = 0; i < dof2quad_array.Size(); i++)
-   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-#ifdef MFEM_THREAD_SAFE
-   DenseMatrix vshape(dof, dim);
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
 #endif
-
-   DofToQuad *d2q = new DofToQuad;
-   const int nqpt = ir.GetNPoints();
-   d2q->FE = this;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = dof;
-   d2q->nqpt = nqpt;
-   if (range_type == SCALAR)
   {
-      d2q->B.SetSize(nqpt*dof);
-      d2q->Bt.SetSize(dof*nqpt);
-
-      Vector shape;
-      vshape.GetColumnReference(0, shape);
-      for (int i = 0; i < nqpt; i++)
+      for (int i = 0; i < dof2quad_array.Size(); i++)
      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcShape(ip, shape);
-         for (int j = 0; j < dof; j++)
-         {
-            d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
-         }
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
      }
-   }
-   else if (range_type == VECTOR)
-   {
-      d2q->B.SetSize(nqpt*dim*dof);
-      d2q->Bt.SetSize(dof*nqpt*dim);
-
-      for (int i = 0; i < nqpt; i++)
+      if (!d2q)
      {
-         const IntegrationPoint &ip = ir.IntPoint(i);
-         CalcVShape(ip, vshape);
-         for (int d = 0; d < dim; d++)
+#ifdef MFEM_THREAD_SAFE
+         DenseMatrix vshape(dof, dim);
+#endif
+         d2q = new DofToQuad;
+         const int nqpt = ir.GetNPoints();
+         d2q->FE = this;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = dof;
+         d2q->nqpt = nqpt;
+         switch (range_type)
         {
-            for (int j = 0; j < dof; j++)
+            case SCALAR:
            {
-               d2q->B[i+nqpt*(d+dim*j)] = d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
-            }
-         }
-      }
-   }
-   else
-   {
-      // Skip B and Bt for unknown range type
-   }
-   switch (deriv_type)
-   {
-      case GRAD:
-      {
-         d2q->G.SetSize(nqpt*dim*dof);
-         d2q->Gt.SetSize(dof*nqpt*dim);
+               d2q->B.SetSize(nqpt*dof);
+               d2q->Bt.SetSize(dof*nqpt);

-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDShape(ip, vshape);
-            for (int d = 0; d < dim; d++)
-            {
-               for (int j = 0; j < dof; j++)
+               Vector shape;
+               vshape.GetColumnReference(0, shape);
+               for (int i = 0; i < nqpt; i++)
               {
-                  d2q->G[i+nqpt*(d+dim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcShape(ip, shape);
+                  for (int j = 0; j < dof; j++)
+                  {
+                     d2q->B[i+nqpt*j] = d2q->Bt[j+dof*i] = shape(j);
+                  }
               }
+               break;
            }
-         }
-         break;
-      }
-      case DIV:
-      {
-         d2q->G.SetSize(nqpt*dof);
-         d2q->Gt.SetSize(dof*nqpt);
-
-         Vector divshape;
-         vshape.GetColumnReference(0, divshape);
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcDivShape(ip, divshape);
-            for (int j = 0; j < dof; j++)
+            case VECTOR:
            {
-               d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
-            }
-         }
-         break;
-      }
-      case CURL:
-      {
-         d2q->G.SetSize(nqpt*cdim*dof);
-         d2q->Gt.SetSize(dof*nqpt*cdim);
+               d2q->B.SetSize(nqpt*dim*dof);
+               d2q->Bt.SetSize(dof*nqpt*dim);

-         DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
-         for (int i = 0; i < nqpt; i++)
-         {
-            const IntegrationPoint &ip = ir.IntPoint(i);
-            CalcCurlShape(ip, curlshape);
-            for (int d = 0; d < cdim; d++)
-            {
-               for (int j = 0; j < dof; j++)
+               for (int i = 0; i < nqpt; i++)
               {
-                  d2q->G[i+nqpt*(d+cdim*j)] = d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcVShape(ip, vshape);
+                  for (int d = 0; d < dim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->B[i+nqpt*(d+dim*j)] =
+                           d2q->Bt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                     }
+                  }
               }
+               break;
            }
+            case UNKNOWN_RANGE_TYPE:
+               // Skip B and Bt for unknown range type
+               break;
         }
-         break;
+         switch (deriv_type)
+         {
+            case GRAD:
+            {
+               d2q->G.SetSize(nqpt*dim*dof);
+               d2q->Gt.SetSize(dof*nqpt*dim);
+
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcDShape(ip, vshape);
+                  for (int d = 0; d < dim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->G[i+nqpt*(d+dim*j)] =
+                           d2q->Gt[j+dof*(i+nqpt*d)] = vshape(j, d);
+                     }
+                  }
+               }
+               break;
+            }
+            case DIV:
+            {
+               d2q->G.SetSize(nqpt*dof);
+               d2q->Gt.SetSize(dof*nqpt);
+
+               Vector divshape;
+               vshape.GetColumnReference(0, divshape);
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcDivShape(ip, divshape);
+                  for (int j = 0; j < dof; j++)
+                  {
+                     d2q->G[i+nqpt*j] = d2q->Gt[j+dof*i] = divshape(j);
+                  }
+               }
+               break;
+            }
+            case CURL:
+            {
+               d2q->G.SetSize(nqpt*cdim*dof);
+               d2q->Gt.SetSize(dof*nqpt*cdim);
+
+               DenseMatrix curlshape(vshape.GetData(), dof, cdim);  // cdim <= dim
+               for (int i = 0; i < nqpt; i++)
+               {
+                  const IntegrationPoint &ip = ir.IntPoint(i);
+                  CalcCurlShape(ip, curlshape);
+                  for (int d = 0; d < cdim; d++)
+                  {
+                     for (int j = 0; j < dof; j++)
+                     {
+                        d2q->G[i+nqpt*(d+cdim*j)] =
+                           d2q->Gt[j+dof*(i+nqpt*d)] = curlshape(j, d);
+                     }
+                  }
+               }
+               break;
+            }
+            case NONE:
+               // Skip G and Gt for unknown derivative type
+               break;
+         }
+         dof2quad_array.Append(d2q);
      }
-      case NONE:
-      default:
-         // Skip G and Gt for unknown derivative type
-         break;
   }
-   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -807,7 +820,7 @@ void NodalFiniteElement::Project(
   else
   {
      DenseMatrix vshape(fe.GetDof(), std::max(Trans.GetSpaceDim(),
-                                               fe.GetVDim()));
+                                               fe.GetRangeDim()));

      I.SetSize(vshape.Width()*dof, fe.GetDof());
      for (int k = 0; k < dof; k++)
@@ -904,14 +917,14 @@ VectorFiniteElement::VectorFiniteElement(int D, Geometry::Type G,
 }

 void VectorFiniteElement::CalcShape(
-   const IntegrationPoint &ip, Vector &shape ) const
+   const IntegrationPoint &ip, Vector &shape) const
 {
   mfem_error("Error: Cannot use scalar CalcShape(...) function with\n"
              "   VectorFiniteElements!");
 }

 void VectorFiniteElement::CalcDShape(
-   const IntegrationPoint &ip, DenseMatrix &dshape ) const
+   const IntegrationPoint &ip, DenseMatrix &dshape) const
 {
   mfem_error("Error: Cannot use scalar CalcDShape(...) function with\n"
              "   VectorFiniteElements!");
@@ -2183,51 +2196,72 @@ void Poly_1D::CalcChebyshev(const int p, const double x, double *u, double *d,

 const double *Poly_1D::GetPoints(const int p, const int btype)
 {
+   Array<double*> *pts;
   BasisType::Check(btype);
   const int qtype = BasisType::GetQuadrature1D(btype);
-
   if (qtype == Quadrature1D::Invalid) { return NULL; }

-   if (points_container.find(btype) == points_container.end())
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (Poly1DGetPoints)
+#endif
   {
-      points_container[btype] = new Array<double*>(h_mt);
+      auto it = points_container.find(btype);
+      if (it != points_container.end())
+      {
+         pts = it->second;
+      }
+      else
+      {
+         pts = new Array<double*>(h_mt);
+         points_container[btype] = pts;
+      }
+      if (pts->Size() <= p)
+      {
+         pts->SetSize(p + 1, NULL);
+      }
+      if ((*pts)[p] == NULL)
+      {
+         (*pts)[p] = new double[p + 1];
+         quad_func.GivePolyPoints(p + 1, (*pts)[p], qtype);
+      }
   }
-   Array<double*> &pts = *points_container[btype];
-   if (pts.Size() <= p)
-   {
-      pts.SetSize(p + 1, NULL);
-   }
-   if (pts[p] == NULL)
-   {
-      pts[p] = new double[p + 1];
-      quad_func.GivePolyPoints(p+1, pts[p], qtype);
-   }
-   return pts[p];
+   return (*pts)[p];
 }

 Poly_1D::Basis &Poly_1D::GetBasis(const int p, const int btype)
 {
+   Array<Basis*> *bases;
   BasisType::Check(btype);

-   if ( bases_container.find(btype) == bases_container.end() )
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (Poly1DGetBasis)
+#endif
   {
-      // we haven't been asked for basis or points of this type yet
-      bases_container[btype] = new Array<Basis*>(h_mt);
+      auto it = bases_container.find(btype);
+      if (it != bases_container.end())
+      {
+         bases = it->second;
+      }
+      else
+      {
+         // we haven't been asked for basis or points of this type yet
+         bases = new Array<Basis*>(h_mt);
+         bases_container[btype] = bases;
+      }
+      if (bases->Size() <= p)
+      {
+         bases->SetSize(p + 1, NULL);
+      }
+      if ((*bases)[p] == NULL)
+      {
+         EvalType etype;
+         if (btype == BasisType::Positive) { etype = Positive; }
+         else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
+         else { etype = Barycentric; }
+         (*bases)[p] = new Basis(p, GetPoints(p, btype), etype);
+      }
   }
-   Array<Basis*> &bases = *bases_container[btype];
-   if (bases.Size() <= p)
-   {
-      bases.SetSize(p + 1, NULL);
-   }
-   if (bases[p] == NULL)
-   {
-      EvalType etype;
-      if (btype == BasisType::Positive) { etype = Positive; }
-      else if (btype == BasisType::IntegratedGLL) { etype = Integrated; }
-      else { etype = Barycentric; }
-      bases[p] = new Basis(p, GetPoints(p, btype), etype);
-   }
-   return *bases[p];
+   return *(*bases)[p];
 }

 Poly_1D::~Poly_1D()
@@ -2236,7 +2270,7 @@ Poly_1D::~Poly_1D()
        it != points_container.end() ; ++it)
   {
      Array<double*>& pts = *it->second;
-      for ( int i = 0 ; i < pts.Size() ; ++i )
+      for (int i = 0; i < pts.Size(); ++i)
      {
         delete [] pts[i];
      }
@@ -2247,7 +2281,7 @@ Poly_1D::~Poly_1D()
        it != bases_container.end() ; ++it)
   {
      Array<Basis*>& bases = *it->second;
-      for ( int i = 0 ; i < bases.Size() ; ++i )
+      for (int i = 0; i < bases.Size(); ++i)
      {
         delete bases[i];
      }
@@ -2461,39 +2495,47 @@ const DofToQuad &TensorBasisElement::GetTensorDofToQuad(
   DofToQuad::Mode mode, const Poly_1D::Basis &basis, bool closed,
   Array<DofToQuad*> &dof2quad_array)
 {
+   DofToQuad *d2q = nullptr;
   MFEM_VERIFY(mode == DofToQuad::TENSOR, "invalid mode requested");

-   for (int i = 0; i < dof2quad_array.Size(); i++)
+#if defined(MFEM_THREAD_SAFE) && defined(MFEM_USE_OPENMP)
+   #pragma omp critical (DofToQuad)
+#endif
   {
-      const DofToQuad &d2q = *dof2quad_array[i];
-      if (d2q.IntRule == &ir && d2q.mode == mode) { return d2q; }
-   }
-
-   DofToQuad *d2q = new DofToQuad;
-   const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
-   const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
-   d2q->FE = &fe;
-   d2q->IntRule = &ir;
-   d2q->mode = mode;
-   d2q->ndof = ndof;
-   d2q->nqpt = nqpt;
-   d2q->B.SetSize(nqpt*ndof);
-   d2q->Bt.SetSize(ndof*nqpt);
-   d2q->G.SetSize(nqpt*ndof);
-   d2q->Gt.SetSize(ndof*nqpt);
-   Vector val(ndof), grad(ndof);
-   for (int i = 0; i < nqpt; i++)
-   {
-      // The first 'nqpt' points in 'ir' have the same x-coordinates as those
-      // of the 1D rule.
-      basis.Eval(ir.IntPoint(i).x, val, grad);
-      for (int j = 0; j < ndof; j++)
+      for (int i = 0; i < dof2quad_array.Size(); i++)
      {
-         d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
-         d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+         d2q = dof2quad_array[i];
+         if (d2q->IntRule != &ir || d2q->mode != mode) { d2q = nullptr; }
+      }
+      if (!d2q)
+      {
+         d2q = new DofToQuad;
+         const int ndof = closed ? fe.GetOrder() + 1 : fe.GetOrder();
+         const int nqpt = (int)floor(pow(ir.GetNPoints(), 1.0/fe.GetDim()) + 0.5);
+         d2q->FE = &fe;
+         d2q->IntRule = &ir;
+         d2q->mode = mode;
+         d2q->ndof = ndof;
+         d2q->nqpt = nqpt;
+         d2q->B.SetSize(nqpt*ndof);
+         d2q->Bt.SetSize(ndof*nqpt);
+         d2q->G.SetSize(nqpt*ndof);
+         d2q->Gt.SetSize(ndof*nqpt);
+         Vector val(ndof), grad(ndof);
+         for (int i = 0; i < nqpt; i++)
+         {
+            // The first 'nqpt' points in 'ir' have the same x-coordinates as those
+            // of the 1D rule.
+            basis.Eval(ir.IntPoint(i).x, val, grad);
+            for (int j = 0; j < ndof; j++)
+            {
+               d2q->B[i+nqpt*j] = d2q->Bt[j+ndof*i] = val(j);
+               d2q->G[i+nqpt*j] = d2q->Gt[j+ndof*i] = grad(j);
+            }
+         }
+         dof2quad_array.Append(d2q);
      }
   }
-   dof2quad_array.Append(d2q);
   return *d2q;
 }

@@ -250,7 +250,7 @@ protected:
   /// Container for all DofToQuad objects created by the FiniteElement.
   /** Multiple DofToQuad objects may be needed when different quadrature rules
       or different DofToQuad::Mode are used. */
-   mutable Array<DofToQuad*> dof2quad_array;
+   mutable Array<DofToQuad *> dof2quad_array;

 public:
   /// Enumeration for range_type and deriv_range_type
@@ -307,19 +307,20 @@ public:
   FiniteElement(int D, Geometry::Type G, int Do, int O,
                 int F = FunctionSpace::Pk);

-   /// Returns the reference space dimension for the finite element
+   /// Returns the reference space dimension for the finite element.
   int GetDim() const { return dim; }

-   /// Returns the vector dimension for vector-valued finite elements
-   int GetVDim() const { return vdim; }
+   /** @brief Returns the vector dimension for vector-valued finite elements,
+       which is also the dimension of the interpolation operatrion. */
+   int GetRangeDim() const { return vdim; }

-   /// Returns the dimension of the curl for vector-valued finite elements
+   /// Returns the dimension of the curl for vector-valued finite elements.
   int GetCurlDim() const { return cdim; }

-   /// Returns the Geometry::Type of the reference element
+   /// Returns the Geometry::Type of the reference element.
   Geometry::Type GetGeomType() const { return geom_type; }

-   /// Returns the number of degrees of freedom in the finite element
+   /// Returns the number of degrees of freedom in the finite element.
   int GetDof() const { return dof; }

   /** @brief Returns the order of the finite element. In the case of
@@ -595,7 +596,7 @@ public:
   /** @brief Return a DoF transformation object for this particular type of
       basis.
   */
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return NULL; }

   /// Deconstruct the FiniteElement
@@ -1025,8 +1026,8 @@ public:
   };

 private:
-   typedef std::map< int, Array<double*>* > PointsMap;
-   typedef std::map< int, Array<Basis*>* > BasisMap;
+   typedef std::map<int, Array<double*>*> PointsMap;
+   typedef std::map<int, Array<Basis*>*> BasisMap;

   MemoryType h_mt;
   PointsMap points_container;
@@ -6031,7 +6031,7 @@ void RT0PyrFiniteElement::CalcVShape(const IntegrationPoint &ip,
   shape(1,2) = z;

   shape(2,0) = x * (2.0 - z) * ozi;
-   shape(2,1) = - y * z * ozi;;
+   shape(2,1) = - y * z * ozi;
   shape(2,2) = z;

   shape(3,0) = - x * z * ozi;
@@ -1852,7 +1852,7 @@ void ND_R1D_SegmentElement::Project(const FiniteElement &fe,
   else
   {
      double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());

      double * tk_ptr = const_cast<double*>(tk);

@@ -2293,7 +2293,7 @@ void ND_R2D_FiniteElement::Project(const FiniteElement &fe,
   else
   {
      double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());

      double * tk_ptr = const_cast<double*>(tk);

@@ -179,7 +179,7 @@ class ND_TetrahedronElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   mutable ND_TetStatelessDofTransformation doftrans;
+   ND_TetDofTransformation doftrans;

 public:
   /// Construct the ND_TetrahedronElement of order @a p
@@ -201,7 +201,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -242,7 +242,7 @@ class ND_TriangleElement : public VectorFiniteElement
   Array<int> dof2tk;
   DenseMatrixInverse Ti;

-   mutable ND_TriStatelessDofTransformation doftrans;
+   ND_TriDofTransformation doftrans;

 public:
   /// Construct the ND_TriangleElement of order @a p
@@ -264,7 +264,7 @@ public:
                                  ElementTransformation &Trans,
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }
-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }
   using FiniteElement::Project;
   virtual void Project(VectorCoefficient &vc,
@@ -346,7 +346,7 @@ private:
 #endif
   Array<int> dof2tk, t_dof, s_dof;

-   mutable ND_WedgeStatelessDofTransformation doftrans;
+   ND_WedgeDofTransformation doftrans;

   H1_TriangleElement H1TriangleFE;
   ND_TriangleElement NDTriangleFE;
@@ -379,7 +379,7 @@ public:
                                  DenseMatrix &I) const
   { LocalInterpolation_ND(CheckVectorFE(fe), tk, dof2tk, Trans, I); }

-   virtual StatelessDofTransformation * GetDofTransformation() const
+   virtual const StatelessDofTransformation *GetDofTransformation() const
   { return &doftrans; }

   using FiniteElement::Project;
@@ -1486,7 +1486,7 @@ void RT_R1D_SegmentElement::Project(const FiniteElement &fe,
   else
   {
      double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());

      double * nk_ptr = const_cast<double*>(nk);

@@ -1523,7 +1523,7 @@ void RT_R1D_SegmentElement::ProjectCurl(const FiniteElement &fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &curl) const
 {
-   DenseMatrix curl_shape(fe.GetDof(), fe.GetVDim());
+   DenseMatrix curl_shape(fe.GetDof(), fe.GetRangeDim());
   Vector curl_k(fe.GetDof());

   double * nk_ptr = const_cast<double*>(nk);
@@ -1849,7 +1849,7 @@ void RT_R2D_FiniteElement::Project(const FiniteElement &fe,
   else
   {
      double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());

      double * nk_ptr = const_cast<double*>(nk);

@@ -1888,7 +1888,7 @@ void RT_R2D_FiniteElement::ProjectCurl(const FiniteElement &fe,
                                       ElementTransformation &Trans,
                                       DenseMatrix &curl) const
 {
-   DenseMatrix curl_shape(fe.GetDof(), fe.GetVDim());
+   DenseMatrix curl_shape(fe.GetDof(), fe.GetRangeDim());
   Vector curl_k(fe.GetDof());

   double * nk_ptr = const_cast<double*>(nk);
@@ -59,7 +59,7 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
   Vector nodalX(p+1);
   Vector nodalY(p+1);

@@ -113,10 +113,9 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,
   {
      double *legX = new double[p-1];
      double *legY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();

-      storeLegendre->CalcLegendre(p-2, x, legX);
-      storeLegendre->CalcLegendre(p-2, y, legY);
+      Poly_1D::CalcLegendre(p-2, x, legX);
+      Poly_1D::CalcLegendre(p-2, y, legY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -131,7 +130,6 @@ void H1Ser_QuadrilateralElement::CalcShape(const IntegrationPoint &ip,

      delete[] legX;
      delete[] legY;
-      delete storeLegendre;
   }
 }

@@ -141,7 +139,7 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
   int p = (this)->GetOrder();
   double x = ip.x, y = ip.y;

-   Poly_1D::Basis edgeNodalBasis(poly1d.GetBasis(p, BasisType::GaussLobatto));
+   Poly_1D::Basis &edgeNodalBasis = poly1d.GetBasis(p, BasisType::GaussLobatto);
   Vector nodalX(p+1);
   Vector DnodalX(p+1);
   Vector nodalY(p+1);
@@ -203,10 +201,9 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      double *legY = new double[p-1];
      double *DlegX = new double[p-1];
      double *DlegY = new double[p-1];
-      Poly_1D *storeLegendre = new Poly_1D();

-      storeLegendre->CalcLegendre(p-2, x, legX, DlegX);
-      storeLegendre->CalcLegendre(p-2, y, legY, DlegY);
+      Poly_1D::CalcLegendre(p-2, x, legX, DlegX);
+      Poly_1D::CalcLegendre(p-2, y, legY, DlegY);

      int interior_total = 0;
      for (int j = 4; j < p + 1; j++)
@@ -224,7 +221,6 @@ void H1Ser_QuadrilateralElement::CalcDShape(const IntegrationPoint &ip,
      delete[] legY;
      delete[] DlegX;
      delete[] DlegY;
-      delete storeLegendre;
   }
 }

@@ -87,6 +87,16 @@ int FiniteElementCollection::GetDerivMapType(int dim) const
   return FiniteElement::UNKNOWN_MAP_TYPE;
 }

+int FiniteElementCollection::GetRangeDim(int dim) const
+{
+   const FiniteElement *fe = FiniteElementForDim(dim);
+   if (fe)
+   {
+      return fe->GetRangeDim();
+   }
+   return 0;
+}
+
 int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
 {
   switch (geom)
@@ -2886,7 +2896,7 @@ ND_FECollection::FiniteElementForGeometry(Geometry::Type GeomType) const
   }
 }

-StatelessDofTransformation *
+const StatelessDofTransformation *
 ND_FECollection::DofTransformationForGeometry(Geometry::Type GeomType) const
 {
   if (!Geometry::IsTensorProduct(GeomType) && this->GetOrder() > 1)
@@ -13,6 +13,7 @@
 #define MFEM_FEM_HPP

 #include "intrules.hpp"
+#include "intrules_cut.hpp"
 #include "geom.hpp"
 #include "fe.hpp"
 #include "fe_coll.hpp"
@@ -26,6 +27,7 @@
 #include "bilininteg.hpp"
 #include "fespace.hpp"
 #include "gridfunc.hpp"
+#include "kdtree.hpp"
 #include "linearform.hpp"
 #include "nonlinearform.hpp"
 #include "bilinearform.hpp"
@@ -53,6 +55,7 @@
 #include "plinearform.hpp"
 #include "pbilinearform.hpp"
 #include "pnonlinearform.hpp"
+#include "pblockform.hpp"
 #endif

 #ifdef MFEM_USE_SIDRE
@@ -214,7 +214,7 @@ class FaceQuadratureInterpolator;
    @par
    Clearly the notion of a @b vdof is relevant in each of the three contexts
    mentioned above so extra care must be taken whenever @b vdim != 1 to ensure
-    that the @b edof, @b ldof, or @b tdof is being interpretted correctly.
+    that the @b edof, @b ldof, or @b tdof is being interpreted correctly.
 */
 class FiniteElementSpace
 {
@@ -271,18 +271,20 @@ protected:
   int own_ext;
   mutable Array<int> face_to_be; // NURBS FE space only

-   Array<DofTransformation*> DoFTrans;
-   mutable VDofTransformation VDoFTrans;
+   Array<StatelessDofTransformation *> DoFTransArray;
+   mutable DofTransformation DoFTrans;

   /** Matrix representing the prolongation from the global conforming dofs to
       a set of intermediate partially conforming dofs, e.g. the dofs associated
       with a "cut" space on a non-conforming mesh. */
-   mutable SparseMatrix *cP; // owned
+   mutable std::unique_ptr<SparseMatrix> cP;
   /// Conforming restriction matrix such that cR.cP=I.
-   mutable SparseMatrix *cR; // owned
+   mutable std::unique_ptr<SparseMatrix> cR;
   /// A version of the conforming restriction matrix for variable-order spaces.
-   mutable SparseMatrix *cR_hp; // owned
+   mutable std::unique_ptr<SparseMatrix> cR_hp;
   mutable bool cP_is_set;
+   /// Operator computing the action of the transpose of the restriction.
+   mutable std::unique_ptr<Operator> R_transpose;

   /// Transformation to apply to GridFunctions after space Update().
   OperatorHandle Th;
@@ -326,8 +328,8 @@ protected:
   void Construct();
   void Destroy();

-   void ConstructDoFTrans();
-   void DestroyDoFTrans();
+   void ConstructDoFTransArray();
+   void DestroyDoFTransArray();

   void BuildElementToDofTable() const;
   void BuildBdrElementToDofTable() const;
@@ -381,6 +383,10 @@ protected:
   int GetEntityDofs(int entity, int index, Array<int> &dofs,
                     Geometry::Type master_geom = Geometry::INVALID,
                     int variant = 0) const;
+   /// Helper to get vertex, edge or face VDOFs (entity=0,1,2 resp.).
+   int GetEntityVDofs(int entity, int index, Array<int> &dofs,
+                      Geometry::Type master_geom = Geometry::INVALID,
+                      int variant = 0) const;

   // Get degenerate face DOFs: see explanation in method implementation.
   int GetDegenerateFaceDofs(int index, Array<int> &dofs,
@@ -414,10 +420,10 @@ protected:
      Table* old_elem_dof; // Owned.
      Table* old_elem_fos; // Owned.

-      Array<DofTransformation*> old_DoFTrans;
-      mutable VDofTransformation old_VDoFTrans;
+      Array<StatelessDofTransformation*> old_DoFTransArray;
+      mutable DofTransformation old_DoFTrans;

-      void ConstructDoFTrans();
+      void ConstructDoFTransArray();

   public:
      /** Construct the operator based on the elem_dof table of the original
@@ -592,10 +598,17 @@ public:
   { return GetConformingProlongation(); }

   /// Return an operator that performs the transpose of GetRestrictionOperator
-   /** The returned operator is owned by the FiniteElementSpace. In serial this
-       is the same as GetProlongationMatrix() */
-   virtual const Operator *GetRestrictionTransposeOperator() const
-   { return GetConformingProlongation(); }
+   /** The returned operator is owned by the FiniteElementSpace.
+
+       For a serial conforming space, this returns NULL, indicating the identity
+       operator.
+
+       For a parallel conforming space, this will return a matrix-free
+       (Device)ConformingProlongationOperator.
+
+       For a non-conforming mesh this will return a TransposeOperator wrapping
+       the restriction matrix. */
+   const Operator *GetRestrictionTransposeOperator() const;

   /// An abstract operator that performs the same action as GetRestrictionMatrix
   /** In some cases this is an optimized matrix-free implementation. The
@@ -794,7 +807,16 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   virtual DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+   DofTransformation *GetElementDofs(int elem, Array<int> &dofs) const;
+
+   /// @brief The same as GetElementDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   virtual void GetElementDofs(int elem, Array<int> &dofs,
+                               DofTransformation &doftrans) const;

   /// @brief Returns indices of degrees of freedom for boundary element 'bel'.
   /// The returned indices are offsets into an @ref ldof vector. See also
@@ -808,13 +830,16 @@ public:
   /// with triangular faces.
   ///
   /// @note The returned object should NOT be deleted by the caller.
-   virtual DofTransformation *GetBdrElementDofs(int bel,
-                                                Array<int> &dofs) const;
+   DofTransformation *GetBdrElementDofs(int bel, Array<int> &dofs) const;

-   /** @brief Returns indices of degrees of freedom for NURBS patch index
-    @a patch. Cartesian ordering is used, for the tensor-product degrees of
-    freedom. */
-   void GetPatchDofs(int patch, Array<int> &dofs) const;
+   /// @brief The same as GetBdrElementDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   virtual void GetBdrElementDofs(int bel, Array<int> &dofs,
+                                  DofTransformation &doftrans) const;

   /// @brief Returns the indices of the degrees of freedom for the specified
   /// face, including the DOFs for the edges and the vertices of the face.
@@ -861,6 +886,13 @@ public:
   /// GetElementInteriorVDofs().
   void GetElementInteriorDofs(int i, Array<int> &dofs) const;

+   /// @brief Returns the number of degrees of freedom associated with the
+   /// interior of the specified element.
+   ///
+   /// See GetElementInteriorDofs() for more information or to obtain the
+   /// relevant indices.
+   int GetNumElementInteriorDofs(int i) const;
+
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified face.
   ///
@@ -873,13 +905,6 @@ public:
   /// GetFaceInteriorVDofs().
   void GetFaceInteriorDofs(int i, Array<int> &dofs) const;

-   /// @brief Returns the number of degrees of freedom associated with the
-   /// interior of the specified element.
-   ///
-   /// See GetElementInteriorDofs() for more information or to obtain the
-   /// relevant indices.
-   int GetNumElementInteriorDofs(int i) const;
-
   /// @brief Returns the indices of the degrees of freedom for the interior
   /// of the specified edge.
   ///
@@ -888,6 +913,11 @@ public:
   void GetEdgeInteriorDofs(int i, Array<int> &dofs) const;
   ///@}

+   /** @brief Returns indices of degrees of freedom for NURBS patch index
+    @a patch. Cartesian ordering is used, for the tensor-product degrees of
+    freedom. */
+   void GetPatchDofs(int patch, Array<int> &dofs) const;
+
   /// @anchor dof2vdof @name DoF To VDoF Conversion methods
   /// These methods convert between local dof and local vector dof using the
   /// appropriate relationship based on the Ordering::Type defined in this
@@ -898,7 +928,7 @@ public:
   /// changed in the forward mappings by passing a value for @a ndofs which
   /// differs from that returned by GetNDofs().
   ///
-   /// @note Thse methods, with the exception of VDofToDof(), are designed to
+   /// @note These methods, with the exception of VDofToDof(), are designed to
   /// produce the correctly encoded values when dof entries are negative,
   /// see @ref ldof for more on negative dof indices.
   ///
@@ -1014,6 +1044,15 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetElementVDofs(int i, Array<int> &vdofs) const;

+   /// @brief The same as GetElementVDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   void GetElementVDofs(int i, Array<int> &vdofs,
+                        DofTransformation &doftrans) const;
+
   /// @brief Returns indices of degrees of freedom for @a i'th boundary
   /// element.
   /// The returned indices are offsets into an @ref ldof vector with @b vdim
@@ -1029,6 +1068,15 @@ public:
   /// @note The returned object should NOT be deleted by the caller.
   DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;

+   /// @brief The same as GetBdrElementVDofs(), but with a user-allocated
+   /// DofTransformation object. @a doftrans must be allocated in advance and
+   /// will be owned by the caller. The user can use the
+   /// DofTransformation::GetDofTransformation method on the returned
+   /// @a doftrans object to detect if the DofTransformation should actually be
+   /// used.
+   void GetBdrElementVDofs(int i, Array<int> &vdofs,
+                           DofTransformation &doftrans) const;
+
   /// Returns indices of degrees of freedom in @a vdofs for NURBS patch @a i.
   void GetPatchVDofs(int i, Array<int> &vdofs) const;

@@ -31,13 +31,13 @@ FmsBasisTypeToMfemBasis(FmsBasisType b)
   switch (b)
   {
      case FMS_NODAL_GAUSS_OPEN:
-         retval = mfem::BasisType::GaussLegendre;;
+         retval = mfem::BasisType::GaussLegendre;
         break;
      case FMS_NODAL_GAUSS_CLOSED:
-         retval = mfem::BasisType::GaussLobatto;;
+         retval = mfem::BasisType::GaussLobatto;
         break;
      case FMS_POSITIVE:
-         retval = mfem::BasisType::Positive;;
+         retval = mfem::BasisType::Positive;
         break;
      case FMS_NODAL_UNIFORM_OPEN:
         retval = mfem::BasisType::OpenUniform;
@@ -1812,22 +1812,22 @@ MeshToFmsMesh(const Mesh *mmesh, FmsMesh *fmesh, FmsComponent *volume)
      switch (betype)
      {
         case Element::POINT:
-            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_VERTEX].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::SEGMENT:
-            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_EDGE].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::TRIANGLE:
-            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_TRIANGLE].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::QUADRILATERAL:
-            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_QUADRILATERAL].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::TETRAHEDRON:
-            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_TETRAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         case Element::HEXAHEDRON:
-            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementEdgeIndex(i));
+            bdr_eles[FMS_HEXAHEDRON].push_back(mmesh->GetBdrElementFaceIndex(i));
            break;
         default:
            MFEM_WARNING("Unsupported boundary element " << betype << " at boundary index "
--- a/Show More
+++ b/Show More