minor edits

adding x field

.github/workflows/build-container.yaml

@@ -1,6 +1,7 @@
 name: "Docker"
 
 on:
+
   # Always have a base image ready to go - this is a nightly build
   schedule:
     - cron: 0 3 * * *
@@ -25,6 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+
         # Dockerfiles to build, a matrix supports future expanded builds
         container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
                     ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
@@ -32,20 +34,15 @@ jobs:
     runs-on: ubuntu-latest
     name: Build
     steps:
-      - name: Run Actions Cleaner
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-          remove-codeql: 'true'
-          remove-docker-images: 'true'
-
       - name: Checkout
         uses: actions/checkout@v3
 
-      # It's easier to reference named variables than indexes of the matrix
+      - name: Make Space For Build
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+
+       # It's easier to reference named variables than indexes of the matrix
       - name: Set Environment
         env:
           dockerfile: ${{ matrix.container[0] }}

.github/workflows/builds-and-tests.yml

@@ -65,16 +65,13 @@ jobs:
         # - Add a new combination.
         #   'build-system: cmake' and 'hypre-target: int64'
         #
-        # Note: we will gather coverage info for any non-debug run except the
+        # note: we will gather coverage info for any non-debug run except the
         # CMake build.
         include:
           - target: dbg
             codecov: NO
           - target: opt
             codecov: YES
-          - os: ubuntu-latest
-            target: dbg
-            config-opts: 'CPPFLAGS+=-Og'
           - os: windows-latest
             codecov: NO
           - os: windows-latest
@@ -101,189 +98,181 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      # This external action allows to interrupt a workflow already running on
-      # the same branch to save resources.
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
+    # This external action allows to interrupt a workflow already running on
+    # the same branch to save resource
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
 
-      # Fix 'No space left on device' errors for Ubuntu builds.
-      - name: Run Actions Cleaner
-        if: matrix.os == 'ubuntu-latest'
-        uses: easimon/maximize-build-space@v8
-        with:
-          overprovision-lvm: 'true'
-          remove-android: 'true'
+    # Checkout MFEM in "mfem" subdirectory. Final path:
+    # /home/runner/work/mfem/mfem/mfem
+    # Note: Done now to access "install-hypre" and "install-metis" actions.
+    - name: checkout mfem
+      uses: actions/checkout@v3
+      with:
+        path: ${{ env.MFEM_TOP_DIR }}
+        # Fetch the complete history for codecov to access commits ID
+        fetch-depth: 0
 
-      # Checkout MFEM in "mfem" subdirectory. Final path:
-      # /home/runner/work/mfem/mfem/mfem
-      # Note: Done now to access "install-hypre" and "install-metis" actions.
-      - name: checkout mfem
-        uses: actions/checkout@v3
-        with:
-          path: ${{ env.MFEM_TOP_DIR }}
-          # Fetch the complete history for codecov to access commits ID
-          fetch-depth: 0
+    # Only get MPI if defined for the job.
+    # TODO: It would be nice to have only one step, e.g. with a dedicated
+    # action, but I (@adrienbernede) don't see how at the moment.
+    - name: get MPI (Linux)
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get install mpich libmpich-dev
 
-      # Only get MPI if defined for the job.
-      # TODO: It would be nice to have only one step, e.g. with a dedicated
-      # action, but I (@adrienbernede) don't see how at the moment.
-      - name: get MPI (Linux)
-        if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install mpich libmpich-dev
+    - name: get lcov (Linux)
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get install lcov
 
-      - name: get lcov (Linux)
-        if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
-        run: |
-          sudo apt-get install lcov
+    # Keep the following section in case we need it again in the future,
+    # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
+    # - name: Set up Homebrew
+    #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
+    #   uses: Homebrew/actions/setup-homebrew@master
 
-      # Keep the following section in case we need it again in the future,
-      # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-      # - name: Set up Homebrew
-      #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-      #   uses: Homebrew/actions/setup-homebrew@master
+    - name: get MPI (MacOS)
+      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
+      run: |
+        export HOMEBREW_NO_INSTALL_CLEANUP=1
+        brew install openmpi
 
-      - name: get MPI (MacOS)
-        if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install openmpi
+    - name: get lcov (MacOS)
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
+      run: |
+        export HOMEBREW_NO_INSTALL_CLEANUP=1
+        brew install lcov
 
-      - name: get lcov (MacOS)
-        if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
-        run: |
-          export HOMEBREW_NO_INSTALL_CLEANUP=1
-          brew install lcov
+    - name: get MPI (Windows)
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      uses: mpi4py/setup-mpi@v1.1.4
 
-      - name: get MPI (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        uses: mpi4py/setup-mpi@v1.1.4
+    # Get Hypre through cache, or build it.
+    # Install will only run on cache miss.
+    - name: cache hypre
+      id: hypre-cache
+      if: matrix.mpi == 'par'
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.HYPRE_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
 
-      # Get Hypre through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache hypre
-        id: hypre-cache
-        if: matrix.mpi == 'par'
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
+    - name: get hypre
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
+      uses: mfem/github-actions/build-hypre@v2.4
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}
+        build-system: make
 
-      - name: get hypre
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: make
+    - name: get hypre (Windows)
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
+      uses: mfem/github-actions/build-hypre@v2.4
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: ${{ matrix.hypre-target }}
+        build-system: cmake
 
-      - name: get hypre (Windows)
-        if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: ${{ matrix.hypre-target }}
-          build-system: cmake
+    # Get Metis through cache, or build it.
+    # Install will only run on cache miss.
+    - name: cache metis
+      id: metis-cache
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.METIS_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
 
-      # Get Metis through cache, or build it.
-      # Install will only run on cache miss.
-      - name: cache metis
-        id: metis-cache
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+    - name: install metis
+      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.4
+      with:
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}
 
-      - name: install metis
-        if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
+    - name: cache vcpkg (Windows)
+      id: vcpkg-cache
+      if: matrix.os == 'windows-latest'
+      uses: actions/cache@v3
+      with:
+        path: vcpkg_cache
+        key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
 
-      - name: cache vcpkg (Windows)
-        id: vcpkg-cache
-        if: matrix.os == 'windows-latest'
-        uses: actions/cache@v3
-        with:
-          path: vcpkg_cache
-          key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
+    - name: prepare vcpkg binary cache location (Windows)
+      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p vcpkg_cache
 
-      - name: prepare vcpkg binary cache location (Windows)
-        if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p vcpkg_cache
+    - name: install metis (Windows)
+      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      env:
+        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+      run: |
+        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
 
-      - name: install metis (Windows)
-        if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        run: |
-          vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
+    # MFEM build and test
+    - name: build
+      uses: mfem/github-actions/build-mfem@v2.4
+      env:
+        VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
+      with:
+        os: ${{ matrix.os }}
+        target: ${{ matrix.target }}
+        codecov: ${{ matrix.codecov }}
+        mpi: ${{ matrix.mpi }}
+        build-system: ${{ matrix.build-system }}
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}
+        mfem-dir: ${{ env.MFEM_TOP_DIR }}
+        config-options: ${{ matrix.config-opts }}
+        library-only: ${{ matrix.target == 'dbg' }}
 
-      # MFEM build and test
-      - name: build
-        uses: mfem/github-actions/build-mfem@v2.4
-        env:
-          VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
-        with:
-          os: ${{ matrix.os }}
-          target: ${{ matrix.target }}
-          codecov: ${{ matrix.codecov }}
-          mpi: ${{ matrix.mpi }}
-          build-system: ${{ matrix.build-system }}
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: ${{ env.MFEM_TOP_DIR }}
-          config-options: ${{ matrix.config-opts }}
-          library-only: ${{ matrix.target == 'dbg' && matrix.os != 'ubuntu-latest' }}
+    # Run checks (and only checks) on debug targets
+    - name: checks
+      if: matrix.build-system == 'make' && matrix.target == 'dbg'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }}  && make check
 
-      # Run checks (and only checks) on debug targets
-      - name: checks
-        if: matrix.build-system == 'make' && matrix.target == 'dbg'
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }}  && make check
+    # Note: 'tests' include the unit tests
+    - name: tests
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }} && make test
 
-      # Note: 'tests' include the unit tests
-      - name: tests
-        if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
-        run: |
-          cd ${{ env.MFEM_TOP_DIR }} && make test
+    - name: cmake checks
+      if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
+      run: |
+        CTEST_CONFIG="Debug"
+        cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
+      shell: bash
 
-      - name: cmake checks
-        if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
-        run: |
-          CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
-        shell: bash
+    - name: cmake unit tests (Ubuntu)
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
+      run: |
+        CTEST_CONFIG="Release"
+        [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
+        cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
+      shell: bash
 
-      - name: cmake unit tests (Ubuntu)
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
-          cd ${{ env.MFEM_TOP_DIR }}/build/tests/unit && ctest --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
+    - name: cmake tests
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
+      run: |
+        CTEST_CONFIG="Release"
+        cd ${{ env.MFEM_TOP_DIR }}/build && \
+        ctest --output-on-failure -C ${CTEST_CONFIG} || \
+        ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
+      shell: bash
 
-      - name: cmake tests
-        if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
-        run: |
-          CTEST_CONFIG="Release"
-          cd ${{ env.MFEM_TOP_DIR }}/build && \
-          ctest --output-on-failure -C ${CTEST_CONFIG} || \
-          ctest --rerun-failed --output-on-failure -C ${CTEST_CONFIG}
-        shell: bash
-
-      # Code coverage (process and upload reports)
-      - name: codecov
-        if: matrix.codecov == 'YES'
-        uses: mfem/github-actions/upload-coverage@v2.4
-        with:
-          name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
-          project_dir: ${{ env.MFEM_TOP_DIR }}
-          directories: "fem general linalg mesh"
+    # Code coverage (process and upload reports)
+    - name: codecov
+      if: matrix.codecov == 'YES'
+      uses: mfem/github-actions/upload-coverage@v2.4
+      with:
+        name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
+        project_dir: ${{ env.MFEM_TOP_DIR }}
+        directories: "fem general linalg mesh"

.github/workflows/codeql-analysis.yml

@@ -13,10 +13,10 @@ name: "Static Analysis"
 
 on:
   push:
-    branches: ["master", "next"]
+    branches: [ "master", "next"]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: ["master"]
+    branches: [ "master" ]
 
 jobs:
   analyze:
@@ -35,35 +35,36 @@ jobs:
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+    - name: Checkout repository
+      uses: actions/checkout@v3
 
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
 
-          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-          # queries: security-extended,security-and-quality
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
 
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
 
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
 
-      #   If the Autobuild fails above, remove it and uncomment the following three lines.
-      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-      # - run: |
-      #   echo "Run, Build Application using script"
-      #   ./location_of_script_within_repo/buildscript.sh
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -34,67 +34,67 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
 
-      - name: checkout MFEM
-        uses: actions/checkout@v3
-        with:
-          path: mfem
+    - name: checkout MFEM
+      uses: actions/checkout@v3
+      with:
+        path: mfem
 
-      - name: Get MPI (Linux)
-        run: |
-          sudo apt-get install mpich libmpich-dev
+    - name: Get MPI (Linux)
+      run: |
+        sudo apt-get install mpich libmpich-dev
 
-      - name: Cache Hypre Install
-        id: hypre-cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.HYPRE_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
+    - name: Cache Hypre Install
+      id: hypre-cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.HYPRE_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
 
-      - name: Get Hypre
-        if: steps.hypre-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-hypre@v2.4
-        with:
-          archive: ${{ env.HYPRE_ARCHIVE }}
-          dir: ${{ env.HYPRE_TOP_DIR }}
-          target: int32
+    - name: Get Hypre
+      if: steps.hypre-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-hypre@v2.4
+      with:
+        archive: ${{ env.HYPRE_ARCHIVE }}
+        dir: ${{ env.HYPRE_TOP_DIR }}
+        target: int32
 
-      - name: Cache Metis Install
-        id: metis-cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.METIS_TOP_DIR }}
-          key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
+    - name: Cache Metis Install
+      id: metis-cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.METIS_TOP_DIR }}
+        key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
 
-      - name: Install Metis
-        if: steps.metis-cache.outputs.cache-hit != 'true'
-        uses: mfem/github-actions/build-metis@v2.4
-        with:
-          archive: ${{ env.METIS_ARCHIVE }}
-          dir: ${{ env.METIS_TOP_DIR }}
+    - name: Install Metis
+      if: steps.metis-cache.outputs.cache-hit != 'true'
+      uses: mfem/github-actions/build-metis@v2.4
+      with:
+        archive: ${{ env.METIS_ARCHIVE }}
+        dir: ${{ env.METIS_TOP_DIR }}
 
-      # MFEM build and test
-      - name: build-mfem
-        uses: mfem/github-actions/build-mfem@v2.4
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          codecov: NO
-          mpi: par
-          build-system: make
-          hypre-dir: ${{ env.HYPRE_TOP_DIR }}
-          metis-dir: ${{ env.METIS_TOP_DIR }}
-          mfem-dir: mfem
+    # MFEM build and test
+    - name: build-mfem
+      uses: mfem/github-actions/build-mfem@v2.4
+      with:
+        os: ${{ runner.os }}
+        target: opt
+        codecov: NO
+        mpi: par
+        build-system: make
+        hypre-dir: ${{ env.HYPRE_TOP_DIR }}
+        metis-dir: ${{ env.METIS_TOP_DIR }}
+        mfem-dir: mfem
 
-      - name: test (no clean)
-        run: |
-          cd mfem && make test-noclean
+    - name: test (no clean)
+      run: |
+        cd mfem && make test-noclean
 
-      - name: gitignore
-        run: |
-          cd mfem/tests/scripts
-          ./runtest gitignore
+    - name: gitignore
+      run: |
+        cd mfem/tests/scripts
+        ./runtest gitignore

.github/workflows/mfem-sanitizer.yml

@@ -27,44 +27,44 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
 
-      - name: MFEM Checkout
-        uses: actions/checkout@v3
-        with:
-          path: mfem
+    - name: MFEM Checkout
+      uses: actions/checkout@v3
+      with:
+        path: mfem
 
-      - name: MFEM Build
-        uses: mfem/github-actions/build-mfem@v2.4
-        with:
-          os: ${{ runner.os }}
-          target: opt
-          mpi: seq
-          hypre-dir: unused-hypre-dir
-          metis-dir: unused-metis-dir
-          mfem-dir: mfem
-          build-system: make
-          library-only: false
-          config-options:
-            CXX="clang++-14"
-            CXXFLAGS="-g -O1 -std=c++11
-                      -fsanitize=address
-                      -fno-omit-frame-pointer
-                      -fsanitize-address-use-after-scope"
+    - name: MFEM Build
+      uses: mfem/github-actions/build-mfem@v2.4
+      with:
+        os: ${{ runner.os }}
+        target: opt
+        mpi: seq
+        hypre-dir: unused-hypre-dir
+        metis-dir: unused-metis-dir
+        mfem-dir: mfem
+        build-system: make
+        library-only: false
+        config-options:
+          CXX="clang++-14"
+          CXXFLAGS="-g -O1 -std=c++11
+                    -fsanitize=address
+                    -fno-omit-frame-pointer
+                    -fsanitize-address-use-after-scope"
 
-      - name: MFEM Info
-        working-directory: mfem
-        run: make info
+    - name: MFEM Info
+      working-directory: mfem
+      run: make info
 
-      - name: MFEM Sanitize
-        working-directory: mfem
-        run:
-          ASAN_OPTIONS="detect_leaks=1,
-                        strict_init_order=1,
-                        strict_string_checks=1,
-                        check_initialization_order=1,
-                        detect_stack_use_after_return=1"
-          make test
+    - name: MFEM Sanitize
+      working-directory: mfem
+      run:
+        ASAN_OPTIONS="detect_leaks=1,
+                      strict_init_order=1,
+                      strict_string_checks=1,
+                      check_initialization_order=1,
+                      detect_stack_use_after_return=1"
+        make test

.github/workflows/repo-check.yml

@@ -33,49 +33,49 @@ jobs:
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.11.0
-        with:
-          access_token: ${{ github.token }}
+    - name: Cancel Previous Runs
+      uses: styfle/cancel-workflow-action@0.11.0
+      with:
+        access_token: ${{ github.token }}
 
-      - name: checkout mfem
-        uses: actions/checkout@v3
+    - name: checkout mfem
+      uses: actions/checkout@v3
 
-      - name: copyright check
-        id: copyright
-        run: |
-          ./config/githooks/pre-push --copyright
+    - name: copyright check
+      id: copyright
+      run: |
+        ./config/githooks/pre-push --copyright
 
-        continue-on-error: true
+      continue-on-error: true
 
-      - name: license check
-        id: license
-        run: |
-          ./config/githooks/pre-push --license
-        continue-on-error: true
+    - name: license check
+      id: license
+      run: |
+        ./config/githooks/pre-push --license
+      continue-on-error: true
 
-      - name: release check
-        id: release
-        run: |
-          ./config/githooks/pre-push --release
-        continue-on-error: true
+    - name: release check
+      id: release
+      run: |
+        ./config/githooks/pre-push --release
+      continue-on-error: true
 
-      - name: wrap-up
-        if: |
-          steps.copyright.outcome != 'success' ||
-          steps.license.outcome != 'success' ||
-          steps.release.outcome != 'success'
-        run: |
-          if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
-            echo "copyright check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.license.outcome }}" != "success" ]]; then
-            echo "license check failed, unroll log for details"
-          fi
-          if [[ "${{ steps.release.outcome }}" != "success" ]]; then
-            echo "release check failed, unroll log for details"
-          fi
-          exit 1
+    - name: wrap-up
+      if: |
+        steps.copyright.outcome != 'success' ||
+        steps.license.outcome != 'success' ||
+        steps.release.outcome != 'success'
+      run: |
+        if [[ "${{ steps.copyright.outcome }}" != "success" ]]; then
+          echo "copyright check failed, unroll log for details"
+        fi
+        if [[ "${{ steps.license.outcome }}" != "success" ]]; then
+          echo "license check failed, unroll log for details"
+        fi
+        if [[ "${{ steps.release.outcome }}" != "success" ]]; then
+          echo "release check failed, unroll log for details"
+        fi
+        exit 1
 
   code-style:
     runs-on: ubuntu-latest
@@ -83,16 +83,16 @@ jobs:
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
-      - name: checkout mfem
-        uses: actions/checkout@v3
+    - name: checkout mfem
+      uses: actions/checkout@v3
 
-      - name: get astyle
-        run: |
-          sudo apt-get install astyle
+    - name: get astyle
+      run: |
+        sudo apt-get install astyle
 
-      - name: style check
-        run: |
-          ./config/githooks/pre-push --style
+    - name: style check
+      run: |
+        ./config/githooks/pre-push --style
 
   documentation:
     runs-on: ubuntu-latest
@@ -100,22 +100,22 @@ jobs:
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
-      - name: checkout mfem
-        uses: actions/checkout@v3
+    - name: checkout mfem
+      uses: actions/checkout@v3
 
-      - name: get doxygen and graphviz
-        run: |
-          sudo apt-get install doxygen graphviz
+    - name: get doxygen and graphviz
+      run: |
+        sudo apt-get install doxygen graphviz
 
-      - name: update doxygen config file
-        run: |
-          cd doc
-          doxygen -u CodeDocumentation.conf.in
+    - name: update doxygen config file
+      run: |
+        cd doc
+        doxygen -u CodeDocumentation.conf.in
 
-      - name: build documentation
-        run: |
-          cd tests/scripts
-          ./runtest documentation
+    - name: build documentation
+      run: |
+        cd tests/scripts
+        ./runtest documentation
 
   branch-history:
     if: |
@@ -125,16 +125,16 @@ jobs:
        github.event.pull_request.head.repo.full_name != github.repository)
     runs-on: ubuntu-latest
     steps:
-      - name: checkout mfem
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
+    - name: checkout mfem
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
 
-      - name: branch-history
-        run: |
-          # We override origin to make sure we point to the main repo.
-          # This is to have consistent test results on PRs from forks.
-          git remote remove origin
-          git remote add origin https://github.com/mfem/mfem.git
-          git checkout -b gh-actions-branch-history
-          ./config/githooks/pre-push --history
+    - name: branch-history
+      run: |
+        # We override origin to make sure we point to the main repo.
+        # This is to have consistent test results on PRs from forks.
+        git remote remove origin
+        git remote add origin https://github.com/mfem/mfem.git
+        git checkout -b gh-actions-branch-history
+        ./config/githooks/pre-push --history

.gitignore

@@ -213,7 +213,6 @@ miniapps/meshing/twist
 miniapps/meshing/mesh-explorer
 miniapps/meshing/shaper
 miniapps/meshing/extruder
-miniapps/meshing/fit-node-position
 miniapps/meshing/trimmer
 miniapps/meshing/reflector
 miniapps/meshing/mesh-optimizer
@@ -266,15 +265,11 @@ miniapps/navier/*_output
 miniapps/nurbs/nurbs_ex1
 miniapps/nurbs/nurbs_ex1p
 miniapps/nurbs/nurbs_ex11p
-miniapps/nurbs/nurbs_patch_ex1
-miniapps/nurbs/nurbs_curveint
 miniapps/nurbs/refined.mesh
 miniapps/nurbs/mesh.*
 miniapps/nurbs/sol.*
 miniapps/nurbs/mode_*
 miniapps/nurbs/Example1*
-miniapps/nurbs/sin-fit.mesh
-miniapps/nurbs/CurveInt
 
 miniapps/performance/ex1
 miniapps/performance/ex1p
@@ -297,14 +292,9 @@ miniapps/tools/display-basis
 miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
-miniapps/tools/plor-transfer
 miniapps/tools/get-values
 miniapps/tools/check-tmop-metric
 miniapps/tools/tmop-metric-magnitude
-miniapps/tools/nodal-transfer
-miniapps/tools/ParaView
-miniapps/tools/gridfunc_*
-miniapps/tools/mesh_*
 
 miniapps/toys/automata
 miniapps/toys/life

CHANGELOG

@@ -8,138 +8,88 @@
                                https://mfem.org
 
 
-Version 4.6.1 (development)
+Version 4.5.3 (development)
 ===========================
 
-
-Version 4.6, released on September 27, 2023
-===========================================
-
-- MFEM is now available in Homebrew and can be installed on a Mac with just
-  "brew install mfem". See https://formulae.brew.sh/formula/mfem.
-
-Meshing improvements
---------------------
-- Added asymptotically-balanced TMOP compound metrics 90, 94, 328, 338. A new
-  tool, tmop-metric-magnitude, can be used to track how metrics change under
-  geometric perturbations. See miniapps/tools.
-
-- Several NURBS meshing improvements:
-  * Support for free connectivity of NURBS patches allowing for more complex
-    patch configurations such as C-meshes.
-  * New methods to set and get attributes on NURBS patches and patch boundaries.
-  * The edge to knot map for NURBS meshes can be determined automatically. It is
-    no longer needed to specify this in the NURBS mesh.
-  * Added curve interpolation method for NURBS.
-  * See miniapps/nurbs for example meshes and miniapps.
-
-Discretization improvements
----------------------------
-- SubMesh and ParSubMesh have been extended to support the transfer of
-  Nedelec and Raviart-Thomas finite element spaces.
-
-- Added support for partial assembly on NURBS patches, and NURBS-patch sparse
-  matrix assembly. Patch matrix assembly includes the option to use reduced
-  approximate integration rules, computed by the newly implemented non-negative
-  least-squares (NNLS) solver.
-
-- Support for parallel transfer of H1 fields using the low-order refined (LOR)
-  transfer operators in L2ProjectionGridTransfer
-
-- Added KDTree class for 2D/3D set of points, which is then utilized in the new
-  KDTreeNodalProjection class to project a function defined on an arbitrary set
-  of points onto an MFEM grid function. This functionality is demonstrated in
-  the nodal-transfer miniapp. The current implementation is serial only. Further
-  extensions can include search in arbitrary dimensional spaces.
-
-- Added support for p-refined meshes in GSLIB-FindPoints.
-
-- Device kernels can now access device-specific DOF and quadrature limits using
-  the DofQuadLimits structure, allowing increased limits when executing on CPU.
-  The limits for the runtime selected device can be accessed in host code using
-  DeviceDofQuadLimits::Get(). The global constants MAX_D1D and MAX_Q1D are no
-  longer available.
-
-- Face restriction operators for Nedelec and Raviart-Thomas finite element
-  spaces are now supported through the ConformingFaceRestriction class.
-
-- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
-
-Linear and nonlinear solvers
-----------------------------
-- Updated the MUMPS interface to support multiple right-hand sides, block
-  low-rank compression, builds using 64-bit integers, and other improvements.
-
-- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
-  The interface provides a serial (OpenMP shared memory) version of Pardiso for
-  use with SparseMatrix. This complements the existing parallel (MPI distributed
-  memory) version already available through the CPardiso MFEM integration.
-
-- Added HIP support to the PETSc and SUNDIALS interfaces.
-
 New and updated examples and miniapps
 -------------------------------------
-- Added a new H(div) solver miniapp demonstrating the use of a matrix-free
-  saddle-point solver methodology, suitable for high-order discretizations and
-  for GPU acceleration. Examples illustrating the solution of Darcy and grad-div
-  problems are included. See miniapps/hdiv-linear-solver.
+- Added a new example code, Example 36/36p, to demonstrate the solution of
+  the obstacle problem with a new finite element method.
+
+- Added a new miniapp, Mesh Quality, for evaluating mesh quality using size,
+  skewness, and aspect-ratio computed from the Jacobian of the transformation.
+
+- Added a new miniapp for interface and boundary fitting to implicit domains
+  defined using level-set functions. See miniapps/meshing/pmesh-fitting.cpp
 
 - Added new Discontinuous Petrov-Galerkin (DPG) miniapp which includes serial
   and parallel examples for diffusion, convection-diffusion, acoustics and
   Maxwell equations. The miniapp includes new classes such as (Par)DPGWeakForm,
   (Par)ComplexDPGWeakForm and (Complex)BlockStaticCondensation. Three new
   integrators are added in support of DPG systems: TraceIntegrator,
-  NormalTraceIntegrator and TangentTraceIntegrator. See miniapps/dpg.
+  NormalTraceIntegrator and TangentTraceIntegrator.
 
-- Added a new miniapp that implements the SPDE method for generating Gaussian
-  random fields of Matern covariance. The resulting random field can be used,
-  e.g., to model material uncertainties. See miniapps/spde.
+- Added new SubMesh examples demonstrating source terms and boundary conditions
+  transferred from SubMesh objects.
 
-- Added a new parallel LOR transfer miniapp, plor-transfer, which mirrors the
-  functionality of the serial LOR transfer miniapp. See miniapps/tools.
-
-- New serial miniapp, nodal-transfer, demonstrating the use of KDTree to map a
-  parallel grid function to a different parallel partitioning of the same mesh.
-
-- Added 3 additional TMOP miniapps in miniapps/meshing:
-  * Mesh-Quality evaluates quality using size, skewness, and aspect-ratio
-    computed from the Jacobian of the transformation.
-  * Mesh-Fitting can be used for interface and boundary fitting to implicit
-    domains defined using level-set functions.
-  * Fit-Node-Position fits selected mesh nodes to specified positions, while
-    maintaining overall mesh quality.
-
-- Added 4 new example codes:
-  * Example 34/34p solves a simple magnetostatic problem where source terms and
-    boundary conditions are transferred with SubMesh objects.
-  * Example 35p implements H1, H(curl) and H(div) variants of a damped harmonic
-    oscillator with field transfer using SubMesh objects.
-  * Example 36/36p demonstrates the solution of the obstacle problem with a new
-    finite element method (proximal Galerkin).
-  * Example 37/37p demonstrates topology optimization with MFEM.
+- Added a new H(div) solvers miniapp in miniapps/hdiv-linear-solver,
+  demonstrating the use of a matrix-free saddle-point solver methodology,
+  suitable for high-order discretizations and for GPU acceleration. Examples
+  illustrating the solution of Darcy and grad-div problems are included.
 
 - Added a random refinement option to the mesh-explorer miniapp to assist users
   in experimenting with nonconforming meshes.
 
 - Moved the distance solver methods from miniapps/shifted to miniapps/common.
 
+Meshing improvements
+--------------------
+- Added new methods in the Mesh class to set and get attributes on NURBS patches
+  and patch boundaries.
+
+- Added HIP support to the SUNDIALS interface.
+
+- TMOP improvement: added asymptotically-balanced compound metrics 90, 94, 328,
+  338. Added the tmop-metric-magnitude tool for tracking how metrics change
+  under geometric perturbations.
+
+Discretization improvements
+---------------------------
+- Face restriction operators for Nedelec and Raviart-Thomas finite element
+  spaces are now supported through the ConformingFaceRestriction class.
+
+- SubMesh and ParSubMesh have been extended to support the transfer of
+  Nedelec and Raviart-Thomas finite element spaces.
+
+- VectorFEBoundaryFluxLFIntegrator is now supported on device/GPU.
+
+- Added support for p-refined meshes in FindPointsGSLIB.
+
+Linear and nonlinear solvers
+----------------------------
+- Updated interface to MUMPS direct solver to support multiple right-hand
+  sides, block low-rank compression, builds using 64-bit integers, and other
+  improvements.
+
+- Added an interface to the MKL Pardiso sparse direct solver developed by Intel.
+  This interface provides a serial (OpenMP shared memory) version of Pardiso for
+  use with SparseMatrix. This complements the existing parallel (MPI distributed
+  memory) version already available through the CPardiso MFEM integration.
+
+Integrations, testing and documentation
+---------------------------------------
+- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
+  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
+
 Miscellaneous
 -------------
 - Improved lambda body debugging with the addition of mfem::forall functions.
   These functions can take the place of the MFEM_FORALL macros, which have been
   preserved for backwards compatibility.
 
-- Added an address sanitizer GitHub action for a serial build/test on Ubuntu,
-  based on Clang/LLVM (https://clang.llvm.org/docs/AddressSanitizer.html).
-
 - Reorganized files for bilinear form, linear form, and nonlinear form integrators
   in the fem/integ/ subdirectory.
 
-- FiniteElementSpace::GetFE has been updated to abort instead of returning NULL for
-  an empty partition.
-
-- Various other simplifications, extensions, and bugfixes in the code.
-
 
 Version 4.5.2, released on March 23, 2023
 =========================================

CMakeLists.txt

@@ -57,7 +57,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.6.1)
+set(${PROJECT_NAME}_VERSION 4.5.3)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -138,7 +138,7 @@ if (MFEM_USE_CUDA)
       set(CUDA_FLAGS "-ccbin=${CMAKE_CXX_COMPILER} ${CUDA_FLAGS}")
       set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER})
    endif()
-   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS}")
+   set(CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} ${CUDA_FLAGS})
    set(CUSPARSE_FOUND TRUE)
    set(CUSPARSE_LIBRARIES "cusparse")
    set(CUBLAS_FOUND TRUE)

CONTRIBUTING.md

@@ -135,7 +135,6 @@ The MFEM source code has the following structure:
   │   ├── adjoint
   │   ├── autodiff
   │   ├── common
-  │   ├── dpg
   │   ├── electromagnetics
   │   ├── gslib
   │   ├── hdiv-linear-solver
@@ -149,7 +148,6 @@ The MFEM source code has the following structure:
   │   ├── performance
   │   ├── shifted
   │   ├── solvers
-  │   ├── spde
   │   ├── tools
   │   └── toys
   └── tests

INSTALL

@@ -699,15 +699,12 @@ The specific libraries and their options are:
   PETSc has been cloned on the same level as mfem and hypre:
     ./configure --download-fblaslapack=yes --download-scalapack=yes \
                 --download-mumps=yes --download-suitesparse=yes \
-                --with-hypre-dir=../hypre/src/hypre \
+                --with-hypre-dir=../hypre-2.10.0b/src/hypre \
                 --with-shared-libraries=0
-  When building PETSc with HIP, one may need to add a flag like -std=c2x to
-  CFLAGS to allow proper parsing of the hipsparse header under C.
   URL: https://www.mcs.anl.gov/petsc
   Options: PETSC_OPT, PETSC_LIB.
-  Versions: PETSc >= 3.8.0   (PETSc build without CUDA/HIP)
+  Versions: PETSc >= 3.8.0   (PETSc build without CUDA)
             PETSc >= 3.15.0  (PETSc built with CUDA)
-            PETSc >= 3.19.0  (PETSc built with HIP, older versions may work too)
 
 - SLEPc (optional), used when MFEM_USE_SLEPC = YES. SLEPc depends on PETSc and
   uses some of the PETSc options when compiled.

config/docker/Dockerfile

@@ -19,7 +19,9 @@ RUN apt-get update && \
     apt-get install -y libcurl4-openssl-dev libssl-dev
 
 ENV PATH=$PATH:/opt/mfem-view/bin
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/mfem-view/lib:/opt/mfem-view/lib64
 ENV DEBIAN_FRONTEND=noninteractive
 
 # The user will see the view on shell into the container
 WORKDIR /opt/mfem-view
+ENTRYPOINT ["/bin/bash"]

config/docker/Dockerfile.base

@@ -34,14 +34,14 @@ RUN cd /opt/mfem-env && \
     . /opt/spack/share/spack/setup-env.sh && \
     spack env activate . && \
     spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps && \
-    spack install
+    spack add mfem@master+examples+miniapps # && \
+#    spack install
 
 # ensure mfem always on various paths
-RUN cd /opt/mfem-env && \
-    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
+#RUN cd /opt/mfem-env && \
+#    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
 
 # Present the software install when we shell in
 # The view is at /opt/mfem-env/.spack-env/view
-WORKDIR /opt/software
-ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]
+#WORKDIR /opt/software
+#ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]

config/docker/README.md

@@ -7,31 +7,21 @@ You can use this image for a demo of using mfem! 🎉️
 Updated containers are built and deployed on merges to the main branch and releases.
 If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
 
-## Usage
+### Usage
 
-We provide two containers, which you can either build or use directly from
-[GitHub packages](https://github.com/orgs/mfem/packages?repo_name=mfem).
-
- - `ghcr.io/mfem/mfem-ubuntu-base`: a "build from scratch" for mfem
- - `ghcr.io/mfem/mfem-ubuntu`: a quick build that uses the base container
-
-In the above, "ghcr.io" means "GitHub Container Registry" and
+Here is how to build the container. Note that we build so it belongs to the same
+namespace as the repository here. "ghcr.io" means "GitHub Container Registry" and
 is the [GitHub packages](https://github.com/features/packages) registry that supports
-Docker images and other OCI artifacts. 
-
-### Ubuntu
-
-> Use or build this container for a multi-stage, slimmer base to develop on top of mfem
-
-Note that this container is provided on GitHub packages [here](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu)
-so you don't need to build it. However, if you want to, you can do the following:
+ Docker images and other OCI artifacts. From the root of the repository:
 
 ```bash
 $ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
+$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
 ```
 
-Note that this will pull the base image. If you want to rebuild it, see [ubuntu base](#ubuntu-base)
-below. Once you have built (or prefer to pull) you can shell into the container as follows:
+### Shell Ubuntu
+
+To shell into the container:
 
 ```bash
 $ docker run -it ghcr.io/mfem/mfem-ubuntu
@@ -47,13 +37,39 @@ bin  etc  include  lib  libexec  sbin  share  var
  - Examples are in share/mfem/examples
  - Examples are in share/mfem/miniapps
 
-Using this container, if you want to develop a tool that _uses_ mfem, you can find the libraries / includes in:
+You can read more about interaction with these examples and miniapps below.
+
+### Shell Ubuntu Base
+
+To shell into the container:
 
 ```bash
-$ ls include/ | grep mfem
-mfem
-mfem-performance.hpp
-mfem.hpp
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
+```
+
+Off the bat, you can see mfem libraries are in your path so you can jump into development:
+
+```bash
+env | grep mfem
+```
+```bash
+PKG_CONFIG_PATH=/opt/mfem-env/.spack-env/view/lib/pkgconfig:/opt/mfem-env/.spack-env/view/share/pkgconfig:/opt/mfem-env/.spack-env/view/lib64/pkgconfig
+PWD=/opt/mfem-env
+MANPATH=/opt/mfem-env/.spack-env/view/share/man:/opt/mfem-env/.spack-env/view/man:
+CMAKE_PREFIX_PATH=/opt/mfem-env/.spack-env/view
+SPACK_ENV=/opt/mfem-env
+ACLOCAL_PATH=/opt/mfem-env/.spack-env/view/share/aclocal
+LD_LIBRARY_PATH=/opt/mfem-env/.spack-env/view/lib:/opt/mfem-env/.spack-env/view/lib64
+PATH=/opt/mfem-env/.spack-env/view/bin:/opt/view/bin:/opt/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+```
+
+#### Examples and MiniApps
+
+If you want to develop a tool that _uses_ mfem, you can find the built libraries in:
+
+```
+$ ls /opt/mfem-env/.spack-env/view/
+bin  etc  include  lib  libexec  sbin  share  var
 ```
 
 And yes, this is the working directory when you shell into the container!
@@ -63,16 +79,6 @@ You can find the examples here:
 ```bash
 cd share/mfem/examples
 ```
-
-Try quickly setting the `LD_LIBRARY_PATH` so we can see the shared libraries
-we need:
-
-```bash
-export LD_LIBRARY_PATH=/opt/mfem-view/lib:$LD_LIBRARY_PATH
-```
-
-And then run:
-
 ```bash
 $ ./ex0
 Options used:
@@ -91,6 +97,7 @@ Number of unknowns: 101
 Average reduction factor = 0.140201
 ```
 
+Try running a few, and look at the associated .cpp file for the source code!
 You can also explore the "mini apps," also in share/mfem, but under miniapps.
 
 ```bash
@@ -123,87 +130,18 @@ Rule:
 Applying rule...done.
 ```
 
-Have fun! As a reminder, this container is ideal for developing your own
-applications that might use mfem, or having a nice environment to test out
-examples.
+Have fun!
 
 
-### Ubuntu Base
-
-> Use this build for a development environment with spack and mfem
-
-This container is also [provided on GitHub packages](https://github.com/mfem/mfem/pkgs/container/mfem-ubuntu-base), 
-however you can build it locally too:
+#### Your own App
+If you want to develop with your own code base
+(and mfem as is in the container) you can bind to somewhere else in the container (e.g., src)
 
 ```bash
-$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
-```
-
-To shell into the container:
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
-```
-
-Change directory to the mfem environment, setup spack, and activate the environment:
-
-```bash
-source /opt/spack/share/spack/setup-env.sh
-cd /opt/mfem-env/
-spack env activate .
-```
-
-Note that this environment is installing to the view at `/opt/view`. Since the environment
-knows to install mfem from `/code` this means that you could make changes in the container (or bind
-`/code` to your container) and then update spack:
-
-```bash
-# Note that concretization takes a hot minute!
-$ spack install
-```
-
-And if you want to load mfem:
-
-```bash
-$ spack load mfem
-$ env | grep mfem
-```
-
-In this development container, you can find the examples and miniapps alongside
-mfem under `/code`:
-
-```bash
-cd /code/examples
-```
-```bash
-$ ./ex0
-```
-```console
-Options used:
-   --mesh ../data/star.mesh
-   --order 1
-Number of unknowns: 101
-   Iteration :   0  (B r, r) = 0.184259
-   Iteration :   1  (B r, r) = 0.102754
-   Iteration :   2  (B r, r) = 0.00558141
-   Iteration :   3  (B r, r) = 1.5247e-05
-   Iteration :   4  (B r, r) = 1.13807e-07
-   Iteration :   5  (B r, r) = 6.27231e-09
-   Iteration :   6  (B r, r) = 3.76268e-11
-   Iteration :   7  (B r, r) = 6.07423e-13
-   Iteration :   8  (B r, r) = 4.10615e-15
-Average reduction factor = 0.140201
-```
-
-This container is likely ideal for someone that wants to develop mfem itself.
-For other use cases, we recommend using the slimmer image. As an example,
-if you want to develop with your own code base (and mfem as is in the container) 
-you can bind to somewhere else in the container (e.g., src)
-
-```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/code bash
+$ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
 ```
 
 In the above, we can pretend your project is in the present working directory (PWD) and we are
 binding to source. You can then use the mfem in the container for development, and if you
 want to distribute your library or app in a container, you can use the mfem container as the base.
+

data/diag-segment-2d.mesh

@@ -1,37 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-1
-
-elements
-4
-1 1 0 1
-1 1 1 2
-1 1 2 3
-1 1 3 4
-
-boundary
-2
-1 0 0
-2 0 4
-
-vertices
-5
-2
-0 0
-0.25 0.25
-0.50 0.50
-0.75 0.75
-1 1

data/diag-segment-3d.mesh

@@ -1,37 +0,0 @@
-MFEM mesh v1.0
-
-#
-# MFEM Geometry Types (see mesh/geom.hpp):
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-#
-
-dimension
-1
-
-elements
-4
-1 1 0 1
-1 1 1 2
-1 1 2 3
-1 1 3 4
-
-boundary
-2
-1 0 0
-2 0 4
-
-vertices
-5
-3
-0 0 0
-0.25 0.25 0.25
-0.50 0.50 0.50
-0.75 0.75 0.75
-1 1 1

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.6.1
+PROJECT_NUMBER         = v4.5.3
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/CodeDocumentation.dox

@@ -105,13 +105,8 @@ namespace mfem {
  * - <a class="el" href="ex32p_8cpp_source.html">Example 32p</a>: parallel anisotropic Maxwell eigensolver
  * - <a class="el" href="ex33_8cpp_source.html">Example 33</a>: nodal H1 FEM for the fractional Laplacian problem
  * - <a class="el" href="ex33p_8cpp_source.html">Example 33p</a>: parallel nodal H1 FEM for the fractional Laplacian problem
- * - <a class="el" href="ex34_8cpp_source.html">Example 34</a>: multi-domain magnetostatics
- * - <a class="el" href="ex34p_8cpp_source.html">Example 34p</a>: parallel multi-domain magnetostatics
- * - <a class="el" href="ex35p_8cpp_source.html">Example 35p</a>: parallel multi-domain damped harmonic oscillators
  * - <a class="el" href="ex36_8cpp_source.html">Example 36</a>: Proximal Galerkin FEM for the obstacle problem
  * - <a class="el" href="ex36p_8cpp_source.html">Example 36p</a>: parallel Proximal Galerkin FEM for the obstacle problem
- * - <a class="el" href="ex37_8cpp_source.html">Example 37</a>: Topology optimization
- * - <a class="el" href="ex37p_8cpp_source.html">Example 37p</a>: parallel topology optimization
  *
  * <H4>AmgX Examples</H4>
  * - Variants of Examples

examples/CMakeLists.txt

@@ -42,7 +42,6 @@ list(APPEND ALL_EXE_SRCS
   ex33.cpp
   ex34.cpp
   ex36.cpp
-  ex37.cpp
   )
 
 if (MFEM_USE_MPI)
@@ -83,7 +82,6 @@ if (MFEM_USE_MPI)
     ex34p.cpp
     ex35p.cpp
     ex36p.cpp
-    ex37p.cpp
     )
 endif()
 
@@ -109,8 +107,6 @@ if (MFEM_ENABLE_TESTING)
       list(APPEND THIS_TEST_OPTIONS "-e" "1")
     elseif(${TEST_NAME} MATCHES "ex27p*")
       list(APPEND THIS_TEST_OPTIONS "-dg")
-    elseif(${TEST_NAME} MATCHES "ex37p*")
-      list(APPEND THIS_TEST_OPTIONS "-mi" "3")
     endif()
 
     if (NOT (${TEST_NAME} MATCHES ".*p$"))

examples/contact/IPsolver.cpp

@@ -0,0 +1,907 @@
+#include "mfem.hpp"
+#include "IPsolver.hpp"
+#include "problems.hpp"
+#include <fstream>
+#include <iostream>
+#include <cstdlib>
+
+using namespace std;
+using namespace mfem;
+
+
+
+InteriorPointSolver::InteriorPointSolver(OptProblem * Problem, ParFiniteElementSpace *Vhin) 
+: problem(Problem), block_offsetsumlz(5), block_offsetsuml(4), block_offsetsx(3),
+ saveLogBarrierIterates(false), Vh(Vhin)
+{
+  tol  = 1.e-2;
+  max_iter = 20;
+  mu_k     = 1.0;
+
+  sMax     = 1.e2;
+  kSig     = 1.e10;   // control deviation from primal Hessian
+  tauMin   = 0.8;     // control rate at which iterates can approach the boundary
+  eta      = 1.e-4;   // backtracking constant
+  thetaMin = 1.e-4;   // allowed violation of the equality constraints
+
+  // constants in line-step A-5.4
+  delta    = 1.0;
+  sTheta   = 1.1;
+  sPhi     = 2.3;
+
+  // control the rate at which the penalty parameter is decreased
+  kMu     = 0.2;
+  thetaMu = 1.5;
+
+
+  thetaMax = 1.e6; // maximum constraint violation
+  // data for the second order correction
+  kSoc     = 0.99;
+
+  // equation (18)
+  gTheta = 1.e-5;
+  gPhi   = 1.e-5;
+
+  kEps   = 1.e1;
+
+  dimU = problem->GetDimU();
+  dimM = problem->GetDimM();
+  dimC = problem->GetDimC();
+  ckSoc.SetSize(dimC);
+  
+  block_offsetsumlz[0] = 0;
+  block_offsetsumlz[1] = dimU; // u
+  block_offsetsumlz[2] = dimM; // m
+  block_offsetsumlz[3] = dimC; // lambda
+  block_offsetsumlz[4] = dimM; // zl
+  block_offsetsumlz.PartialSum();
+
+  for(int i = 0; i < block_offsetsuml.Size(); i++)  { block_offsetsuml[i] = block_offsetsumlz[i]; }
+  for(int i = 0; i < block_offsetsx.Size(); i++)    { block_offsetsx[i]   = block_offsetsuml[i] ; }
+
+  // lower-bound for the inequality constraint m >= ml
+  ml = problem->Getml();
+  
+  lk.SetSize(dimC);   lk  = 0.0;
+  zlk.SetSize(dimM); zlk = 0.0;
+  mf.SetSize(dimM);   mf = 0.0;
+
+  linSolver = 0;
+  MyRank = 0;
+  iAmRoot = MyRank == 0 ? true : false;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xl, Vector &xhat, double tau)
+{
+  double alphaMaxloc = 1.0;
+  double alphaTmp;
+  for(int i = 0; i < x.Size(); i++)
+  {
+    if( xhat(i) < 0. )
+    {
+      alphaTmp = -1. * tau * (x(i) - xl(i)) / xhat(i);
+      alphaMaxloc = min(alphaMaxloc, alphaTmp);
+    } 
+  }
+
+  // alphaMaxloc is the local maximum step size which is
+  // distinct on each MPI process. Need to compute
+  // the global maximum step size 
+  double alphaMaxglb;
+  alphaMaxglb = alphaMaxloc;
+  return alphaMaxglb;
+}
+
+double InteriorPointSolver::MaxStepSize(Vector &x, Vector &xhat, double tau)
+{
+  Vector zero(x.Size()); zero = 0.0;
+  return MaxStepSize(x, zero, xhat, tau);
+}
+
+
+void InteriorPointSolver::Mult(const Vector &x0, Vector &xf)
+{
+  BlockVector x0block(block_offsetsx); x0block = 0.0;
+  x0block.GetBlock(0).Set(1.0, x0);
+  // hard coded initialization :(
+  x0block.GetBlock(1) = 1.0;
+  x0block.GetBlock(1).Add(1.0, ml);
+  BlockVector xfblock(block_offsetsx); xfblock = 0.0;
+  Mult(x0block, xfblock);
+  xf.Set(1.0, xfblock.GetBlock(0));
+  mf.Set(1.0, xfblock.GetBlock(1));
+}
+
+void InteriorPointSolver::Mult(const BlockVector &x0, BlockVector &xf)
+{
+  converged = false;
+  IPNewtonKrylovIters.open("IPNewtonKrylovIters.dat", ios::out | ios::trunc);  
+  BlockVector xk(block_offsetsx), xhat(block_offsetsx); xk = 0; xhat = 0.0;
+  BlockVector Xk(block_offsetsumlz), Xhat(block_offsetsumlz); Xk = 0.0; Xhat = 0.0;
+  BlockVector Xhatuml(block_offsetsuml); Xhatuml = 0.0;
+  Vector zlhat(dimM); zlhat = 0.0;
+
+  xk.GetBlock(0).Set(1.0, x0.GetBlock(0));
+  xk.GetBlock(1).Set(1.0, x0.GetBlock(1));
+  // running estimate of the final values of the Lagrange multipliers
+  lk  = 0.0;
+  zlk = 0.0;
+
+  for(int i = 0; i < dimM; i++)
+  {
+    zlk(i) = 1.e1 * mu_k / (xk(i+dimU) - ml(i));
+  }
+
+  Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+  Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+  Xk.GetBlock(2).Set(1.0, lk);
+  Xk.GetBlock(3).Set(1.0, zlk);
+
+  /* set theta0 = theta(x0)
+   *     thetaMin
+   *     thetaMax
+   * when theta(xk) < thetaMin and the switching condition holds
+   * then we ask for the Armijo sufficient decrease of the barrier
+   * objective to be satisfied, in order to accept the trial step length alphakl
+   * 
+   * thetaMax controls how the filter is initialized for each log-barrier subproblem
+   * F0 = {(th, phi) s.t. th > thetaMax}
+   * that is the filter does not allow for iterates where the constraint violation
+   * is larger than that of thetaMax
+   */
+  double theta0 = theta(xk);
+  thetaMin = 1.e-4 * max(1.0, theta0);
+  thetaMax = 1.e8  * thetaMin;
+
+  double Eeval, maxBarrierSolves, Eevalmu0;
+  bool printOptimalityError; // control optimality error print to console for log-barrier subproblems
+  
+  maxBarrierSolves = 10;
+
+  for(jOpt = 0; jOpt < max_iter; jOpt++)
+  {
+    if(iAmRoot)
+    {
+      cout << "interior-point solve step " << jOpt << endl;
+    }
+    // A-2. Check convergence of overall optimization problem
+    printOptimalityError = false;
+    Eevalmu0 = E(xk, lk, zlk, printOptimalityError);
+    if(Eevalmu0 < tol)
+    {
+      converged = true;
+      if(iAmRoot)
+      {
+	      IPNewtonKrylovIters.close();
+        cout << "solved optimization problem :)\n";
+      }
+      break;
+    }
+    
+    if(jOpt > 0) { maxBarrierSolves = 1; }
+    
+    for(int i = 0; i < maxBarrierSolves; i++)
+    {
+      // A-3. Check convergence of the barrier subproblem
+      printOptimalityError = true;
+      Eeval = E(xk, lk, zlk, mu_k, printOptimalityError);
+      if(Eeval < kEps * mu_k)
+      {
+        if(iAmRoot)
+        {
+          cout << "solved barrier subproblem :), for mu = " << mu_k << endl;
+        }
+        // A-3.1. Recompute the barrier parameter
+        mu_k  = max(tol / 10., min(kMu * mu_k, pow(mu_k, thetaMu)));
+        // A-3.2. Re-initialize the filter
+        F1.DeleteAll();
+        F2.DeleteAll();
+      }
+      else
+      {
+        break;
+      }
+    }
+    
+    // A-4. Compute the search direction
+    // solve for (uhat, mhat, lhat)
+    if(iAmRoot)
+    {
+      cout << "\n** A-4. IP-Newton solve **\n";
+    }
+    zlhat = 0.0; Xhatuml = 0.0;
+    // why do we have Xhatuml ....???
+    // TO DO: remove Xhatuml in favor of passing Xhat
+    IPNewtonSolve(xk, lk, zlk, zlhat, Xhatuml, mu_k, false); 
+
+
+    // assign data stack, X = (u, m, l, zl)
+    Xk = 0.0;
+    Xk.GetBlock(0).Set(1.0, xk.GetBlock(0));
+    Xk.GetBlock(1).Set(1.0, xk.GetBlock(1));
+    Xk.GetBlock(2).Set(1.0, lk);
+    Xk.GetBlock(3).Set(1.0, zlk);
+
+    // assign data stack, Xhat = (uhat, mhat, lhat, zlhat)
+    Xhat = 0.0;
+    for(int i = 0; i < 3; i++)
+    {
+      Xhat.GetBlock(i).Set(1.0, Xhatuml.GetBlock(i));
+    }
+    Xhat.GetBlock(3).Set(1.0, zlhat);
+
+
+    // A-5. Backtracking line search.
+    if(iAmRoot)
+    {
+      cout << "\n** A-5. Linesearch **\n";
+      cout << "mu = " << mu_k << endl;
+    }
+    lineSearch(Xk, Xhat, mu_k);
+
+    if(lineSearchSuccess)
+    {
+      if(iAmRoot)
+      {
+        cout << "lineSearch successful :)\n";
+      }
+      if(!switchCondition || !sufficientDecrease)
+      {
+        F1.Append( (1. - gTheta) * thx0);
+        F2.Append( phx0 - gPhi * thx0);
+      }
+      // ----- A-6: Accept the trial point
+      // print info regarding zl...
+      xk.GetBlock(0).Add(alpha, Xhat.GetBlock(0));
+      xk.GetBlock(1).Add(alpha, Xhat.GetBlock(1));
+      lk.Add(alpha,   Xhat.GetBlock(2));
+      zlk.Add(alphaz, Xhat.GetBlock(3));
+      projectZ(xk, zlk, mu_k);
+    }
+    else
+    {
+      if(iAmRoot)
+      {
+        cout << "lineSearch not successful :(\n";
+        cout << "attempting feasibility restoration with theta = " << thx0 << endl;
+        cout << "no feasibility restoration implemented, exiting now \n";
+      }
+      break;
+      //cout << "feasibility restoration!!! :( :( :(\n";
+      //problem->feasibilityRestoration(x, 1.e-12);
+      //      break;
+    }
+    //
+    if(jOpt + 1 == max_iter && iAmRoot) 
+    { 
+      cout << "maximum optimization iterations :(\n";
+      IPNewtonKrylovIters.close();
+    }
+  }
+  // done with optimization routine, just reassign data to xf reference so
+  // that the application code has access to the optimal point
+  xf = 0.0;
+  xf.GetBlock(0).Set(1.0, xk.GetBlock(0));
+  xf.GetBlock(1).Set(1.0, xk.GetBlock(1));
+}
+
+void InteriorPointSolver::FormIPNewtonMat(BlockVector & x, Vector & l, Vector &zl, BlockOperator &Ak)
+{
+  // WARNING: Huu, Hum, Hmu, Hmm should all be Hessian terms of the Lagrangian, currently we 
+  //          them by Hessian terms of the objective function and neglect the Hessian of l^T c
+
+  Huu = problem->Duuf(x); Hum = problem->Dumf(x);
+  Hmu = problem->Dmuf(x); Hmm = problem->Dmmf(x);
+
+  Vector DiagLogBar(dimM); DiagLogBar = 0.0;
+  for(int ii = 0; ii < dimM; ii++)
+  {
+    DiagLogBar(ii) = zl(ii) / (x(ii+dimU) - ml(ii));
+  }
+  if(saveLogBarrierIterates)
+  {
+    std::ofstream diagStream;
+    char diagString[100];
+    snprintf(diagString, 100, "logBarrierHessiandata/D%d.dat", jOpt);
+    diagStream.open(diagString, ios::out | ios::trunc);
+    for(int ii = 0; ii < dimM; ii++)
+    {
+      diagStream << setprecision(30) << DiagLogBar(ii) << endl;
+    }
+    diagStream.close();
+  } 
+
+  delete Wmm;
+  if(Hmm != nullptr)
+  {
+    SparseMatrix * D = new SparseMatrix(DiagLogBar);
+    Wmm = Add(*Hmm, *D);
+    delete D;
+  }
+  else
+  {
+    Wmm = new SparseMatrix(DiagLogBar);
+  }
+
+  delete JuT;
+  delete JmT;
+  Ju = problem->Duc(x); JuT = Transpose(*Ju);
+  Jm = problem->Dmc(x); JmT = Transpose(*Jm);
+  
+  //         IP-Newton system matrix
+  //    Ak = [[H_(u,u)  H_(u,m)   J_u^T]
+  //          [H_(m,u)  W_(m,m)   J_m^T]
+  //          [ J_u      J_m       0  ]]
+
+  Ak.SetBlock(0, 0, Huu);                         Ak.SetBlock(0, 2, JuT);
+                          Ak.SetBlock(1, 1, Wmm); Ak.SetBlock(1, 2, JmT);
+  Ak.SetBlock(2, 0,  Ju); Ak.SetBlock(2, 1,  Jm);
+
+  if(Hum != nullptr) { Ak.SetBlock(0, 1, Hum); Ak.SetBlock(1, 0, Hmu); }
+}
+
+
+// perturbed KKT system solve
+// determine the search direction
+void InteriorPointSolver::IPNewtonSolve(BlockVector &x, Vector &l, Vector &zl, Vector &zlhat, BlockVector &Xhat, double mu, bool socSolve)
+{
+  // solve A x = b, where A is the IP-Newton matrix
+  BlockOperator A(block_offsetsuml, block_offsetsuml); BlockVector b(block_offsetsuml); b = 0.0;
+  FormIPNewtonMat(x, l, zl, A);
+
+  //       [grad_u phi + Ju^T l]
+  // b = - [grad_m phi + Jm^T l]
+  //       [          c        ]
+  BlockVector gradphi(block_offsetsx); gradphi = 0.0;
+  BlockVector JTl(block_offsetsx); JTl = 0.0;
+  Dxphi(x, mu, gradphi);
+  
+  (A.GetBlock(0,2)).Mult(l, JTl.GetBlock(0));
+  (A.GetBlock(1,2)).Mult(l, JTl.GetBlock(1));
+
+  for(int ii = 0; ii < 2; ii++)
+  {
+    b.GetBlock(ii).Set(1.0, gradphi.GetBlock(ii));
+    b.GetBlock(ii).Add(1.0, JTl.GetBlock(ii));
+  }
+  if(!socSolve) 
+  {
+    problem->c(x, b.GetBlock(2));
+  }
+  else
+  {
+    b.GetBlock(2).Set(1.0, ckSoc);
+  }
+  b *= -1.0; 
+  Xhat = 0.0;
+
+  #ifdef MFEM_USE_SUITESPARSE
+    // Direct solve for IP-Newton saddle-point system
+    // A = [ [ Huu   0    Ju^T]
+    //       [  0    D     -I ]
+    //       [ Ju   -I      0 ]]
+    if(linSolver == 0)
+    {
+      BlockMatrix ABlockMatrix(block_offsetsuml, block_offsetsuml);
+      for(int ii = 0; ii < 3; ii++)
+      {
+        for(int jj = 0; jj < 3; jj++)
+        {
+          if(!A.IsZeroBlock(ii, jj))
+          {
+            ABlockMatrix.SetBlock(ii, jj, dynamic_cast<SparseMatrix *>(&(A.GetBlock(ii, jj))));
+          }
+        }
+      }
+      /* direct solve of the 3x3 IP-Newton linear system */
+      UMFPackSolver ASolver;
+      SparseMatrix *ASparse = ABlockMatrix.CreateMonolithic();
+      ASolver.SetOperator(*ASparse);
+      ASolver.Mult(b, Xhat);
+
+      Vector residual(Xhat.Size());
+      ASparse->Mult(Xhat, residual);
+      residual.Add(-1.0, b);
+      delete ASparse;
+    }
+    else if(linSolver == 1)
+    {
+      // Direct solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+      // where Wmm = D for contact problems
+      SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+      SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+      SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+      SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+      Vector Dvec(dimM); Dvec = 0.0;
+      Vector one(dimM); one = 1.0;
+      Wmmloc->Mult(one, Dvec);
+      SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+      SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+      
+      /* prepare the reduced rhs */
+      // breduced = bu + Ju^T (bm + Wmm bl)
+      Vector breduced(dimU); breduced = 0.0;
+      Vector tempVec(dimM); tempVec = 0.0;
+      Wmmloc->Mult(b.GetBlock(2), tempVec);
+      tempVec.Add(1.0, b.GetBlock(1));
+      JuTloc->Mult(tempVec, breduced);
+      breduced.Add(1.0, b.GetBlock(0));
+
+      // solve the reduced linear system
+      UMFPackSolver AreducedSolver;
+      AreducedSolver.SetOperator(*Areduced);
+      AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+
+      // now propagate solved uhat to obtain mhat and lhat
+      // xm = Ju xu - bl
+      Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+      Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+      // xl = Wmm xm - bm
+      Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+      Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+      delete Wmmloc;
+      delete Huuloc;
+      delete JuTDJu;
+      delete Juloc;
+      delete Areduced;
+    }
+  #else
+    MFEM_VERIFY(linSolver > 1, "linSolver = 0, 1 require MFEM_USE_SUITESPARSE=YES");
+  #endif
+  if (linSolver == 2 || linSolver == 3)
+  { 
+    // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+    // where Wmm = D for contact problems
+    // here the iterative solver is a Jacobi-preconditioned CG-solve
+    SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+    SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+    SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+    SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+    // Vector Dvec(dimM); Dvec = 0.0;
+    // Vector one(dimM); one = 1.0;
+    // Wmmloc->Mult(one, Dvec);
+
+    // SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+    SparseMatrix *JuTDJu   = RAP(*Juloc,*Wmmloc,*Juloc);     // Ju^T D Ju
+    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+    /* prepare the reduced rhs */
+    // breduced = bu + Ju^T (bm + Wmm bl)
+    Vector breduced(dimU); breduced = 0.0;
+    Vector tempVec(dimM); tempVec = 0.0;
+    Wmmloc->Mult(b.GetBlock(2), tempVec);
+    tempVec.Add(1.0, b.GetBlock(1));
+    JuTloc->Mult(tempVec, breduced);
+    breduced.Add(1.0, b.GetBlock(0));
+    
+    /* set up an iterative solver */
+    int globalNumRows = dimU;
+    HYPRE_BigInt rowStarts[2];
+    rowStarts[0] = 0;
+    rowStarts[1] = dimU;
+    HypreParMatrix * Ahypre = new HypreParMatrix(MPI_COMM_WORLD, globalNumRows, rowStarts, Areduced);
+    // CGSolver Asolver(MPI_COMM_WORLD);
+    HyprePCG Asolver(MPI_COMM_WORLD);
+    HypreBoomerAMG * Aprec = new HypreBoomerAMG(*Ahypre);
+    Aprec->SetPrintLevel(0);
+    if(linSolver == 3)
+    {
+      Aprec->SetElasticityOptions(Vh);
+    }
+    Aprec->SetSystemsOptions(3,false);
+
+    Asolver.SetOperator(*Ahypre);
+    Asolver.SetPrintLevel(2);
+    Asolver.SetMaxIter(1000);
+    // Asolver.SetResidualConvergenceOptions();
+    Asolver.SetTol(1.e-6);
+    Asolver.SetPreconditioner(*Aprec);
+    // Asolver.SetResidualConvergenceOptions();
+
+    Asolver.Mult(breduced, Xhat.GetBlock(0));
+    int num_iterations;
+    Asolver.GetNumIterations(num_iterations);
+    cgnum_iterations.Append(num_iterations);
+    // int numNewtonKrylovIters = -1;
+    // numNewtonKrylovIters = Asolver.GetNumIterations();
+    // IPNewtonKrylovIters << numNewtonKrylovIters << endl;
+
+    delete Aprec;
+    delete Ahypre;
+    
+    // now propagate solved uhat to obtain mhat and lhat
+    // xm = Ju xu - bl
+    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+    // // xl = Wmm xm - bm
+    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+    
+
+    delete Wmmloc;
+    delete Huuloc;
+    delete JuTDJu;
+    delete Juloc;
+    delete Areduced;
+  }
+  else if(linSolver > 2)
+  { 
+    // Iterative solve for 0,0 Schur complement of IP-Newton system, Huu + Ju^T Wmm Ju,
+    // where Wmm = D for contact problems
+    // here the iterative solver is a Jacobi-preconditioned CG-solve
+    SparseMatrix * Huuloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 0))));
+    SparseMatrix * Wmmloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(1, 1))));
+    SparseMatrix * Juloc  = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(2, 0))));
+    SparseMatrix * JuTloc = new SparseMatrix(*dynamic_cast<SparseMatrix *>(&(A.GetBlock(0, 2))));
+    Vector Dvec(dimM); Dvec = 0.0;
+    Vector one(dimM); one = 1.0;
+    Wmmloc->Mult(one, Dvec);
+    SparseMatrix *JuTDJu = Mult_AtDA(*Juloc, Dvec);     // Ju^T D Ju
+    SparseMatrix *Areduced = Add(*Huuloc, *JuTDJu);  // Huu + Ju^T D Ju
+
+    /* prepare the reduced rhs */
+    // breduced = bu + Ju^T (bm + Wmm bl)
+    Vector breduced(dimU); breduced = 0.0;
+    Vector tempVec(dimM); tempVec = 0.0;
+    Wmmloc->Mult(b.GetBlock(2), tempVec);
+    tempVec.Add(1.0, b.GetBlock(1));
+    JuTloc->Mult(tempVec, breduced);
+    breduced.Add(1.0, b.GetBlock(0));
+    
+    /* set up an iterative solver */
+    GSSmoother AreducedPrec((SparseMatrix &)(*Areduced));
+    GMRESSolver AreducedSolver;
+    AreducedSolver.SetOperator(*Areduced);
+    AreducedSolver.SetAbsTol(1.e-12);
+    AreducedSolver.SetRelTol(1.e-8);
+    AreducedSolver.SetMaxIter(500);
+    AreducedSolver.SetPreconditioner(AreducedPrec);
+    AreducedSolver.SetPrintLevel(1);
+    AreducedSolver.Mult(breduced, Xhat.GetBlock(0));
+    
+    // now propagate solved uhat to obtain mhat and lhat
+    // xm = Ju xu - bl
+    Juloc->Mult(Xhat.GetBlock(0), Xhat.GetBlock(1));
+    Xhat.GetBlock(1).Add(-1.0, b.GetBlock(2));
+
+    // xl = Wmm xm - bm
+    Wmmloc->Mult(Xhat.GetBlock(1), Xhat.GetBlock(2));
+    Xhat.GetBlock(2).Add(-1.0, b.GetBlock(1));
+
+    delete Wmmloc;
+    delete Huuloc;
+    delete JuTDJu;
+    delete Juloc;
+    delete Areduced;
+  }
+    
+  
+  /* backsolve to determine zlhat */
+  for(int ii = 0; ii < dimM; ii++)
+  {
+    zlhat(ii) = -1.*(zl(ii) + (zl(ii) * Xhat(ii + dimU) - mu) / (x(ii + dimU) - ml(ii)) );
+  }
+}
+
+// here Xhat, X will be BlockVectors w.r.t. the 4 partitioning X = (u, m, l, zl)
+
+void InteriorPointSolver::lineSearch(BlockVector& X0, BlockVector& Xhat, double mu)
+{
+  double tau  = max(tauMin, 1.0 - mu);
+  Vector u0   = X0.GetBlock(0);
+  Vector m0   = X0.GetBlock(1);
+  Vector l0   = X0.GetBlock(2);
+  Vector z0   = X0.GetBlock(3);
+  Vector uhat = Xhat.GetBlock(0);
+  Vector mhat = Xhat.GetBlock(1);
+  Vector lhat = Xhat.GetBlock(2);
+  Vector zhat = Xhat.GetBlock(3);
+  double alphaMax  = MaxStepSize(m0, ml, mhat, tau);
+  double alphaMaxz = MaxStepSize(z0, zhat, tau);
+  alphaz = alphaMaxz;
+
+
+  BlockVector x0(block_offsetsx); x0 = 0.0;
+  x0.GetBlock(0).Set(1.0, u0);
+  x0.GetBlock(1).Set(1.0, m0);
+  
+  BlockVector xhat(block_offsetsx); xhat = 0.0;
+  xhat.GetBlock(0).Set(1.0, uhat);
+  xhat.GetBlock(1).Set(1.0, mhat);
+  
+  BlockVector xtrial(block_offsetsx); xtrial = 0.0;
+  BlockVector Dxphi0(block_offsetsx); Dxphi0 = 0.0;
+  int maxBacktrack = 20;
+  alpha = alphaMax;
+
+
+  Vector ck0(dimC); ck0 = 0.0;
+  Vector zhatsoc(dimM); zhatsoc = 0.0;
+  BlockVector Xhatumlsoc(block_offsetsuml); Xhatumlsoc = 0.0;
+  BlockVector xhatsoc(block_offsetsx); xhatsoc = 0.0;
+  Vector uhatsoc(dimU); uhatsoc = 0.0;
+  Vector mhatsoc(dimM); mhatsoc = 0.0;
+
+  Dxphi(x0, mu, Dxphi0);
+  Dxphi0_xhat = InnerProduct(Dxphi0, xhat);
+  double xhat_L2norm = sqrt(InnerProduct(xhat, xhat));
+  double Dxphi_L2norm = sqrt(InnerProduct(Dxphi0, Dxphi0));
+  descentDirection = Dxphi0_xhat < 0. ? true : false;
+  if(descentDirection)
+  {
+    cout << "is a descent direction for the log-barrier objective\n";
+  }
+  else
+  {
+    cout << "is not a descent direction for the log-barrier objective\n";
+  }
+  cout << "Dxphi^T xhat / (|| Dxphi ||_2 * || xhat ||_2) = " << Dxphi0_xhat / (xhat_L2norm * Dxphi_L2norm) << endl;
+  thx0 = theta(x0);
+  phx0 = phi(x0, mu);
+
+  lineSearchSuccess = false;
+  for(int i = 0; i < maxBacktrack; i++)
+  {
+    cout << "\n--------- alpha = " << alpha << " ---------\n";
+
+    // ----- A-5.2. Compute trial point: xtrial = x0 + alpha_i xhat
+    xtrial.Set(1.0, x0);
+    xtrial.Add(alpha, xhat);
+
+    // ------ A-5.3. if not in filter region go to A.5.4 otherwise go to A-5.5.
+    thxtrial = theta(xtrial);
+    phxtrial = phi(xtrial, mu);
+    filterCheck(thxtrial, phxtrial);    
+    if(!inFilterRegion)
+    {
+      cout << "not in filter region :)\n";
+      // ------ A.5.4: Check sufficient decrease
+      if(!descentDirection)
+      {
+        switchCondition = false;
+      }
+      else
+      {
+        switchCondition = (alpha * pow(abs(Dxphi0_xhat), sPhi) > delta * pow(thx0, sTheta)) ? true : false;
+      }
+      cout << "alpha |Dxphi(x0)^T xhat|^sPhi = " << alpha * pow(abs(Dxphi0_xhat), sPhi) << endl;
+      cout << "delta * theta(x0)^sTheta = " << delta * pow(thx0, sTheta) << endl;
+      cout << "theta(x0) = "     << thx0     << ", thetaMin = "                  << thetaMin             << endl;
+      cout << "theta(xtrial) = " << thxtrial << ", (1-gTheta) *theta(x0) = "     << (1. - gTheta) * thx0 << endl;
+      cout << "phi(xtrial) = "   << phxtrial << ", phi(x0) - gPhi *theta(x0) = " << phx0 - gPhi * thx0   << endl;
+      
+      // Case I      
+      if(thx0 <= thetaMin && switchCondition)
+      {
+        sufficientDecrease = phxtrial <= phx0 + eta * alpha * Dxphi0_xhat ? true : false;
+        if(sufficientDecrease)
+        {
+          if(iAmRoot) { cout << "A-5.4. Case I -- accepted step length.\n"; } 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      else
+      {
+        if(thxtrial <= (1. - gTheta) * thx0 || phxtrial <= phx0 - gPhi * thx0)
+        {
+          if(iAmRoot) { cout << "A-5.4. Case II -- accepted step length.\n"; } 
+          // accept the trial step
+          lineSearchSuccess = true;
+          break;
+        }
+      }
+      // A-5.5: Initialize the second-order correction
+      if((!(thx0 < thxtrial)) && i == 0)
+      {
+        cout << "second order correction\n";
+        problem->c(xtrial, ckSoc);
+        problem->c(x0, ck0);
+        ckSoc.Add(alphaMax, ck0);
+        // A-5.6 Compute the second-order correction.
+        IPNewtonSolve(x0, l0, z0, zhatsoc, Xhatumlsoc, mu, true);
+        mhatsoc.Set(1.0, Xhatumlsoc.GetBlock(1));
+        // alphasoc = MaxStepSize(m0, ml, mhatsoc, tau);
+        //WARNING: not complete but currently solver isn't entering this region
+      }
+    }
+    else
+    {
+      cout << "in filter region :(\n"; 
+    }
+
+    // include more if needed
+    alpha *= 0.5;
+
+  } 
+}
+
+void InteriorPointSolver::projectZ(const Vector &x, Vector &z, double mu)
+{
+  double zi;
+  double mudivmml;
+  for(int i = 0; i < dimM; i++)
+  {
+    zi = z(i);
+    mudivmml = mu / (x(i + dimU) - ml(i));
+    z(i) = max(min(zi, kSig * mudivmml), mudivmml / kSig);
+  }
+}
+
+void InteriorPointSolver::filterCheck(double th, double ph)
+{
+  inFilterRegion = false;
+  if(th > thetaMax)
+  {
+    inFilterRegion = true;
+  }
+  else
+  {
+    for(int i = 0; i < F1.Size(); i++)
+    {
+      if(th >= F1[i] && ph >= F2[i])
+      {
+        inFilterRegion = true;
+        break;
+      }
+    }
+  }
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, double mu, bool print)
+{
+  double E1, E2, E3;
+  double sc, sd;
+  BlockVector gradL(block_offsetsx); gradL = 0.0; // stationarity grad L = grad f + J^T l - z
+  Vector cx(dimC); cx = 0.0;     // feasibility c = c(x)
+  Vector comp(dimM); comp = 0.0; // complementarity M Z - mu 1
+
+  DxL(x, l, zl, gradL);
+  E1 = gradL.Normlinf();
+
+  problem->c(x, cx);
+  E2 = cx.Normlinf();
+
+  for(int ii = 0; ii < dimM; ii++) 
+  { 
+    comp(ii) = x(dimU + ii) * zl(ii) - mu;
+  }
+  E3 = comp.Normlinf();
+
+  double ll1, zl1;
+  zl1 = zl.Norml1() / double(dimC + dimM);
+  ll1 =  l.Norml1();
+  sc = max(sMax, zl1 / (double(dimM)) ) / sMax;
+  sd = max(sMax, (ll1 + zl1) / (double(dimC + dimM))) / sMax;
+  if(iAmRoot && print)
+  {
+    cout << "evaluating optimality error for mu = " << mu << endl;
+    cout << "stationarity measure = "    << E1 / sd << endl;
+    cout << "feasibility measure  = "    << E2      << endl;
+    cout << "complimentarity measure = " << E3 / sc << endl;
+  }
+  return max(max(E1 / sd, E2), E3 / sc);
+}
+
+double InteriorPointSolver::E(const BlockVector &x, const Vector &l, const Vector &zl, bool print)
+{
+  return E(x, l, zl, 0.0, print);
+}
+
+double InteriorPointSolver::theta(const BlockVector &x)
+{
+  Vector cx(dimC); cx = 0.0;
+  problem->c(x, cx);
+  return sqrt(InnerProduct(cx, cx));
+}
+
+// log-barrier objective
+double InteriorPointSolver::phi(const BlockVector &x, double mu)
+{
+  double fx = problem->CalcObjective(x); 
+  double logBarrierLoc = 0.0;
+  for(int i = 0; i < dimM; i++) 
+  { 
+    logBarrierLoc += log(x(dimU+i)-ml(i));
+  }
+  double logBarrierGlb = 0.0;
+  logBarrierGlb = logBarrierLoc;
+  return fx - mu * logBarrierGlb;
+}
+
+
+// gradient of log-barrier objective with respect to x = (u, m)
+void InteriorPointSolver::Dxphi(const BlockVector &x, double mu, BlockVector &y)
+{
+  problem->CalcObjectiveGrad(x, y);
+  for(int i = 0; i < dimM; i++) 
+  { 
+    y(dimU + i) -= mu / (x(dimU + i) - ml(i));
+  } 
+}
+
+// Lagrangian function evaluation
+// L(x, l, zl) = f(x) + l^T c(x) - zl^T m
+double InteriorPointSolver::L(const BlockVector &x, const Vector &l, const Vector &zl)
+{
+  double fx = problem->CalcObjective(x);
+  Vector cx(dimC); problem->c(x, cx);
+  return (fx + InnerProduct(cx, l) - InnerProduct(x.GetBlock(1), zl));
+}
+
+void InteriorPointSolver::DxL(const BlockVector &x, const Vector &l, const Vector &zl, BlockVector &y)
+{
+  // evaluate the gradient of the objective with respect to the primal variables x = (u, m)
+  BlockVector gradxf(block_offsetsx); gradxf = 0.0;
+  problem->CalcObjectiveGrad(x, gradxf);
+  
+  SparseMatrix *Jacu, *Jacm, *JacuT, *JacmT;
+  Jacu = problem->Duc(x); Jacm = problem->Dmc(x);
+  JacuT = Transpose(*Jacu);
+  JacmT = Transpose(*Jacm);
+  JacuT->Mult(l, y.GetBlock(0));
+  JacmT->Mult(l, y.GetBlock(1));
+  delete Jacu; delete JacuT;
+  delete Jacm; delete JacmT;
+  y.Add(1.0, gradxf);
+  (y.GetBlock(1)).Add(-1.0, zl);
+}
+
+
+bool InteriorPointSolver::GetConverged() const
+{
+  return converged;
+}
+
+void InteriorPointSolver::SetTol(double Tol)
+{
+  tol = Tol;
+}
+
+void InteriorPointSolver::SetMaxIter(int max_it)
+{
+  max_iter = max_it;
+}
+
+void InteriorPointSolver::SetBarrierParameter(double mu_0)
+{
+  mu_k = mu_0;
+}
+
+void InteriorPointSolver::SaveLogBarrierHessianIterates(bool save)
+{
+  MFEM_ASSERT(MyRank == 0 || save == false, "currently can only save logbarrier hessian in serial codes");
+  saveLogBarrierIterates = save;
+}
+
+void InteriorPointSolver::SetLinearSolver(int LinSolver)
+{
+  linSolver = LinSolver;
+}
+
+
+
+InteriorPointSolver::~InteriorPointSolver() 
+{
+  delete Wmm;
+  delete Huu;
+  delete Hum;
+  delete Hmu;
+  delete Hmm;
+  delete Hum;
+  delete Ju;
+  delete Jm;
+  delete JuT;
+  delete JmT;
+
+  F1.DeleteAll();
+  F2.DeleteAll();
+  block_offsetsx.DeleteAll();
+  block_offsetsumlz.DeleteAll();
+  block_offsetsuml.DeleteAll();
+  ml.SetSize(0);
+}

examples/contact/IPsolver.hpp

@@ -0,0 +1,103 @@
+#include "mfem.hpp"
+#include "problems.hpp"
+#include <fstream>
+#include <iostream>
+
+using namespace std;
+using namespace mfem;
+
+
+#ifndef IPSOLVER 
+#define IPSOLVER
+
+class InteriorPointSolver
+{
+protected:
+    OptProblem* problem;
+    double tol;
+    int  max_iter;
+    double mu_k; // \mu_k
+    Vector lk, zlk, mf;
+
+    double sMax, kSig, tauMin, eta, thetaMin, delta, sTheta, sPhi, kMu, thetaMu;
+    double thetaMax, kSoc, gTheta, gPhi, kEps;
+	
+    // filter
+    Array<double> F1, F2;
+	
+    // quantities computed in lineSearch
+    double alpha, alphaz;
+    double thx0, thxtrial;
+    double phx0, phxtrial;
+    bool descentDirection, switchCondition, sufficientDecrease, lineSearchSuccess, inFilterRegion;
+    double Dxphi0_xhat;
+
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsumlz, block_offsetsuml, block_offsetsx;
+    Vector ml;
+
+    Vector ckSoc;
+    SparseMatrix * Huu = nullptr; 
+    SparseMatrix * Hum = nullptr; 
+    SparseMatrix * Hmu = nullptr; 
+    SparseMatrix * Hmm = nullptr; 
+    SparseMatrix * Wmm = nullptr; 
+    SparseMatrix * Ju  = nullptr; 
+    SparseMatrix * Jm  = nullptr; 
+    SparseMatrix * JuT = nullptr; 
+    SparseMatrix * JmT  = nullptr;;
+    
+    int jOpt;
+    bool converged;
+    
+    int MyRank;
+    bool iAmRoot;
+
+    bool saveLogBarrierIterates;
+
+    int linSolver;
+    std::ofstream IPNewtonKrylovIters;
+
+    ParFiniteElementSpace *Vh;
+    Array<int> cgnum_iterations;
+
+
+    // not sure if this data is needed or if it can 
+    // all be accounted for in the problem class
+    // which variables have equality constraints
+    //Array<int> eqConstrainedVariables;
+    //Array<double> eqConstrainedValues;
+
+
+
+public:
+    InteriorPointSolver(OptProblem*, ParFiniteElementSpace *);
+    void Mult(const BlockVector& , BlockVector&); // used when the user wants to be aware of bound-constrained variable m >= ml
+    void Mult(const Vector&, Vector &); // useful when the user doesn't need to know about bound-constrained variable   m >= ml
+    double MaxStepSize(Vector& , Vector& , Vector& , double);
+    double MaxStepSize(Vector& , Vector& , double);
+    void FormIPNewtonMat(BlockVector& , Vector& , Vector& , BlockOperator &);
+    void IPNewtonSolve(BlockVector& , Vector& , Vector& , Vector&, BlockVector& , double, bool);
+    void lineSearch(BlockVector& , BlockVector& , double);
+    void projectZ(const Vector & , Vector &, double);
+    void filterCheck(double, double);
+    double E(const BlockVector &, const Vector &, const Vector &, double, bool);
+    double E(const BlockVector &, const Vector &, const Vector &, bool);
+    bool GetConverged() const;
+    // TO DO: include Hessian of Lagrangian
+    double theta(const BlockVector &);
+    double phi(const BlockVector &, double);
+    void Dxphi(const BlockVector &, double, BlockVector &);
+    double L(const BlockVector &, const Vector &, const Vector &);
+    void DxL(const BlockVector &, const Vector &, const Vector &, BlockVector &);
+    void SetTol(double);
+    void SetMaxIter(int);
+    void SetBarrierParameter(double);    
+    void SaveLogBarrierHessianIterates(bool);
+    void SetLinearSolver(int);
+    Vector GetBoundConstrainedVariable() {return mf;}
+    Array<int> & GetCGIterNumbers() {return cgnum_iterations;}
+    virtual ~InteriorPointSolver();
+};
+
+#endif

examples/contact/README.md

@@ -0,0 +1,17 @@
+# OneProcessAMGContact
+
+
+
+Be sure to edit the makefile so that it points to a parallel MFEM build
+
+specifically the MFEM_BUILD_DIR
+
+
+after building exQPContactBlockTL one can
+
+1. run the bash script scalingJobArray.bat via `source scalingJobArray.bat' which will populate the CG iterations required to solve
+      various linear systems into the data/ subdirectory
+2. run the python script data/process.py in order to put the scaling information into the single files algorithmicScaling_Elasticity.dat and algorithmicScaling_noElasticity.dat
+      in order to see the number of average AMG-CG iterations per optimization solve. 
+
+

examples/contact/exQPContactBlockTL.cpp

@@ -0,0 +1,274 @@
+//                                Contact example
+//
+// Compile with: make contact
+//
+// Sample runs:  ./contact -m1 block1.mesh -m2 block2.mesh -at "5 6 7 8"
+// Sample runs:  ./contact -m1 block1_d.mesh -m2 block2_d.mesh -at "5 6 7 8"
+
+#include <fstream>
+#include <iostream>
+#include <array>
+
+#include "mfem.hpp"
+#include "problems.hpp"
+#include "IPsolver.hpp"
+
+using namespace std;
+using namespace mfem;
+
+
+int main(int argc, char *argv[])
+{
+  Mpi::Init(argc, argv);
+  Hypre::Init();
+  int linSolver = 2;
+  int maxIPMiters = 30;
+  bool iAmRoot = true;
+  int ref_levels = 0;
+
+  OptionsParser args(argc, argv);
+  args.AddOption(&linSolver, "-linSolver", "--linearSolver", \
+       "IP-Newton linear system solution strategy.");
+  args.AddOption(&maxIPMiters, "-IPMiters", "--IPMiters",\
+		  "Maximum number of IPM iterations");
+  args.AddOption(&ref_levels, "-r", "--mesh_refinement", \
+		  "Mesh Refinement");
+
+   
+  args.Parse();
+  if(!args.Good())
+  {
+    args.PrintUsage(cout);
+    return 1;
+  }
+  else
+  {
+    if( iAmRoot )
+    {
+      args.PrintOptions(cout);
+    }
+  }
+	
+  // Create an instance of the nlp
+  ExContactBlockTL * contact = new ExContactBlockTL(ref_levels);
+  int ndofs = contact->GetDimD();
+  int nconstraints = contact->GetDimS();
+  std::ofstream problemDimStream;
+  problemDimStream.open("problemDim.dat", ios::out | ios::trunc);
+  problemDimStream << ndofs << endl;
+  problemDimStream.close();
+  std::ofstream problemDimConstraintsStream;
+  problemDimConstraintsStream.open("problemDimConstraints.dat", ios::out | ios::trunc);
+  problemDimConstraintsStream << nconstraints << endl;
+  problemDimConstraintsStream.close();
+  
+  // set up a QP-problem
+  // E(d) = 1 / 2 d^T K d + f^T d
+  // g(d) = J d + g0
+  // where K, J, f and g0 are evaluated at d0 (a valid configuration)
+  
+  // to do: seems more appropriate to evaluate at a valid configuration...
+  //        that is one where the Dirichlet conditions hold... need to pull
+  //        this data from contactBlockTL...
+  Vector d0(ndofs); d0 = 0.0;
+  Array<int> DirichletDofs = contact->GetDirichletDofs();
+  Array<double> DirichletVals = contact->GetDirichletVals();
+  SparseMatrix *K;
+  Vector f(ndofs); f = 0.0;
+  contact->DdE(d0, f); K = contact->DddE(d0);
+  for(int i = 0; i < DirichletDofs.Size(); i++)
+  {
+    d0(DirichletDofs[i]) = DirichletVals[i];
+  }
+  SparseMatrix *J;
+  Vector g0(nconstraints); g0 = 0.0;
+  J = contact->Ddg(d0); contact->g(d0, g0);
+  Vector temp(nconstraints);
+  J->Mult(d0, temp);
+  g0.Add(-1.0, temp);
+
+  // check which rows of the Jacobian are zero!
+  Vector ei(nconstraints); ei = 0.0;
+  Vector JTei(ndofs); JTei = 0.0;
+
+  double normJTei;
+
+  int reduced_nconstraints = 0; // find actual number of constraints
+  
+  
+  Array<int> nonZeroRows;
+  for(int i = 0; i < nconstraints; i++)
+  {
+    ei(i) = 1.0;
+    J->MultTranspose(ei, JTei);
+    // nullify contributions from Dirichlet constrined dofs
+    for(int j = 0; j < DirichletDofs.Size(); j++)
+    {
+      JTei(DirichletDofs[j]) = 0.0;
+    }
+    normJTei = sqrt(InnerProduct(JTei, JTei));
+    if (normJTei > 1.e-12)
+    {
+      reduced_nconstraints += 1;
+      nonZeroRows.Append(i);
+    }
+    ei(i) = 0.0;
+  }
+  cout << "number of linearized constraints = " << reduced_nconstraints << endl; // 9 constraints 
+  
+  // remove zero rows of the gap function Jacobian and corresponding gap function entries
+  SparseMatrix * Jreduced = new SparseMatrix(reduced_nconstraints, ndofs);
+  Vector g0reduced(reduced_nconstraints); g0reduced = 0.0; 
+  
+  
+  for(int i = 0; i < reduced_nconstraints; i++)
+  {
+    Array<int> col_tmp;
+    Vector v_tmp; v_tmp = 0.0;
+    J->GetRow(nonZeroRows[i], col_tmp, v_tmp);
+    
+    /* obtain subset of columns of the given nonZero Jacobian row that are not Dirichlet constrained */
+    bool freeDof;
+    Array<int> loc_indicies;
+    for(int j = 0; j < col_tmp.Size(); j++)
+    {
+      freeDof = true;
+      for(int k = 0; k < DirichletDofs.Size(); k++)
+      {
+        if(col_tmp[j] == DirichletDofs[k])
+        {
+          freeDof = false;
+        }
+      }
+      if(freeDof)
+      {
+        loc_indicies.Append(j);
+      }
+    }
+
+    Array<int> col_tmp_reduced(loc_indicies.Size());
+    Vector v_tmp_reduced(loc_indicies.Size());
+    for(int j = 0; j < loc_indicies.Size(); j++)
+    {
+      col_tmp_reduced[j] = col_tmp[loc_indicies[j]];
+      v_tmp_reduced(j)   = v_tmp(loc_indicies[j]);
+    }
+
+    Jreduced->SetRow(i, col_tmp_reduced, v_tmp_reduced);
+    g0reduced(i) = g0(nonZeroRows[i]);
+  }
+
+
+  QPContactProblem *QPContact = new QPContactProblem(*K, *Jreduced, f, g0reduced);
+  
+  Mesh * mesh1 = new Mesh("meshes/block1.mesh", 1, 1);
+  Mesh * mesh2 = new Mesh("meshes/rotatedblock2.mesh", 1, 1);
+  for(int i = 0; i < ref_levels; i++)
+  {
+     mesh1->UniformRefinement();
+     mesh2->UniformRefinement(); 
+  }
+  
+  int numMeshes = 2;
+  Mesh *meshArray[numMeshes];
+  meshArray[0] = mesh1;
+  meshArray[1] = mesh2;
+  Mesh mesh(meshArray, numMeshes);
+  
+  ParMesh pmesh(MPI_COMM_WORLD, mesh);  
+  H1_FECollection fec(1, mesh.Dimension());
+  ParFiniteElementSpace fespace(&pmesh, &fec, mesh.Dimension(), Ordering::byVDIM);
+
+  InteriorPointSolver * QPContactOptimizer = new InteriorPointSolver(QPContact, &fespace);
+  QPContactOptimizer->SetTol(1.e-6);
+  QPContactOptimizer->SetLinearSolver(linSolver);
+  QPContactOptimizer->SetMaxIter(50);
+  Vector x0(ndofs); x0 = 0.0;
+  for(int i = 0; i < DirichletDofs.Size(); i++)
+  {
+    x0(DirichletDofs[i]) = DirichletVals[i];
+  }
+  Vector xf(ndofs); xf = 0.0;
+  QPContactOptimizer->Mult(x0, xf);
+  
+  double Einitial = QPContact->E(x0);
+  double Efinal = QPContact->E(xf);
+  cout << "Energy objective at initial point = " << Einitial << endl;
+  cout << "Energy objective at QP optimizer = " << Efinal << endl;
+  QPContactOptimizer->GetCGIterNumbers().Print(mfem::out, 20);
+  MFEM_VERIFY(QPContactOptimizer->GetConverged(), "Interior point solver did not converge.");
+  
+  
+  //Mesh * mesh1 = new Mesh("meshes/block1.mesh", 1, 1);
+  //Mesh * mesh2 = new Mesh("meshes/rotatedblock2.mesh", 1, 1);
+  //for(int i = 0; i < ref_levels; i++)
+  //{
+  //   mesh1->UniformRefinement();
+  //   mesh2->UniformRefinement(); 
+  //}
+  //int gdim = mesh1->Dimension();
+  //FiniteElementCollection * fec = new H1_FECollection(1, gdim);
+  //FiniteElementSpace * fespace1 = new FiniteElementSpace(mesh1, fec, gdim, Ordering::byVDIM);
+  //FiniteElementSpace * fespace2 = new FiniteElementSpace(mesh2, fec, gdim, Ordering::byVDIM);
+  // 
+  //GridFunction x1_gf(fespace1);
+  //GridFunction x2_gf(fespace2);
+
+  //int ndof1 = fespace1->GetTrueVSize();
+  //int ndof2 = fespace2->GetTrueVSize();
+  //int ndof  = ndof1 + ndof2;
+  //for(int i = 0; i < ndof1; i++)
+  //{
+  //  x1_gf(i) = xf(i);
+  //}
+  //for(int i = ndof1; i < ndof; i++)
+  //{
+  //  x2_gf(i - ndof1) = xf(i);
+  //}
+
+  //mesh1->SetNodalFESpace(fespace1);
+  //mesh2->SetNodalFESpace(fespace2);
+  //GridFunction *nodes1 = mesh1->GetNodes();
+  //GridFunction *nodes2 = mesh2->GetNodes();
+
+  //{
+  //  *nodes1 += x1_gf;
+  //  *nodes2 += x2_gf;
+  //}
+  //
+
+  //ParaViewDataCollection paraview_dc1("QPContactBody1", mesh1);
+  //paraview_dc1.SetPrefixPath("ParaView");
+  //paraview_dc1.SetLevelsOfDetail(1);
+  //paraview_dc1.SetDataFormat(VTKFormat::BINARY);
+  //paraview_dc1.SetHighOrderOutput(true);
+  //paraview_dc1.SetCycle(0);
+  //paraview_dc1.SetTime(0.0);
+  //paraview_dc1.RegisterField("Body1", &x1_gf);
+  //paraview_dc1.Save();
+  //
+  //ParaViewDataCollection paraview_dc2("QPContactBody2", mesh2);
+  //paraview_dc2.SetPrefixPath("ParaView");
+  //paraview_dc2.SetLevelsOfDetail(1);
+  //paraview_dc2.SetDataFormat(VTKFormat::BINARY);
+  //paraview_dc2.SetHighOrderOutput(true);
+  //paraview_dc2.SetCycle(0);
+  //paraview_dc2.SetTime(0.0);
+  //paraview_dc2.RegisterField("Body2", &x2_gf);
+  //paraview_dc2.Save();
+
+  //delete fespace1;
+  //delete fespace2;
+  //delete fec;
+  //delete mesh1;
+  //delete mesh2;
+  
+  delete QPContact;
+  delete QPContactOptimizer;
+  
+  delete K;
+  delete J;
+  delete Jreduced;
+  delete contact;
+  return 0;
+}

examples/contact/makefile

@@ -0,0 +1,36 @@
+# Use the MFEM build directory
+MFEM_DIR ?= ../..
+MFEM_BUILD_DIR ?= ../..
+SRC = ./
+
+CONFIG_MK = $(MFEM_BUILD_DIR)/config/config.mk
+
+MFEM_LIB_FILE = mfem_is_not_built
+-include $(CONFIG_MK)
+
+# Remove built-in rule
+#%: %.cpp
+
+exQPContactBlockTL: exQPContactBlockTL.o problems.o IPsolver.o $(MFEM_LIB_FILE)
+	$(MFEM_CXX) $(MFEM_FLAGS) exQPContactBlockTL.o problems.o IPsolver.o -o $@ $(MFEM_LIBS)
+
+
+
+exQPContactBlockTL.o: exQPContactBlockTL.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+problems.o: problems.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $< 
+
+IPsolver.o: IPsolver.cpp $(CONFIG_MK)
+	$(MFEM_CXX) $(MFEM_FLAGS) -c $<
+
+# Generate an error message if the MFEM library is not built and exit
+$(MFEM_LIB_FILE):
+	$(error The MFEM library is not built)
+
+.PHONY: clean
+clean:
+	rm -f *.o exQPContactBlockTL
+
+

examples/contact/meshes/block1.mesh

@@ -0,0 +1,103 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+elements
+9
+1   5     0      1     3     2     8    9     11    10
+1   5     2      3     5     4    10    11    13    12
+1   5     4      5     7     6    12    13    15    14
+1   5     8      9    11    10    16    17    19    18
+1   5     10    11    13    12    18    19    21    20
+1   5     12    13    15    14    20    21    23    22
+1   5     16    17    19    18    24    25    27    26
+1   5     18    19    21    20    26    27    29    28
+1   5     20    21    23    22    28    29    31    30
+
+
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+30
+1   3    1     0     2     3
+1   3    3     2     4     5
+1   3    5     4     6     7
+1   3    24    25    27    26
+1   3    26    27    29    28
+1   3    28    29    31    30
+2   3    2     0     8    10
+2   3    4     2    10    12
+2   3    6     4    12    14
+2  3    10    8    16    18
+2  3    12    10    18    20
+2  3    14    12    20    22
+2  3    18    16    24    26
+2  3    20    18    26    28
+2  3    22    20    28    30
+3  3    1     3    11    9
+3  3     3     5    13    11
+3  3     5     7    15    13
+3  3    9    11    19    17
+3  3    11    13    21    19
+3  3    13    15    23    21
+3  3    17    19    27    25
+3  3    19    21    29    27
+3  3    21    23    31    29
+1  3     8     0     1    9
+1  3    16     8    9    17
+1  3    24    16    17    25
+1  3     6    14    15     7
+1  3    14    22    23    15
+1  3    22    30    31    23
+
+
+vertices
+32
+3
+   -1.0000         0         0
+         0         0         0
+   -1.0000    0.3000         0
+         0    0.3000         0
+   -1.0000    0.6500         0
+         0    0.6500         0
+   -1.0000    1.0000         0
+         0    1.0000         0
+   -1.0000         0    0.3000
+         0         0    0.3000
+   -1.0000    0.3000    0.3500
+         0    0.3000    0.3500
+   -1.0000    0.6500    0.3000
+         0    0.6500    0.3000
+   -1.0000    1.0000    0.3000
+         0    1.0000    0.3000
+   -1.0000         0    0.6500
+         0         0    0.6500
+   -1.0000    0.3000    0.6500
+         0    0.3000    0.6500
+   -1.0000    0.6500    0.6500
+         0    0.6500    0.6500
+   -1.0000    1.0000    0.6500
+         0    1.0000    0.6500
+   -1.0000         0    1.0000
+         0         0    1.0000
+   -1.0000    0.3000    1.0000
+         0    0.3000    1.0000
+   -1.0000    0.6500    1.0000
+         0    0.6500    1.0000
+   -1.0000    1.0000    1.0000
+         0    1.0000    1.0000

examples/contact/meshes/rotatedblock2.mesh

@@ -0,0 +1,70 @@
+
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+# PRISM       = 6
+#
+
+dimension
+3
+
+# 1 nothing
+elements
+4
+1   5     0     1     3     2     6     7    9     8
+1   5     2     3     5     4     8     9    11    10
+1   5     6     7     9     8    12    13    15    14
+1   5     8     9    11    10    14    15    17    16
+
+# 0 nothing
+# 1 dirichlet bc
+# 2 contact
+boundary
+16
+1   3     1     0     2     3
+1   3     3     2     4     5
+1   3    12    13    15    14
+1   3    14    15    17    16
+3   3     2     0     6     8
+3   3     4     2     8    10
+3   3     8     6    12    14
+3   3    10     8    14    16
+2   3     1     3     9     7
+2   3     3     5    11    9
+2   3     7    9    15    13
+2   3     9    11    17    15
+1   3     6     0     1     7
+1   3    12     6     7    13
+1   3     4    10    11     5
+1   3    10    16    17    11
+
+vertices
+18
+3
+
+0.000000000000 0.145770950245 0.443895630208
+0.507100000000 0.145770950245 0.443895630208
+0.000000000000 0.350937660019 0.294833290227
+0.507100000000 0.350937660019 0.294833290227
+0.000000000000 0.556104369792 0.145770950245
+0.507100000000 0.556104369792 0.145770950245
+0.000000000000 0.294833290227 0.649062339981
+0.507100000000 0.294833290227 0.649062339981
+0.000000000000 0.500000000000 0.500000000000
+0.507100000000 0.500000000000 0.500000000000
+0.000000000000 0.705166709773 0.350937660019
+0.507100000000 0.705166709773 0.350937660019
+0.000000000000 0.443895630208 0.854229049755
+0.507100000000 0.443895630208 0.854229049755
+0.000000000000 0.649062339981 0.705166709773
+0.507100000000 0.649062339981 0.705166709773
+0.000000000000 0.854229049755 0.556104369792
+0.507100000000 0.854229049755 0.556104369792

examples/contact/nodepair.hpp

@@ -0,0 +1,897 @@
+
+using namespace std;
+using namespace mfem;
+
+void BasisEval(const Vector xi, Vector &N, DenseMatrix &dNdxi) // dNdxi is 2*4
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]);
+   dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);
+   dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]);
+   dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);
+   dNdxi(1,3) = 0.25*(1-xi[0]);
+}
+
+
+void BasisEvalDerivs(const Vector xi, Vector& N, DenseMatrix& dNdxi,
+                     DenseMatrix& dN2dxi)
+{
+   N[0] = 0.25*(1-xi[0])*(1-xi[1]);
+   N[1] = 0.25*(1+xi[0])*(1-xi[1]);
+   N[2] = 0.25*(1+xi[0])*(1+xi[1]);
+   N[3] = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(2,4); dNdxi = 0.0;
+   dN2dxi.SetSize(3,4);
+   dN2dxi = 0.0;  // first row dxi2, second detadxi, third deta2
+
+   dNdxi(0,0) = 0.25*(-1+xi[1]); dNdxi(0,1) = 0.25*(1-xi[1]);
+   dNdxi(0,2) = 0.25*(1+xi[1]);  dNdxi(0,3) = 0.25*(-1-xi[1]);
+   dNdxi(1,0) = 0.25*(-1+xi[0]); dNdxi(1,1) = 0.25*(-1-xi[0]);
+   dNdxi(1,2) = 0.25*(1+xi[0]);  dNdxi(1,3) = 0.25*(1-xi[0]);
+
+   dN2dxi(1,0) = 0.25; dN2dxi(1,1) = -0.25; dN2dxi(1,2) = 0.25;
+   dN2dxi(1,3) = -0.25;
+}
+
+// returns the vector and matrix form of the shape functions and its derivative
+void BasisVectorDerivs(const Vector xi, DenseMatrix& N, DenseMatrix& dNdxi,
+                       DenseMatrix& ddNdxi)
+{
+   N.SetSize(3,12); N = 0.0;
+   N(0,0) = 0.25*(1-xi[0])*(1-xi[1]); N(0,3) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(0,6) = 0.25*(1+xi[0])*(1+xi[1]); N(0,9) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(1,1) = 0.25*(1-xi[0])*(1-xi[1]); N(1,4) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(1,7) = 0.25*(1+xi[0])*(1+xi[1]); N(1,10) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   N(2,2) = 0.25*(1-xi[0])*(1-xi[1]); N(2,5) = 0.25*(1+xi[0])*(1-xi[1]);
+   N(2,8) = 0.25*(1+xi[0])*(1+xi[1]); N(2,11) = 0.25*(1-xi[0])*(1+xi[1]);
+
+   dNdxi.SetSize(3*2, 3*4);  dNdxi = 0.0;
+   dNdxi(0,0) = 0.25*(-1+xi[1]);  dNdxi(0,3) = 0.25*(1-xi[1]);
+   dNdxi(0,6) = 0.25*(1+xi[1]);  dNdxi(0,9) = 0.25*(-1-xi[1]);
+   dNdxi(1,1) = 0.25*(-1+xi[1]);  dNdxi(1,4) = 0.25*(1-xi[1]);
+   dNdxi(1,7) = 0.25*(1+xi[1]);  dNdxi(1,10) = 0.25*(-1-xi[1]);
+   dNdxi(2,2) = 0.25*(-1+xi[1]);  dNdxi(2,5) = 0.25*(1-xi[1]);
+   dNdxi(2,8) = 0.25*(1+xi[1]);  dNdxi(2,11) = 0.25*(-1-xi[1]);
+
+   dNdxi(3,0) = 0.25*(-1+xi[0]);  dNdxi(3,3) = 0.25*(-1-xi[0]);
+   dNdxi(3,6) = 0.25*(1+xi[0]);  dNdxi(3,9) = 0.25*(1-xi[0]);
+   dNdxi(4,1) = 0.25*(-1+xi[0]);  dNdxi(4,4) = 0.25*(-1-xi[0]);
+   dNdxi(4,7) = 0.25*(1+xi[0]);  dNdxi(4,10) = 0.25*(1-xi[0]);
+   dNdxi(5,2) = 0.25*(-1+xi[0]);  dNdxi(5,5) = 0.25*(-1-xi[0]);
+   dNdxi(5,8) = 0.25*(1+xi[0]);  dNdxi(5,11) = 0.25*(1-xi[0]);
+
+   ddNdxi.SetSize(3*4, 3*4); ddNdxi = 0.0;
+   ddNdxi(3,0) = 0.25; ddNdxi(3,3) = -0.25;
+   ddNdxi(3,6) = 0.25; ddNdxi(3,9) = -0.25;
+   ddNdxi(4,1) = 0.25; ddNdxi(4,4) = -0.25;
+   ddNdxi(4,7) = 0.25; ddNdxi(4,10) = -0.25;
+   ddNdxi(5,2) = 0.25; ddNdxi(5,5) = -0.25;
+   ddNdxi(5,8) = 0.25; ddNdxi(5,11) = -0.25;
+
+   ddNdxi(6,0) = 0.25; ddNdxi(6,3) = -0.25;
+   ddNdxi(6,6) = 0.25; ddNdxi(6,9) = -0.25;
+   ddNdxi(7,1) = 0.25; ddNdxi(7,4) = -0.25;
+   ddNdxi(7,7) = 0.25; ddNdxi(7,10) = -0.25;
+   ddNdxi(8,2) = 0.25; ddNdxi(8,5) = -0.25;
+   ddNdxi(8,8) = 0.25; ddNdxi(8,11) = -0.25;
+}
+
+
+void cross(const Vector a, const Vector b, Vector& c)
+{
+   assert(a.Size()==3);
+   c.SetSize(3);
+   c[0] = a[1]*b[2] - a[2]*b[1];
+   c[1] = -a[0]*b[2] + b[0]*a[2];
+   c[2] = a[0]*b[1] - a[1]*b[0];
+
+}
+// a outer b
+void outer(const Vector a, const Vector b, DenseMatrix& c)
+{
+   int m = a.Size();
+   int n = b.Size();
+   assert(c.Height()==m);
+   assert(c.Width() ==n);
+   for (int i=0; i<m; i++)
+   {
+      for (int j=0; j<n; j++)
+      {
+         c(i,j) = a[i]*b[j];
+      }
+   }
+}
+// dphidxi 2*4
+// coords 4*3
+void ComputeNormal(const DenseMatrix& dphidxi, const DenseMatrix& coords,
+                   Vector& normal, double& nnorm)
+{
+
+   DenseMatrix dxdxi(2,3);
+   Mult(dphidxi, coords, dxdxi);
+   Vector dxdxi1(3);
+   Vector dxdxi2(3);
+
+   dxdxi.GetRow(0,dxdxi1);
+   dxdxi.GetRow(1,dxdxi2);
+
+   cross(dxdxi1, dxdxi2, normal);  // is there a cross product?  no
+   // VectorCrossProductCoefficient::Eval has hard-coded cross product
+   nnorm = normal.Norml2( );
+   normal /=  nnorm;
+}
+
+void SlaveToMaster(const DenseMatrix& m_coords, const Vector& s_x, Vector& xi)
+{
+   bool converged = false;
+   bool pt_on_elem = false;
+   int dim = 3;
+   xi.SetSize(dim-1);
+   xi = 0.0;
+   int max_iter = 15;
+   double off_el_xi = 1e-2;
+   double proj_newton_tol = 1e-13;
+   double proj_max_gap = 0.5;
+   Vector gap_v(dim);
+   // warm start from linear solution
+
+   for (int it=0; it<max_iter; it++)
+   {
+      //cout<<it<<endl;
+      Vector m_N(4);
+      m_N = 0.;
+      DenseMatrix m_dN(2,4);
+      m_dN = 0.;
+      DenseMatrix m_dN2(3,4);
+      m_dN2 = 0.;
+      BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+      Vector x_c(dim);
+      m_coords.MultTranspose(m_N, x_c);
+
+      gap_v = s_x;
+      gap_v -= x_c;
+
+      DenseMatrix m_dx(2,3);
+      m_dx = 0.;
+      Mult(m_dN, m_coords, m_dx);
+
+      Vector r(dim-1);
+      r = 0.0;
+      m_dx.Mult(gap_v, r);
+
+      if (r.Normlinf() < proj_newton_tol)
+      {
+         converged = true;
+         break;
+      }
+
+      DenseMatrix drdxi(dim-1,dim-1);
+      drdxi = 0.;
+      MultABt(m_dx, m_dx, drdxi); // m_dx * m_dx.T
+      drdxi *= -1.0;
+
+      DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+      Mult(m_dN2,m_coords, m_dx2);
+
+      //m_d2x = m_dN(:,:,2) * m_elem_coords(1:4,:);  //m_dN(:,:,2) is 3*4
+      for (int d=0; d<3; d++)
+      {
+         DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+         Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+         Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+         drdxi.Add(gap_v[d], Mtemp);
+      }
+
+      //cond_num = rcond(drdxi);  condition number?
+      //drdxi.TestInversion();
+      DenseMatrixInverse drdxi_inv(drdxi);
+      Vector xi_tmp(dim-1);
+
+      drdxi_inv.Mult(r,xi_tmp);
+      xi -= xi_tmp;
+   }
+   if (!converged)
+   {
+      xi = 0.0;
+   }
+   off_el_xi += 1 ; // tolerance of offset of xi outside [-1,1]
+
+   //cout<<gap_v.Norml2()<<" " <<xi.Normlinf()<<endl;
+   //
+   // Discuss with Frank... what is happening here
+   if (gap_v.Norml2() < proj_max_gap && xi.Normlinf() <= off_el_xi)
+   {
+      pt_on_elem = true;
+   }
+
+   if (pt_on_elem)
+   {
+     //cout << "convergence of node to segment projection? " << converged << endl;
+     //for(int i = 0; i < 2; i++)
+     //{
+     //  cout << "xi_" << i << " = " << xi(i) << endl;
+     //}
+   }
+   MFEM_VERIFY(pt_on_elem == true, "xi went out of bounds");
+   MFEM_VERIFY(converged == true, "projection didn't converge");
+}
+
+
+
+// m_coords is expected to be 4 * 3
+void  ComputeGapJacobian(const Vector x_s, const Vector xi,
+                         const DenseMatrix m_coords,
+                         double& gap, Vector& normal, Vector& dgdxm, Vector& dgdxs)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N, x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   double nnorm = 0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   gap = gap_v * normal; // gap function value, dot product between vectors
+
+   //dr_dx = zeros(2,4,3); % nsegment, nodes in quad, ndim
+
+   DenseMatrix dr_dx_res1(4,3); dr_dx_res1 = 0.;
+   DenseMatrix dr_dx_res2(4,3); dr_dx_res2 = 0.;
+
+   Vector m_dxrow1(3);
+   m_dx.GetRow(0, m_dxrow1);
+   outer(m_N, m_dxrow1, dr_dx_res1);// 4*1 times 1*3
+   dr_dx_res1 *= -1.0;
+
+   Vector m_dxrow2(3);
+   m_dx.GetRow(1, m_dxrow2);
+   outer(m_N, m_dxrow2, dr_dx_res2);// 4*1 times 1*3
+   dr_dx_res2 *= -1.0;
+
+   Vector m_dNrow1(4); m_dN.GetRow(0, m_dNrow1);
+   Vector m_dNrow2(4); m_dN.GetRow(1, m_dNrow2);
+
+   DenseMatrix dr_dx_res1_tmp(4,3); dr_dx_res1_tmp = 0.;
+   DenseMatrix dr_dx_res2_tmp(4,3); dr_dx_res2_tmp = 0.;
+   outer(m_dNrow1, gap_v, dr_dx_res1_tmp);// 4*1 times 1*3
+   outer(m_dNrow2, gap_v, dr_dx_res2_tmp);// 4*1 times 1*3
+
+   dr_dx_res1 += dr_dx_res1_tmp; // outer product in vector?
+   dr_dx_res2 += dr_dx_res2_tmp;
+
+
+   DenseMatrix K_dxidx1(2,2); // 2*2
+   K_dxidx1 = 0.;
+   MultABt(m_dx, m_dx, K_dxidx1); // m_dx * m_dx.T
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   // how to get 2nd order? multidimensional matrix?
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= K_dxidx1;
+   K_dxidx += K_dxidx2;
+
+   // resize the vectors and matrices
+   Vector dxidx(24); dxidx = 0.0;
+   Vector drdx_r(24); drdx_r = 0.0;
+
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         drdx_r[4*j+i] = dr_dx_res1(i,j);
+         drdx_r[4*j+i+12] = dr_dx_res2(i,j);
+
+      }
+   }
+   //drdx_r(1:4*3,1) = reshape(dr_dx_res(:,:,1),4*3,1);
+   //drdx_r(4*3+1:2*4*3,1) = reshape(dr_dx_res(:,:,2),4*3,1);
+   DenseMatrix drdx_K(24,24); drdx_K = 0.;
+   for (int i =0; i<12; i++)
+   {
+      drdx_K(i,i) = K_dxidx(0,0);
+      drdx_K(i,12+i) = K_dxidx(0,1);
+      drdx_K(12+i,i) = K_dxidx(1,0);
+      drdx_K(12+i,12+i) = K_dxidx(1,1);
+   }
+
+   DenseMatrixInverse drdxK_inv(drdx_K);
+   drdxK_inv.Mult(drdx_r,dxidx);
+   // LinearSolve (drdx_K,drdx_r, dxidx) ; //???
+   dxidx *= -1.0;
+
+
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+
+   Vector dxidxs(6); dxidxs = 0.0;
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   //dxidx = reshape(dxidx, 4,3,2); dxidxs = reshape(dxidxs, 1,3,2);
+
+   dgdxm.SetSize(12); dgdxm = 0.;
+   DenseMatrix dgdxm_tmp(4,3);
+   outer(m_N, normal,dgdxm_tmp);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         dgdxm[3*i+j] = -dgdxm_tmp(i,j);
+      }
+   }
+   //dxidx_M = -m_dN(1:2,:,1) * (m_coords(1:4,:)*normal'); % this turns out to be 0
+
+   dgdxs.SetSize(3);
+   dgdxs += normal;
+   //dgdxs = dgdxs + dxidx_M(1) * dxidxs(:,:,1) + dxidx_M(2) * dxidxs(:,:,2);
+};
+
+void ComputeGapHessian(const Vector x_s, const Vector xi,
+                       const DenseMatrix m_coords,
+                       DenseMatrix& dg2dx)
+{
+   Vector m_N(4);
+   DenseMatrix m_dN(2,4);
+   DenseMatrix m_dN2(3,4);
+   BasisEvalDerivs(xi, m_N, m_dN, m_dN2);
+
+   int dim = 3;
+   int num_dofs1 = dim;
+   int num_dofs2 = 4*dim;
+   int num_dofs = num_dofs1 + num_dofs2;
+   dg2dx.SetSize(num_dofs,num_dofs); dg2dx = 0.0;
+
+   Vector x_c(3);
+   m_coords.MultTranspose(m_N,x_c);
+
+   Vector gap_v(3); gap_v = 0.0;
+   gap_v  = x_s;
+   gap_v -= x_c;
+
+   DenseMatrix m_dx(2,3);
+   Mult(m_dN, m_coords, m_dx);
+
+   DenseMatrix m_dx2(3,3); m_dx2 = 0.0;
+   Mult(m_dN2,m_coords, m_dx2);
+   double nnorm = 0.0;
+   Vector normal(3); normal = 0.0;
+   ComputeNormal(m_dN, m_coords, normal, nnorm);
+
+   double gap = gap_v * normal; // gap function value, dot product between vectors
+
+   DenseMatrix M(2,2); M = 0.0;
+   MultABt(m_dx, m_dx, M);
+
+   DenseMatrix f(2, num_dofs2); f = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix Mtemp(2,2); Mtemp = 0.0;
+      Mtemp(0,0) = m_dx2(0,d); Mtemp(0,1) = m_dx2(1,d);
+      Mtemp(1,0) = m_dx2(1,d); Mtemp(1,1) = m_dx2(2,d);
+
+      M.Add(-gap_v[d], Mtemp);
+
+      Vector m_dxcol(2); m_dx.GetColumn(d, m_dxcol);
+      DenseMatrix ftmp(2,4);
+      outer(m_dxcol, m_N, ftmp);
+      ftmp *= -1;
+      ftmp.Add( gap_v[d], m_dN);   // 2*4
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         f(0,d+j*3) = ftmp(0,j);
+         f(1,d+j*3) = ftmp(1,j);
+      }
+   }
+   //fprintf('hess dxidxm\n');
+   DenseMatrixInverse Minv(M);
+   DenseMatrix dxidxm(2,num_dofs2); dxidxm = 0.0;
+   Minv.Mult(f, dxidxm);
+   //LinearSolve??
+   //dxidxm = M\f;
+
+   DenseMatrix nde2(2,2); nde2 = 0.0;
+   DenseMatrix Nndx2(2,num_dofs2); Nndx2 = 0.0;
+
+   for (int d=0; d<3; d++)
+   {
+      DenseMatrix ndetmp(2,2); ndetmp = 0.0;
+      ndetmp(0,0) = normal(d)*m_dx2(0,d); ndetmp(0,1) = normal(d)*m_dx2(1,d);
+      ndetmp(1,0) = normal(d)*m_dx2(1,d); ndetmp(1,1) = normal(d)*m_dx2(2,d);
+
+      nde2 += ndetmp;
+
+      for (int j=0; j<4; j++)
+      {
+         assert(d+3*j<num_dofs2);
+         Nndx2(0,d+j*3) = normal[d]*m_dN(0,j);
+         Nndx2(1,d+j*3) = normal[d]*m_dN(1,j);
+      }
+
+   }
+
+   DenseMatrix Ndn(2,num_dofs2); Ndn = 0.0;
+   Ndn += Nndx2;
+   AddMult(nde2, dxidxm, Ndn);
+
+
+   DenseMatrix M2(2,2); M2 = 0.0;
+   MultABt(m_dx, m_dx, M2);
+   DenseMatrixInverse M2inv(M2);
+   DenseMatrix diag2(2,2); diag2(0,0) = 1.0; diag2(1,1) = 1.0;
+   DenseMatrix m_con(2,2); m_con = 0.0;
+
+   M2inv.Mult(diag2, m_con);
+
+   DenseMatrix dg2dxm(num_dofs2, num_dofs2); dg2dxm = 0.0;
+
+   DenseMatrix dg2dxm_tmp(num_dofs2,2); dg2dxm_tmp = 0.0;
+   MultAtB(Ndn, m_con, dg2dxm_tmp);
+   Mult(dg2dxm_tmp, Ndn, dg2dxm);
+   dg2dxm *= gap;
+
+   DenseMatrix dg2dxm_tmp2(num_dofs2,num_dofs2); dg2dxm_tmp2 = 0.0;
+   MultAtB(Nndx2, dxidxm, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   dg2dxm_tmp = 0.0;
+   MultAtB(dxidxm, nde2, dg2dxm_tmp);
+
+   AddMult_a(-1.0, dg2dxm_tmp, dxidxm, dg2dxm);
+
+   dg2dxm_tmp2 = 0.0;
+   MultAtB(dxidxm, Nndx2, dg2dxm_tmp2);
+   dg2dxm.Add(-1.0, dg2dxm_tmp2);
+
+   Vector v_dxidx2(4);
+   m_coords.Mult(gap_v, v_dxidx2); // m_coords * gap_v;  // 4*3 * 3 = 4
+
+   DenseMatrix K_dxidx2(2,2); K_dxidx2 = 0.0;
+
+   Vector m_dN2row1(4); m_dN2.GetRow(0, m_dN2row1);
+   Vector m_dN2row2(4); m_dN2.GetRow(1, m_dN2row2);
+   Vector m_dN2row3(4); m_dN2.GetRow(2, m_dN2row3);
+   K_dxidx2(0,0) = m_dN2row1 * v_dxidx2; // how would 4*1 * 1*4 be computed?
+   K_dxidx2(0,1) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,0) = m_dN2row2 * v_dxidx2;
+   K_dxidx2(1,1) = m_dN2row3 * v_dxidx2;
+
+   DenseMatrix K_dxidx(2,2);
+   K_dxidx -= M2;
+   K_dxidx += K_dxidx2;
+
+   Vector drdxs_r(6);
+   drdxs_r[0] = m_dx(0,0); drdxs_r[1] = m_dx(0,1); drdxs_r[2] = m_dx(0,2);
+   drdxs_r[3] = m_dx(1,0); drdxs_r[4] = m_dx(1,1); drdxs_r[5] = m_dx(1,2);
+
+   DenseMatrix drdxs_K(6,6); drdxs_K = 0.;
+   for (int i=0; i<3; i++)
+   {
+      drdxs_K(i,i) = K_dxidx(0,0);
+      drdxs_K(i,3+i) = K_dxidx(0,1);
+      drdxs_K(i+3,i) = K_dxidx(1,0);
+      drdxs_K(i+3,i+3) = K_dxidx(1,1);
+   }
+   Vector dxidxs(6);
+
+   DenseMatrixInverse drdxsK_inv(drdxs_K);
+   drdxsK_inv.Mult(drdxs_r,dxidxs);
+   dxidxs *= -1.0;
+   //dxidxs = -drdxs_K\drdxs_r;
+
+   DenseMatrix dxidxs_m(2,3); dxidxs_m = 0.0;
+   dxidxs_m(0,0) = dxidxs[0]; dxidxs_m(0,1) = dxidxs[1]; dxidxs_m(0,2) = dxidxs[2];
+   dxidxs_m(1,0) = dxidxs[3]; dxidxs_m(1,1) = dxidxs[4]; dxidxs_m(1,2) = dxidxs[5];
+
+   DenseMatrix dtao1dxs(3,3); dtao1dxs = 0.0;
+   DenseMatrix dtao2dxs(3,3); dtao2dxs = 0.0;
+
+   Vector dxidxs_row1(3); dxidxs_row1 = 0.0; Vector dxidxs_row2(3);
+   dxidxs_row2 = 0.0;
+   Vector mdx2_row1(3); mdx2_row1 = 0.0; Vector mdx2_row2(3); mdx2_row2 = 0.0;
+   Vector mdx2_row3(3); mdx2_row3 = 0.0;
+   dxidxs_m.GetRow(0,dxidxs_row1);
+   dxidxs_m.GetRow(1,dxidxs_row2);
+   m_dx2.GetRow(0,mdx2_row1);
+   m_dx2.GetRow(1,mdx2_row2);
+   m_dx2.GetRow(2,mdx2_row3);
+
+   DenseMatrix dtaotmp(3,3); dtaotmp = 0.0;
+   outer(mdx2_row1, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row2, dxidxs_row1,dtaotmp);
+   dtao1dxs += dtaotmp; dtaotmp = 0.0;
+
+   outer(mdx2_row2, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+   outer(mdx2_row3, dxidxs_row2, dtaotmp);
+   dtao2dxs += dtaotmp; dtaotmp = 0.0;
+
+   DenseMatrix dtaodxs(3,3); dtaodxs = 0.0; //tao = tao1 cross tao2
+
+   for (int d=0; d<3; d++)
+   {
+      Vector dtao1dxs_tmp(3); dtao1dxs_tmp = 0.0;
+      dtao1dxs.GetColumn(d,dtao1dxs_tmp);
+      Vector m_dxrow(3); m_dx.GetRow(1, m_dxrow);
+
+      Vector dtaodxs_tmp(3); dtaodxs_tmp = 0.0;
+      cross(dtao1dxs_tmp, m_dxrow, dtaodxs_tmp);
+
+      Vector dtaodxs_tmp2(3); dtaodxs_tmp2 = 0.0;
+      m_dx.GetRow(0, m_dxrow);
+      dtao1dxs_tmp = 0.0;  // reuse the same vector for dtao2
+      dtao2dxs.GetColumn(d,dtao1dxs_tmp);
+      cross(m_dxrow, dtao1dxs_tmp, dtaodxs_tmp2);
+
+      dtaodxs_tmp2 += dtaodxs_tmp;
+      dtaodxs.SetCol(d, dtaodxs_tmp2);
+   }
+
+   DenseMatrix dndxs(3,3); dndxs = 0.0; dndxs += dtaodxs; dndxs *= 1.0/nnorm;
+   DenseMatrix dndxs_tmp(3,3); dndxs_tmp = 0.0;
+   outer(normal, normal, dndxs_tmp);
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxs, dndxs);
+
+   DenseMatrix dgvdxs(3,3); dgvdxs = 0.0;
+   MultAtB(m_dx, dxidxs_m, dgvdxs);
+   dgvdxs *= -1;
+   for (int d=0; d<3; d++)
+   {
+      dgvdxs(d,d) += 1.0;
+   }
+   //dxidxs: 2*3
+
+   DenseMatrix dg2dxs(3,3); dg2dxs = 0.0;
+   DenseMatrix dg2dxs_tmp(3,2); dg2dxs_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dg2dxs_tmp);
+   AddMult_a(-1.0, dg2dxs_tmp, dxidxs_m, dg2dxs);
+   DenseMatrix dg2dxs_tmp2(3,3); dg2dxs_tmp2 = 0.0;
+   MultAtB(dgvdxs, dndxs, dg2dxs_tmp2);
+   dg2dxs += dg2dxs_tmp2;
+   dg2dxs_tmp2 = 0.0;
+   MultAtB(dndxs, dndxs_tmp, dg2dxs_tmp2);
+   AddMult(dg2dxs_tmp2, dgvdxs, dg2dxs);
+
+   DenseMatrix Ne(3,12), Be(6,12), dBe(12,12);
+   BasisVectorDerivs(xi, Ne, Be, dBe);
+
+   DenseMatrix dtao1dxm(3,12); dtao1dxm.CopyRows(Be, 0, 2);
+   DenseMatrix dtao2dxm(3,12); dtao2dxm.CopyRows(Be, 3, 5);
+
+   Vector m_coords_v(12);
+   for (int i=0; i<4; i++)
+   {
+      for (int j=0; j<3; j++)
+      {
+         m_coords_v[i*3+j] = m_coords(i,j);
+      }
+   }
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix dBe_tmp(3,12);
+      dBe_tmp.CopyRows(dBe,i*3,(i+1)*3-1);
+
+      DenseMatrix dtaodxm_tmp(12,12); dtaodxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dtaodxm_tmp);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao1dxm);
+
+      //dtao1dxm += dBe(:,:,i)*reshape(m_coords(1:4,:)',12,1)*reshape(dxidxm(i,:),1,12); % 3*12
+      dBe_tmp = 0.0;
+      dBe_tmp.CopyRows(dBe,(i+2)*3,(i+3)*3-1);
+      AddMult(dBe_tmp, dtaodxm_tmp, dtao2dxm);
+
+   }
+
+   DenseMatrix dtaodxm(3,12); dtaodxm = 0.0;//tao = tao1 cross tao2
+
+   for (int d=0; d<12; d++)
+   {
+      Vector dtaodxm_tmp(3); dtaodxm_tmp = 0.0;
+      Vector dtaodxm_tmp2(3); dtaodxm_tmp2 = 0.0;
+      Vector tmp1(3); tmp1 = 0.0;  dtao1dxm.GetColumn(d,tmp1);
+      Vector m_dxrow2(3); m_dx.GetRow(1, m_dxrow2);
+      Vector m_dxrow1(3); m_dx.GetRow(0, m_dxrow1);
+      Vector tmp2(3); tmp2 = 0.0;  dtao2dxm.GetColumn(d,tmp2);
+
+      cross(tmp1, m_dxrow2, dtaodxm_tmp);
+      cross(m_dxrow1,tmp2, dtaodxm_tmp2);
+      dtaodxm_tmp += dtaodxm_tmp2;
+
+      dtaodxm.SetCol(d, dtaodxm_tmp);
+   }
+
+   DenseMatrix dndxm(3,12); dndxm = 0.0;
+   dndxm += dtaodxm;
+   dndxm *= 1.0/nnorm;
+   AddMult_a(-1/nnorm, dndxs_tmp, dtaodxm, dndxm); //dndxs_tmp = normal'*normal
+
+   DenseMatrix dgvdxm(3,12); dgvdxm = 0.0;
+   dgvdxm -= Ne;
+
+   for (int i=0; i<2; i++)
+   {
+      Vector dxidxm_tmp(num_dofs2); dxidxm_tmp = 0.0;
+      dxidxm.GetRow(i,dxidxm_tmp);
+
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      DenseMatrix dgvdxm_tmp(12,12); dgvdxm_tmp = 0.0;
+      outer(m_coords_v, dxidxm_tmp, dgvdxm_tmp);
+      AddMult_a(-1.0, Be_tmp, dgvdxm_tmp, dgvdxm);
+
+   }
+
+   DenseMatrix dg2dxsxm(3,12); dg2dxsxm = 0.0;
+   DenseMatrix dg2dxsxm_tmp(3,3); dg2dxsxm_tmp = 0.0;
+   MultAtB(dgvdxs, dndxm, dg2dxsxm);
+
+   MultAtB(dndxs, dndxs_tmp, dg2dxsxm_tmp);
+   AddMult(dg2dxsxm_tmp, dgvdxm, dg2dxsxm); // += dndxs'*normal'*normal*dgvdxm;
+
+   DenseMatrix dgvdxsxmn(3,12); dgvdxsxmn = 0.0;
+   DenseMatrix dgvdxsxmn_tmp(3,2); dgvdxsxmn_tmp = 0.0;
+   MultAtB(dxidxs_m, nde2, dgvdxsxmn_tmp);   //dxidxs_m: 2*3
+
+   AddMult_a(-1.0, dgvdxsxmn_tmp, dxidxm, dgvdxsxmn);
+
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxsxmn_tmp2(3,3); dgvdxsxmn_tmp2 = 0.0;
+      outer(dxidxs_row, normal, dgvdxsxmn_tmp2);
+      AddMult_a(-1.0, dgvdxsxmn_tmp2, Be_tmp, dgvdxsxmn);
+   }
+
+   dg2dxsxm += dgvdxsxmn;
+
+   DenseMatrix dg2dxmxs(12,3); dg2dxmxs = 0.0;
+   DenseMatrix dg2dxmxs_tmp(12,3); dg2dxmxs_tmp = 0.0;
+   MultAtB(dgvdxm, dndxs, dg2dxmxs);
+   MultAtB(dndxm, dndxs_tmp, dg2dxmxs_tmp);
+   AddMult(dg2dxmxs_tmp, dgvdxs, dg2dxmxs);
+
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   DenseMatrix dgvdxmxsn_tmp(12,2); dgvdxmxsn_tmp = 0.0;
+
+   MultAtB(dxidxm, nde2, dgvdxmxsn_tmp);
+   dgvdxmxsn_tmp *= -1.0;
+   AddMult(dgvdxmxsn_tmp, dxidxs_m, dgvdxmxsn);
+
+   for (int i =0; i<2; i++)
+   {
+      DenseMatrix Be_tmp(3,12);
+      Be_tmp.CopyRows(Be,i*3,(i+1)*3-1);
+      Be_tmp.Transpose(); // Be is now 12*3
+
+      Vector dxidxs_row(3); dxidxs_row = 0.0; dxidxs_m.GetRow(i,dxidxs_row);
+      DenseMatrix dgvdxmxsn_tmp2(3,3); dgvdxmxsn_tmp2 = 0.0;
+      outer(normal, dxidxs_row, dgvdxmxsn_tmp2);
+      AddMult_a(-1.0, Be_tmp, dgvdxmxsn_tmp2, dgvdxmxsn);
+
+   }
+
+   dg2dxmxs += dgvdxmxsn;
+
+   dg2dx.CopyMN(dg2dxs, 0, 0);
+   dg2dx.CopyMN(dg2dxm, 3, 3);
+   dg2dx.CopyMN(dg2dxsxm, 0, 3);
+   dg2dx.CopyMN(dg2dxmxs, 3, 0);
+
+};
+
+
+
+void NodeSegConPairs(const Vector x1, const Vector xi2,
+                     const DenseMatrix coords2,
+                     double& node_g, Vector& node_dg, DenseMatrix& node_dg2)
+{
+   double gap = 0.0;
+   Vector normal(3); normal = 0.0;
+   Vector dgdxm(12); dgdxm = 0.0;
+   Vector dgdxs(3); dgdxs = 0.0;
+
+   ComputeGapJacobian(x1, xi2, coords2, gap, normal, dgdxm, dgdxs);
+   node_g = gap;
+
+   node_dg.SetSize(12+3);
+   for (int i=0; i<3; i++) { node_dg[i] = dgdxs[i]; }
+   for (int i=0; i<12; i++) { node_dg[i+3] = dgdxm[i]; }
+
+   DenseMatrix dg2dx(15,15); dg2dx = 0.0;
+   DenseMatrix dgvdxmxsn(12,3); dgvdxmxsn = 0.0;
+   ComputeGapHessian(x1, xi2, coords2, dg2dx);
+
+   node_dg2.SetSize(15,15);
+   node_dg2 = dg2dx;
+
+   /*
+   if(obj.space1.conns{e1}(i)==150)  % for debugging purpose
+
+   v1 = 1:3;
+   v2 = 1:12;
+   %v1 = ones(1,3)
+   %v2 = ones(1,12)
+   v2 = reshape(v2,4,3);
+   x1n1 = x1 + 0.01*v1;
+   coords2n1 = coords2 + 0.001*v2;
+      [xi2n1, gapv1, ~, ~] = SlaveToMaster(obj, coords2n1, x1n1);
+      [gapn1, n1,dgdxmn1, dgdxsn1] = ComputeGapJacobian(obj, x1n1, xi2n1, coords2n1);
+   x1n2 = x1 - 0.01*v1;
+   coords2n2 = coords2 - 0.001*v2;
+      [xi2n2, gapv2, ~, ~] = SlaveToMaster(obj, coords2n2, x1n2);
+      [gapn2, n2,dgdxmn2, dgdxsn2] = ComputeGapJacobian(obj, x1n2, xi2n2, coords2n2);
+   fprintf('fd\n');
+      %gapv1-gapv2
+   [dgdxsn1(:)',dgdxmn1(:)'] - [dgdxsn2(:)',dgdxmn2(:)']
+
+   %dgdxsn1-dgdxsn2
+   fprintf('code\n');
+   v2n = v2';
+   %dg2dx(1:3,1:3)*0.04*ones(3,1)
+   temp = zeros(12,3);
+   for i = 1:4
+   temp1 = dg2dx(3+(i-1)*3+1:3+i*3,1:3);
+   temp((i-1)*3+1:i*3,:) = temp1';
+   end
+   temp2 = zeros(3,12);
+   for i = 1:4
+   temp3 = dg2dx(1:3,3+(i-1)*3+1:3+i*3);
+   temp2(:,(i-1)*3+1:i*3) = temp3';
+   end
+   %dg2dx
+   %dg2dx(4:end,1:3) = temp;
+   %dg2dx(1:3,4:end) = temp2;
+   %dgvdxm * 0.002*v2n(:)
+   (dg2dx*[0.02*v1(:)',0.002*v2n(:)']')'
+   %dg2dx(4:end,1:3)
+   end*/
+
+};
+
+
+// coordsm : (npoints*4, 3) use what class?
+// m_conn: (npoints*4)
+void Assemble_Contact(const int m, const int npoints, const int ndofs,
+                      const Vector x_s,
+                      const Vector xi, const DenseMatrix coordsm, const Array<int> s_conn,
+                      const Array<int> m_conn, Vector& g, SparseMatrix& M,
+                      std::vector<SparseMatrix>& dM)
+{
+   int ndim = 3;
+
+   g.SetSize(m);
+   g = 0.0;
+
+   //SparseMatrix M(m, n);  // M needs to be the correct size
+
+   //dM.resize(m); // needs to clear?
+
+   double g_tmp = 0.;
+   Vector dg(4*ndim+ndim);
+   dg = 0.;
+   DenseMatrix dg2(4*ndim+ndim,4*ndim+ndim);
+   dg2 = 0.;
+
+   for (int i=0; i<npoints; i++)
+   {
+      Vector x1(ndim);
+      x1[0] = x_s[i*ndim];
+      x1[1] = x_s[i*ndim+1];
+      x1[2] = x_s[i*ndim+2];
+
+      Vector xi2(ndim-1);
+      xi2[0] = xi[i*(ndim-1)];
+      xi2[1] = xi[i*(ndim-1)+1];
+
+      DenseMatrix coords2(4,3);
+      coords2.CopyRows(coordsm, i*4,(i+1)*4-1);
+
+      //how to get coords2?
+      dg = 0.0;
+      dg2 = 0.;
+      NodeSegConPairs(x1, xi2, coords2, g_tmp, dg, dg2);
+      g[s_conn[i]] = g_tmp; // should be unique
+      Array<int> m_conn_i(4);
+      m_conn.GetSubArray(4*i, 4, m_conn_i);
+
+      Array<int> node_conn(5);
+      node_conn[0] = s_conn[i];
+      for (int j=0; j<4; j++)
+      {
+         node_conn[j+1] = m_conn_i[j];
+      }
+
+      Array<int>  M_i_tmp(1);
+      M_i_tmp[0] = s_conn[i];
+
+      //j_idx = (node_conn-1)*obj.disp_field.num_components +repmat((1:obj.disp_field.num_components)', 1, length(node_conn{i}));
+      Array<int> j_idx(5*ndim); j_idx = 0;
+      for (int j=0; j< 5; j++)
+      {
+         for (int k=0; k<ndim; k++)
+         {
+            j_idx[j*ndim+k] = node_conn[j]*ndim+k;
+         }
+      }
+      DenseMatrix M_v_tmp(1, ndim*(4+1));   // SetData now?
+      M_v_tmp.SetRow(0, dg);
+
+      M.AddSubMatrix(M_i_tmp, j_idx, M_v_tmp);
+
+      Array<int> dM_i(ndim*(4+1));
+      Array<int> dM_j(ndim*(4+1));
+
+      for (int j=0; j< ndim*(4+1); j++)
+      {
+         dM_i[j] = j_idx[j];
+         dM_j[j] = j_idx[j];
+      }
+      dM[s_conn[i]].AddSubMatrix(dM_i,dM_j, dg2);
+      dM[s_conn[i]].Finalize();
+      dM[s_conn[i]].Threshold(0.0);
+      dM[s_conn[i]].SortColumnIndices();
+   }
+   M.Finalize();
+   M.Threshold(0.0);
+   M.SortColumnIndices();
+};
+

examples/contact/problems.hpp

@@ -0,0 +1,396 @@
+#include "mfem.hpp"
+#include <fstream>
+#include <iostream>
+#include <set>
+using namespace std;
+using namespace mfem;
+
+#ifndef PROBLEM_DEFS
+#define PROBLEM_DEFS
+
+
+
+// abstract OptProblem class
+// of the form
+// min_(u,m) f(u,m) s.t. c(u,m)=0 and m>=ml
+// the primal variable (u, m) is represented as a BlockVector
+
+class OptProblem
+{
+protected:
+    int dimU, dimM, dimC;
+    Array<int> block_offsetsx;
+    Vector ml;
+public:
+    OptProblem();
+    virtual double CalcObjective(const BlockVector &) const = 0;
+    virtual void Duf(const BlockVector &, Vector &) const = 0;
+    virtual void Dmf(const BlockVector &, Vector &) const = 0;
+    void CalcObjectiveGrad(const BlockVector &, BlockVector &) const;
+    virtual SparseMatrix* Duuf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dumf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmuf(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmmf(const BlockVector &) = 0;
+    virtual void c(const BlockVector &, Vector &) const = 0;
+    virtual SparseMatrix* Duc(const BlockVector &) = 0;
+    virtual SparseMatrix* Dmc(const BlockVector &) = 0;
+    // TO DO: include Hessian terms of constraint c
+    // TO DO: include log-barrier lumped-mass and pass that
+    // to the optimizer
+    //virtual SparseMatrix* GetLogBarrierLumpedMass() = 0;
+    int GetDimU() const { return dimU; };
+    int GetDimM() const { return dimM; }; 
+    int GetDimC() const { return dimC; };
+    Vector Getml() const { return ml; };
+    ~OptProblem();
+};
+
+
+// abstract ContactProblem class
+// of the form
+// min_d e(d) s.t. g(d) >= 0
+// TO DO: add functionality for gap function Hessian apply 
+class ContactProblem : public OptProblem
+{
+protected:
+    int dimD;
+    int dimS;
+    Array<int> block_offsetsx;
+public:
+    //ContactProblem(int, int);        // constructor
+    ContactProblem();
+    void InitializeParentData(int, int);
+    double CalcObjective(const BlockVector &) const; // objective e
+    void Duf(const BlockVector &, Vector &) const;
+    void Dmf(const BlockVector &, Vector &) const;
+    SparseMatrix* Duuf(const BlockVector &);
+    SparseMatrix* Dumf(const BlockVector &);
+    SparseMatrix* Dmuf(const BlockVector &);
+    SparseMatrix* Dmmf(const BlockVector &);
+    void c(const BlockVector &, Vector &) const;
+    SparseMatrix* Duc(const BlockVector &);
+    SparseMatrix* Dmc(const BlockVector &);
+    virtual double E(const Vector &) const = 0;           // objective e(d) (energy function)
+    virtual void DdE(const Vector &, Vector &) const = 0; // gradient of objective De / Dd
+    virtual SparseMatrix* DddE(const Vector &) = 0;       // Hessian of objective D^2 e / D d^2
+    virtual void g(const Vector &, Vector &) const = 0;   // inequality constraint g(d) >= 0 (gap function)
+    virtual SparseMatrix* Ddg(const Vector &) = 0;        // Jacobian of inequality constraint Dg / Dd
+    int GetDimD() const { return dimD; };
+    int GetDimS() const { return dimS; };
+    virtual ~ContactProblem();
+};
+
+
+class ObstacleProblem : public ContactProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d >= 0
+   // stiffness matrix used to define objective
+   BilinearForm *Kform;
+   LinearForm   *fform;
+   Array<int> empty_tdof_list; // needed for calls to FormSystemMatrix
+   SparseMatrix  K;
+   SparseMatrix  *J;
+   FiniteElementSpace *Vh;
+   Vector f;
+public : 
+   ObstacleProblem(FiniteElementSpace* , double (*fSource)(const Vector &));
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   // TO DO: include lumped-mass for the log-barrier term
+   //SparseMatrix* GetLogBarrierLumpedMass();
+   virtual ~ObstacleProblem();
+};
+
+class DirichletObstacleProblem : public ContactProblem
+{
+protected:
+   // data to define energy objective function e(d) = 0.5 d^T K d - f^T d, g(d) = d + \psi >= 0
+   // stiffness matrix used to define objective
+   BilinearForm *Kform;
+   LinearForm   *fform;
+   Array<int> ess_tdof_list; // needed for calls to FormSystemMatrix
+   SparseMatrix  *K;
+   SparseMatrix  *J;
+   FiniteElementSpace *Vh;
+   Vector f;
+   Vector psi;
+   Vector xDC;
+public : 
+   DirichletObstacleProblem(FiniteElementSpace*, Vector&,  double (*fSource)(const Vector &), double (*obstacleSource)(const Vector &), Array<int> tdof_list, bool);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   virtual ~DirichletObstacleProblem();
+};
+
+
+// abstract out technology for removing null rows of the Jacobian from an existing contact problem
+class ReducedContactProblem : public ContactProblem
+{
+protected:
+   Array<int> activeConstraints;
+   Array<int> fixedDofs;
+   ContactProblem * contact;
+   int dimSin;
+public:
+   ReducedContactProblem(ContactProblem * contact, Array<int> activeConstraints, Array<int> fixedDofs);
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   virtual ~ReducedContactProblem();
+};
+
+
+class QPContactProblem : public ContactProblem
+{
+protected:
+  SparseMatrix *K;
+  SparseMatrix *J;
+  Vector f;
+  Vector g0;
+public:
+  QPContactProblem(const SparseMatrix, const SparseMatrix, const Vector, const Vector);
+  double E(const Vector &) const;
+  void DdE(const Vector &, Vector &) const;
+  SparseMatrix* DddE(const Vector &);
+  void g(const Vector &, Vector &) const;
+  SparseMatrix* Ddg(const Vector &);
+  virtual ~QPContactProblem();
+};
+
+
+typedef int Index;
+typedef double Number;
+
+class ExContactBlockTL : public ContactProblem
+{
+public:
+   double E(const Vector &) const;
+   void DdE(const Vector &, Vector &) const;
+   SparseMatrix* DddE(const Vector &);
+   void g(const Vector &, Vector &) const;
+   SparseMatrix* Ddg(const Vector &);
+   FiniteElementSpace GetVh1();
+   FiniteElementSpace GetVh2();
+
+public:
+   /** default constructor */
+   ExContactBlockTL(int );
+   
+
+   /** default destructor */
+   virtual ~ExContactBlockTL();
+
+   ///**@name Overloaded from TNLP */
+   ///** Method to return some info about the nlp */
+   //virtual bool get_nlp_info(
+   //   Index&          n,
+   //   Index&          m,
+   //   Index&          nnz_jac_g,
+   //   Index&          nnz_h_lag,
+   //   IndexStyleEnum& index_style
+   //);
+
+   ///** Method to return the bounds for my problem */
+   //virtual bool get_bounds_info(
+   //   Index   n,
+   //   Number* x_l,
+   //   Number* x_u,
+   //   Index   m,
+   //   Number* g_l,
+   //   Number* g_u
+   //);
+
+   ///** Method to return the starting point for the algorithm */
+   //virtual bool get_starting_point(
+   //   Index   n,
+   //   bool    init_x,
+   //   Number* x,
+   //   bool    init_z,
+   //   Number* z_L,
+   //   Number* z_U,
+   //   Index   m,
+   //   bool    init_lambda,
+   //   Number* lambda
+   //);
+
+   /* Method to return the objective value */
+   virtual bool eval_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number&       obj_value
+   ) const;
+
+   /* Method to return the gradient of the objective */
+   virtual bool eval_grad_f(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number*       grad_f
+   ) const;
+
+   /* Method to return the constraint residuals */
+   virtual bool eval_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Number*       cons
+   ) const;
+
+   /* Method to return:
+      1) The structure of the Jacobian (if "values" is NULL)
+      2) The values of the Jacobian (if "values" is not NULL)
+   */
+   virtual bool eval_jac_g(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Index         m,
+      Index         nele_jac,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   ) const;
+
+   /* Method to return:
+    *   1) The structure of the Hessian of the Lagrangian (if "values" is NULL)
+    *   2) The values of the Hessian of the Lagrangian (if "values" is not NULL)
+   */
+   virtual bool eval_h(
+      Index         n,
+      const Number* x,
+      bool          new_x,
+      Number        obj_factor,
+      Index         m,
+      const Number* lambda,
+      bool          new_lambda,
+      Index         nele_hess,
+      Index*        iRow,
+      Index*        jCol,
+      Number*       values
+   );
+
+   ///** This method is called when the algorithm is complete so the TNLP can store/write the solution */
+   //virtual void finalize_solution(
+   //   SolverReturn               status,
+   //   Index                      n,
+   //   const Number*              x,
+   //   const Number*              z_L,
+   //   const Number*              z_U,
+   //   Index                      m,
+   //   const Number*              g,
+   //   const Number*              lambda,
+   //   Number                     obj_value,
+   //   const IpoptData*           ip_data,
+   //   IpoptCalculatedQuantities* ip_cq
+   //);
+
+private:
+   void update_g() const;
+   void update_jac();
+   void update_hess();
+
+private:
+   /**@name Methods to block default compiler methods.
+    *
+    * The compiler automatically generates the following three methods.
+    *  Since the default compiler implementation is generally not what
+    *  you want (for all but the most simple classes), we usually
+    *  put the declarations of these methods in the private section
+    *  and never implement them. This prevents the compiler from
+    *  implementing an incorrect "default" behavior without us
+    *  knowing. (See Scott Meyers book, "Effective C++")
+    */
+   ExContactBlockTL(
+      const ExContactBlockTL&
+   );
+
+   ExContactBlockTL& operator=(
+      const ExContactBlockTL&
+   );
+
+   Array<int> attr;
+   Array<int> m_attr;
+   Array<int> s_conn; // connectivity of the second/slave mesh
+   std::string mesh_file1;
+   std::string mesh_file2;
+   Mesh* mesh1;
+   Mesh* mesh2;
+   FiniteElementCollection* fec1;
+   FiniteElementCollection* fec2;
+   FiniteElementSpace* fespace1;
+   FiniteElementSpace* fespace2;
+   Array<int> ess_tdof_list1;
+   Array<int> ess_tdof_list2;
+   GridFunction nodes0;
+   GridFunction* nodes1;
+   GridFunction* nodes2;
+   mutable GridFunction* x1;
+   mutable GridFunction* x2;
+   LinearForm* b1;
+   LinearForm* b2;
+   PWConstCoefficient* lambda1_func;
+   PWConstCoefficient* lambda2_func;
+   PWConstCoefficient* mu1_func;
+   PWConstCoefficient* mu2_func;
+   BilinearForm* a1;
+   BilinearForm* a2;
+
+   mfem::Vector lambda1;
+   mfem::Vector lambda2;
+   mfem::Vector mu1;
+   mfem::Vector mu2;
+   mutable mfem::Vector xyz;
+
+   std::set<int> bdryVerts2;
+
+   int dim;
+   // degrees of freedom of both meshes
+   int ndof_1;
+   int ndof_2;
+   int ndofs;
+   // number of nodes for each mesh
+   int nnd_1;
+   int nnd_2;
+   int nnd;
+
+   int npoints;
+
+   SparseMatrix A1;
+   mfem::Vector B1, X1;
+   SparseMatrix A2;
+   mfem::Vector B2, X2;
+
+   SparseMatrix* K;
+   mutable mfem::Vector gapv;
+   mutable mfem::Vector m_xi;
+   mutable mfem::Vector xs;
+
+   mutable Array<int> m_conn; // only works for linear elements that have 4 vertices!
+   mutable DenseMatrix* coordsm;
+   mutable SparseMatrix* M;
+
+   mutable std::vector<SparseMatrix>* dM;
+
+   Array<int> Dirichlet_dof;
+   Array<double> Dirichlet_val;
+
+public: 
+   Mesh * GetMesh1() {return mesh1;}
+   Mesh * GetMesh2() {return mesh2;}
+   Array<int> GetDirichletDofs() {return Dirichlet_dof;}
+   Array<double> GetDirichletVals() {return Dirichlet_val;}
+
+};
+
+#endif

examples/ex0.cpp

@@ -22,7 +22,7 @@ using namespace mfem;
 int main(int argc, char *argv[])
 {
    // 1. Parse command line options.
-   string mesh_file = "../data/star.mesh";
+   const char *mesh_file = "../data/star.mesh";
    int order = 1;
 
    OptionsParser args(argc, argv);

examples/ex0p.cpp

@@ -26,7 +26,7 @@ int main(int argc, char *argv[])
    Hypre::Init();
 
    // 2. Parse command line options.
-   string mesh_file = "../data/star.mesh";
+   const char *mesh_file = "../data/star.mesh";
    int order = 1;
 
    OptionsParser args(argc, argv);

examples/ex2p.cpp

@@ -100,21 +100,6 @@ int main(int argc, char *argv[])
    Device device(device_config);
    if (myid == 0) { device.Print(); }
 
-   if (mfem::Device::Allows(mfem::Backend::DEVICE_MASK))
-   {
-      HYPRE_SetMemoryLocation(HYPRE_MEMORY_DEVICE);
-      HYPRE_SetExecutionPolicy(HYPRE_EXEC_DEVICE);
-      HYPRE_DeviceInitialize();
-   }
-   else
-   {
-      HYPRE_SetMemoryLocation(HYPRE_MEMORY_HOST);
-      HYPRE_SetExecutionPolicy(HYPRE_EXEC_HOST);
-   }
-
-   auto loc = mfem::GetHypreMemoryLocation();
-   auto exec = mfem::GetHypreExecutionPolicy();
-
    // 4. Read the (serial) mesh from the given mesh file on all processors.  We
    //    can handle triangular, quadrilateral, tetrahedral, hexahedral, surface
    //    and volume meshes with the same code.

examples/ex34.cpp

@@ -267,9 +267,9 @@ int main(int argc, char *argv[])
                 << "window_geometry 400 0 400 350" << flush;
    }
 
-   // 7. Define a parallel finite element space on the full mesh. Here we use
-   //    the H(curl) finite elements for the vector potential and H(div) for the
-   //    current density.
+   // 7. Define a parallel finite element space on the full mesh. Here we
+   //    use the H(curl) finite elements for the vector potential and H(div)
+   //    for the current density.
    ND_FECollection fec_nd(order, dim);
    RT_FECollection fec_rt(order - 1, dim);
    FiniteElementSpace fespace_nd(&mesh, &fec_nd);
@@ -292,10 +292,10 @@ int main(int argc, char *argv[])
    }
 
    // 8. Determine the list of true (i.e. parallel conforming) essential
-   //    boundary dofs. In this example, the boundary conditions are defined by
-   //    marking all the boundary attributes except for those on a symmetry
-   //    plane as essential (Dirichlet) and converting them to a list of true
-   //    dofs.
+   //    boundary dofs. In this example, the boundary conditions are defined
+   //    by marking all the boundary attributes except for those on a symmetry
+   //    plane as essential (Dirichlet) and converting them to a list of
+   //    true dofs.
    Array<int> ess_tdof_list;
    Array<int> ess_bdr;
    if (mesh.bdr_attributes.Size())
@@ -324,13 +324,14 @@ int main(int argc, char *argv[])
    GridFunction x(&fespace_nd);
    x = 0.0;
 
-   // 11. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + delta I, by adding the curl-curl and the
-   //     mass domain integrators. For standard magnetostatics equations choose
-   //     delta << 1. Larger values of delta should make the linear system
-   //     easier to solve at the expense of resembling a diffusive quasistatic
-   //     magnetic field.  A reasonable balance must be found whenever the mesh
-   //     or problem setup is altered.
+   // 11. Set up the parallel bilinear form corresponding to the EM
+   //     diffusion operator curl muinv curl + delta I, by adding the
+   //     curl-curl and the mass domain integrators. For standard
+   //     magnetostatics equations choose delta << 1. Larger values of
+   //     delta should make the linear system easier to solve at the
+   //     expense of resembling a diffusive quasistatic magnetic field.
+   //     A reasonable balance must be found whenever the mesh or problem
+   //     setup is altered.
    ConstantCoefficient muinv(1.0);
    ConstantCoefficient delta(delta_const);
    BilinearForm a(&fespace_nd);
@@ -422,8 +423,8 @@ int main(int argc, char *argv[])
    GridFunction dx(&fespace_rt);
    curl.Mult(x, dx);
 
-   // 18. Save the curl of the solution in parallel. This output can be viewed
-   //     later using GLVis: "glvis -np <np> -m mesh -g dsol".
+   // 18. Save the curl of the solution in parallel. This output can
+   //     be viewed later using GLVis: "glvis -np <np> -m mesh -g dsol".
    {
       ostringstream dsol_name;
       dsol_name << "dsol.gf";
@@ -455,18 +456,18 @@ void ComputeCurrentDensityOnSubMesh(int order,
                                     const Array<int> &jn_zero_attr,
                                     GridFunction &j_cond)
 {
-   // Extract the finite element space and mesh on which j_cond is defined
+   // Exract the finite element space and mesh on which j_cond is defined
    FiniteElementSpace &fes_cond_rt = *j_cond.FESpace();
    Mesh &mesh_cond = *fes_cond_rt.GetMesh();
    int dim  = mesh_cond.Dimension();
 
-   // Define a parallel finite element space on the SubMesh. Here we use the H1
-   // finite elements for the electrostatic potential.
+   // Define a parallel finite element space on the SubMesh. Here we use the
+   // H1 finite elements for the electrostatic potential.
    H1_FECollection fec_h1(order, dim);
    FiniteElementSpace fes_cond_h1(&mesh_cond, &fec_h1);
 
-   // Define the conductivity coefficient and the boundaries associated with the
-   // fixed potentials phi0 and phi1 which will drive the current.
+   // Define the conductivity coefficient and the boundaries associated with
+   // the fixed potentials phi0 and phi1 which will drive the current.
    ConstantCoefficient sigmaCoef(1.0);
    Array<int> ess_bdr_phi(mesh_cond.bdr_attributes.Max());
    Array<int> ess_bdr_j(mesh_cond.bdr_attributes.Max());
@@ -577,9 +578,9 @@ void ComputeCurrentDensityOnSubMesh(int order,
                 << "window_geometry 0 0 400 350" << flush;
    }
 
-   // Solve for the current density J = -sigma Grad phi with boundary conditions
-   // J.n = 0 on the walls of the conductor but not on the ports where phi=0 and
-   // phi=1.
+   // Solve for the current density J = -sigma Grad phi with boundary
+   // conditions J.n = 0 on the walls of the conductor but not on the
+   // ports where phi=0 and phi=1.
 
    // J will be computed in H(div) so we need an RT mass matrix
    BilinearForm m_rt(&fes_cond_rt);

examples/ex34p.cpp

@@ -302,9 +302,9 @@ int main(int argc, char *argv[])
                 << "window_geometry 400 0 400 350" << flush;
    }
 
-   // 8. Define a parallel finite element space on the full mesh. Here we use
-   //    the H(curl) finite elements for the vector potential and H(div) for the
-   //    current density.
+   // 8. Define a parallel finite element space on the full mesh. Here we
+   //    use the H(curl) finite elements for the vector potential and H(div)
+   //    for the current density.
    ND_FECollection fec_nd(order, dim);
    RT_FECollection fec_rt(order - 1, dim);
    ParFiniteElementSpace fespace_nd(&pmesh, &fec_nd);
@@ -360,13 +360,14 @@ int main(int argc, char *argv[])
    ParGridFunction x(&fespace_nd);
    x = 0.0;
 
-   // 12. Set up the parallel bilinear form corresponding to the EM diffusion
-   //     operator curl muinv curl + delta I, by adding the curl-curl and the
-   //     mass domain integrators. For standard magnetostatics equations choose
-   //     delta << 1. Larger values of delta should make the linear system
-   //     easier to solve at the expense of resembling a diffusive quasistatic
-   //     magnetic field.  A reasonable balance must be found whenever the mesh
-   //     or problem setup is altered.
+   // 12. Set up the parallel bilinear form corresponding to the EM
+   //     diffusion operator curl muinv curl + delta I, by adding the
+   //     curl-curl and the mass domain integrators. For standard
+   //     magnetostatics equations choose delta << 1. Larger values of
+   //     delta should make the linear system easier to solve at the
+   //     expense of resembling a diffusive quasistatic magnetic field.
+   //     A reasonable balance must be found whenever the mesh or problem
+   //     setup is altered.
    ConstantCoefficient muinv(1.0);
    ConstantCoefficient delta(delta_const);
    ParBilinearForm a(&fespace_nd);
@@ -503,7 +504,7 @@ void ComputeCurrentDensityOnSubMesh(int order,
                                     const Array<int> &jn_zero_attr,
                                     ParGridFunction &j_cond)
 {
-   // Extract the finite element space and mesh on which j_cond is defined
+   // Exract the finite element space and mesh on which j_cond is defined
    ParFiniteElementSpace &fes_cond_rt = *j_cond.ParFESpace();
    ParMesh &pmesh_cond = *fes_cond_rt.GetParMesh();
    int myid = fes_cond_rt.GetMyRank();
@@ -514,8 +515,8 @@ void ComputeCurrentDensityOnSubMesh(int order,
    H1_FECollection fec_h1(order, dim);
    ParFiniteElementSpace fes_cond_h1(&pmesh_cond, &fec_h1);
 
-   // Define the conductivity coefficient and the boundaries associated with the
-   // fixed potentials phi0 and phi1 which will drive the current.
+   // Define the conductivity coefficient and the boundaries associated with
+   // the fixed potentials phi0 and phi1 which will drive the current.
    ConstantCoefficient sigmaCoef(1.0);
    Array<int> ess_bdr_phi(pmesh_cond.bdr_attributes.Max());
    Array<int> ess_bdr_j(pmesh_cond.bdr_attributes.Max());
@@ -598,9 +599,9 @@ void ComputeCurrentDensityOnSubMesh(int order,
                 << "window_geometry 0 0 400 350" << flush;
    }
 
-   // Solve for the current density J = -sigma Grad phi with boundary conditions
-   // J.n = 0 on the walls of the conductor but not on the ports where phi=0 and
-   // phi=1.
+   // Solve for the current density J = -sigma Grad phi with boundary
+   // conditions J.n = 0 on the walls of the conductor but not on the
+   // ports where phi=0 and phi=1.
 
    // J will be computed in H(div) so we need an RT mass matrix
    ParBilinearForm m_rt(&fes_cond_rt);

examples/ex35p.cpp

@@ -35,10 +35,10 @@
 //               conductivity, sigma = c. The user can specify these constants
 //               using either set of names.
 //
-//               This example demonstrates how to transfer fields computed on a
-//               boundary generated SubMesh to the full mesh and apply them as
-//               boundary conditions. The default mesh and corresponding
-//               boundary attributes were chosen to verify proper behavior on
+//               This example demonstrates how to transfer fields computed on
+//               a boundary generated SubMesh to the full mesh and apply them
+//               as boundary conditions. The default mesh and corresponding
+//               boundary attriburtes were chosen to verify proper behavior on
 //               both triangular and quadrilateral faces of tetrahedral,
 //               wedge-shaped, and hexahedral elements.
 //
@@ -420,6 +420,7 @@ int main(int argc, char *argv[])
       //
       //      2) A vector H(Div) field
       //         -Grad(a Div) - omega^2 b + i omega c
+      //
       ParBilinearForm pcOp(&fespace);
       if (pa) { pcOp.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
       switch (prob)
@@ -444,8 +445,8 @@ int main(int argc, char *argv[])
       pcOp.Assemble();
 
       // 14b. Define and apply a parallel FGMRES solver for AU=B with a block
-      //      diagonal preconditioner based on the appropriate multigrid
-      //      preconditioner from hypre.
+      //     diagonal preconditioner based on the appropriate multigrid
+      //     preconditioner from hypre.
       Array<int> blockTrueOffsets;
       blockTrueOffsets.SetSize(3);
       blockTrueOffsets[0] = 0;
@@ -608,9 +609,10 @@ int main(int argc, char *argv[])
 }
 
 /**
-   Solves the eigenvalue problem -Div(Grad x) = lambda x with homogeneous
-   Dirichlet boundary conditions on the boundary of the domain. Returns mode
-   number "mode" (counting from zero) in the ParGridFunction "x".
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with
+   homogeneous Dirichlet boundary conditions on the boundary of the
+   domain. Returns mode number "mode" (counting from zero) in the
+   ParGridFunction "x".
 */
 void ScalarWaveGuide(int mode, ParGridFunction &x)
 {
@@ -665,10 +667,10 @@ void ScalarWaveGuide(int mode, ParGridFunction &x)
 }
 
 /**
-   Solves the eigenvalue problem -Curl(Curl x) = lambda x with homogeneous
-   Dirichlet boundary conditions, on the tangential component of x, on the
-   boundary of the domain. Returns mode number "mode" (counting from zero) in
-   the ParGridFunction "x".
+   Solves the eigenvalue problem -Curl(Curl x) = lambda x with
+   homogeneous Dirichlet boundary conditions, on the tangential
+   component of x, on the boundary of the domain. Returns mode number
+   "mode" (counting from zero) in the ParGridFunction "x".
 */
 void VectorWaveGuide(int mode, ParGridFunction &x)
 {
@@ -721,12 +723,13 @@ void VectorWaveGuide(int mode, ParGridFunction &x)
 }
 
 /**
-   Solves the eigenvalue problem -Div(Grad x) = lambda x with homogeneous
-   Neumann boundary conditions on the boundary of the domain. Returns mode
-   number "mode" (counting from zero) in the ParGridFunction "x_l2". Note that
-   mode 0 is a constant field so higher mode numbers are often more
-   interesting. The eigenmode is solved using continuous H1 basis of the
-   appropriate order and then projected onto the L2 basis and returned.
+   Solves the eigenvalue problem -Div(Grad x) = lambda x with
+   homogeneous Neumann boundary conditions on the boundary of the
+   domain. Returns mode number "mode" (counting from zero) in the
+   ParGridFunction "x_l2". Note that mode 0 is a constant field so
+   higher mode numbers are often more interesting. The eigenmode is
+   solved using continuous H1 basis of the appropriate order and then
+   projected onto the L2 basis and returned.
 */
 void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
 {
@@ -788,9 +791,9 @@ void PseudoScalarWaveGuide(int mode, ParGridFunction &x_l2)
    delete M;
 }
 
-// Compute eigenmode "mode" of either a Dirichlet or Neumann Laplacian or of a
-// Dirichlet curl curl operator based on the problem type and dimension of the
-// domain.
+// Compute eigenmode "mode" of either a Dirichlet or Neumann Laplacian
+// or of a Dirichlet curl curl operator based on the problem type and
+// dimension of the domain.
 void SetPortBC(int prob, int dim, int mode, ParGridFunction &port_bc)
 {
    switch (prob)

examples/ex36.cpp

@@ -1,10 +1,12 @@
 //                                MFEM Example 36
 //
+//
 // Compile with: make ex36
 //
 // Sample runs: ex36 -o 2
 //              ex36 -o 2 -r 4
 //
+//
 // Description: This example code demonstrates the use of MFEM to solve the
 //              bound-constrained energy minimization problem
 //
@@ -26,10 +28,12 @@
 //              order solutions to variation inequality problems and
 //              showcases how to set up and solve nonlinear mixed methods.
 //
+//
 // [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
 //     preserving finite element method for pointwise bound constraints.
 //     arXiv:2307.12444 [math.NA]
 
+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -59,7 +63,7 @@ public:
 class ExponentialGridFunctionCoefficient : public Coefficient
 {
 protected:
-   GridFunction *u;
+   GridFunction *u; // grid function
    Coefficient *obstacle;
    double min_val;
    double max_val;
@@ -84,7 +88,7 @@ int main(int argc, char *argv[])
 
    OptionsParser args(argc, argv);
    args.AddOption(&order, "-o", "--order",
-                  "Finite element order (polynomial degree).");
+                  "Finite element order (polynomial degree)");
    args.AddOption(&ref_levels, "-r", "--refs",
                   "Number of h-refinements.");
    args.AddOption(&max_it, "-mi", "--max-it",
@@ -194,7 +198,7 @@ int main(int argc, char *argv[])
    u_gf.ProjectCoefficient(IC_coef);
    u_old_gf = u_gf;
 
-   // 9. Initialize the slack variable ψₕ = ln(uₕ)
+   // 9. Initialize the slack variable ψₕ = exp(uₕ)
    LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
    psi_gf.ProjectCoefficient(ln_u);
    psi_old_gf = psi_gf;

examples/ex36p.cpp

@@ -1,10 +1,12 @@
-//                       MFEM Example 36 - Parallel Version
+//                         MFEM Example 36 - Parallel Version
+//
 //
 // Compile with: make ex36p
 //
 // Sample runs: mpirun -np 4 ex36p -o 2
 //              mpirun -np 4 ex36p -o 2 -r 4
 //
+//
 // Description: This example code demonstrates the use of MFEM to solve the
 //              bound-constrained energy minimization problem
 //
@@ -26,10 +28,12 @@
 //              order solutions to variation inequality problems and
 //              showcases how to set up and solve nonlinear mixed methods.
 //
+//
 // [1] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
 //     preserving finite element method for pointwise bound constraints.
 //     arXiv:2307.12444 [math.NA]
 
+
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
@@ -59,7 +63,7 @@ public:
 class ExponentialGridFunctionCoefficient : public Coefficient
 {
 protected:
-   GridFunction *u;
+   GridFunction *u; // grid function
    Coefficient *obstacle;
    double min_val;
    double max_val;
@@ -216,6 +220,7 @@ int main(int argc, char *argv[])
    u_old_gf = 0.0;
    psi_old_gf = 0.0;
 
+
    // 8. Define the function coefficients for the solution and use them to
    //    initialize the initial guess
    FunctionCoefficient exact_coef(exact_solution_obstacle);
@@ -226,7 +231,7 @@ int main(int argc, char *argv[])
    u_gf.ProjectCoefficient(IC_coef);
    u_old_gf = u_gf;
 
-   // 9. Initialize the slack variable ψₕ = ln(uₕ)
+   // 9. Initialize the slack variable ψₕ = exp(uₕ)
    LogarithmGridFunctionCoefficient ln_u(u_gf, obstacle);
    psi_gf.ProjectCoefficient(ln_u);
    psi_old_gf = psi_gf;

examples/ex37.cpp

@@ -1,466 +0,0 @@
-//                                MFEM Example 37
-//
-// Compile with: make ex37
-//
-// Sample runs:
-//     ex37 -alpha 10
-//     ex37 -alpha 10 -pv
-//     ex37 -lambda 0.1 -mu 0.1
-//     ex37 -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
-//     ex37 -r 6 -o 1 -alpha 25.0 -epsilon 0.02 -mi 50 -ntol 1e-5
-//
-// Description: This example code demonstrates the use of MFEM to solve a
-//              density-filtered [3] topology optimization problem. The
-//              objective is to minimize the compliance
-//
-//                  minimize ∫_Ω f⋅u dx over u ∈ [H¹(Ω)]² and ρ ∈ L¹(Ω)
-//
-//                  subject to
-//
-//                    -Div(r(ρ̃)Cε(u)) = f       in Ω + BCs
-//                    -ϵ²Δρ̃ + ρ̃ = ρ             in Ω + Neumann BCs
-//                    0 ≤ ρ ≤ 1                 in Ω
-//                    ∫_Ω ρ dx = θ vol(Ω)
-//
-//              Here, r(ρ̃) = ρ₀ + ρ̃³ (1-ρ₀) is the solid isotropic material
-//              penalization (SIMP) law, C is the elasticity tensor for an
-//              isotropic linearly elastic material, ϵ > 0 is the design
-//              length scale, and 0 < θ < 1 is the volume fraction.
-//
-//              The problem is discretized and gradients are computing using
-//              finite elements [1]. The design is optimized using an entropic
-//              mirror descent algorithm introduced by Keith and Surowiec [2]
-//              that is tailored to the bound constraint 0 ≤ ρ ≤ 1.
-//
-//              This example highlights the ability of MFEM to deliver high-
-//              order solutions to inverse design problems and showcases how
-//              to set up and solve PDE-constrained optimization problems
-//              using the so-called reduced space approach.
-//
-// [1] Andreassen, E., Clausen, A., Schevenels, M., Lazarov, B. S., & Sigmund, O.
-//     (2011). Efficient topology optimization in MATLAB using 88 lines of
-//     code. Structural and Multidisciplinary Optimization, 43(1), 1-16.
-// [2] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
-//     preserving finite element method for pointwise bound constraints.
-//     arXiv:2307.12444 [math.NA]
-// [3] Lazarov, B. S., & Sigmund, O. (2011). Filters in topology optimization
-//     based on Helmholtz‐type differential equations. International Journal
-//     for Numerical Methods in Engineering, 86(6), 765-781.
-
-#include "mfem.hpp"
-#include <iostream>
-#include <fstream>
-#include "ex37.hpp"
-
-using namespace std;
-using namespace mfem;
-
-/**
- * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
- *        ∫_Ω ρ dx = θ vol(Ω) as follows:
- *
- *        1. Compute the root of the R → R function
- *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
- *        2. Set ψ ← ψ + c.
- *
- * @param psi a GridFunction to be updated
- * @param target_volume θ vol(Ω)
- * @param tol Newton iteration tolerance
- * @param max_its Newton maximum iteration number
- * @return double Final volume, ∫_Ω sigmoid(ψ)
- */
-double proj(GridFunction &psi, double target_volume, double tol=1e-12,
-            int max_its=10)
-{
-   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
-   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
-
-   LinearForm int_sigmoid_psi(psi.FESpace());
-   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
-   LinearForm int_der_sigmoid_psi(psi.FESpace());
-   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
-                                              der_sigmoid_psi));
-   bool done = false;
-   for (int k=0; k<max_its; k++) // Newton iteration
-   {
-      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
-      const double f = int_sigmoid_psi.Sum() - target_volume;
-
-      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
-      const double df = int_der_sigmoid_psi.Sum();
-
-      const double dc = -f/df;
-      psi += dc;
-      if (abs(dc) < tol) { done = true; break; }
-   }
-   if (!done)
-   {
-      mfem_warning("Projection reached maximum iteration without converging. "
-                   "Result may not be accurate.");
-   }
-   int_sigmoid_psi.Assemble();
-   return int_sigmoid_psi.Sum();
-}
-
-/**
- * ---------------------------------------------------------------
- *                      ALGORITHM PREAMBLE
- * ---------------------------------------------------------------
- *
- *  The Lagrangian for this problem is
- *
- *          L(u,ρ,ρ̃,w,w̃) = (f,u) - (r(ρ̃) C ε(u),ε(w)) + (f,w)
- *                       - (ϵ² ∇ρ̃,∇w̃) - (ρ̃,w̃) + (ρ,w̃)
- *
- *  where
- *
- *    r(ρ̃) = ρ₀ + ρ̃³ (1 - ρ₀)       (SIMP rule)
- *
- *    ε(u) = (∇u + ∇uᵀ)/2           (symmetric gradient)
- *
- *    C e = λtr(e)I + 2μe           (isotropic material)
- *
- *  NOTE: The Lame parameters can be computed from Young's modulus E
- *        and Poisson's ratio ν as follows:
- *
- *             λ = E ν/((1+ν)(1-2ν)),      μ = E/(2(1+ν))
- *
- * ---------------------------------------------------------------
- *
- *  Discretization choices:
- *
- *     u ∈ V ⊂ (H¹)ᵈ (order p)
- *     ψ ∈ L² (order p - 1), ρ = sigmoid(ψ)
- *     ρ̃ ∈ H¹ (order p)
- *     w ∈ V  (order p)
- *     w̃ ∈ H¹ (order p)
- *
- * ---------------------------------------------------------------
- *                          ALGORITHM
- * ---------------------------------------------------------------
- *
- *  Update ρ with projected mirror descent via the following algorithm.
- *
- *  1. Initialize ψ = inv_sigmoid(vol_fraction) so that ∫ sigmoid(ψ) = θ vol(Ω)
- *
- *  While not converged:
- *
- *     2. Solve filter equation ∂_w̃ L = 0; i.e.,
- *
- *           (ϵ² ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)   ∀ v ∈ H¹.
- *
- *     3. Solve primal problem ∂_w L = 0; i.e.,
- *
- *      (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)   ∀ v ∈ V.
- *
- *     NB. The dual problem ∂_u L = 0 is the negative of the primal problem due to symmetry.
- *
- *     4. Solve for filtered gradient ∂_ρ̃ L = 0; i.e.,
- *
- *      (ϵ² ∇ w̃ , ∇ v ) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)   ∀ v ∈ H¹.
- *
- *     5. Project the gradient onto the discrete latent space; i.e., solve
- *
- *                         (G,v) = (w̃,v)   ∀ v ∈ L².
- *
- *     6. Bregman proximal gradient update; i.e.,
- *
- *                            ψ ← ψ - αG + c,
- *
- *     where α > 0 is a step size parameter and c ∈ R is a constant ensuring
- *
- *                     ∫_Ω sigmoid(ψ - αG + c) dx = θ vol(Ω).
- *
- *  end
- */
-
-int main(int argc, char *argv[])
-{
-   // 1. Parse command-line options.
-   int ref_levels = 5;
-   int order = 2;
-   double alpha = 1.0;
-   double epsilon = 0.01;
-   double vol_fraction = 0.5;
-   int max_it = 1e3;
-   double itol = 1e-1;
-   double ntol = 1e-4;
-   double rho_min = 1e-6;
-   double lambda = 1.0;
-   double mu = 1.0;
-   bool glvis_visualization = true;
-   bool paraview_output = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&alpha, "-alpha", "--alpha-step-length",
-                  "Step length for gradient descent.");
-   args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
-                  "Length scale for ρ.");
-   args.AddOption(&max_it, "-mi", "--max-it",
-                  "Maximum number of gradient descent iterations.");
-   args.AddOption(&ntol, "-ntol", "--rel-tol",
-                  "Normalized exit tolerance.");
-   args.AddOption(&itol, "-itol", "--abs-tol",
-                  "Increment exit tolerance.");
-   args.AddOption(&vol_fraction, "-vf", "--volume-fraction",
-                  "Volume fraction for the material density.");
-   args.AddOption(&lambda, "-lambda", "--lambda",
-                  "Lamé constant λ.");
-   args.AddOption(&mu, "-mu", "--mu",
-                  "Lamé constant μ.");
-   args.AddOption(&rho_min, "-rmin", "--psi-min",
-                  "Minimum of density coefficient.");
-   args.AddOption(&glvis_visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&paraview_output, "-pv", "--paraview", "-no-pv",
-                  "--no-paraview",
-                  "Enable or disable ParaView output.");
-   args.Parse();
-   if (!args.Good())
-   {
-      args.PrintUsage(mfem::out);
-      return 1;
-   }
-   args.PrintOptions(mfem::out);
-
-   Mesh mesh = Mesh::MakeCartesian2D(3, 1, mfem::Element::Type::QUADRILATERAL,
-                                     true, 3.0, 1.0);
-   int dim = mesh.Dimension();
-
-   // 2. Set BCs.
-   for (int i = 0; i<mesh.GetNBE(); i++)
-   {
-      Element * be = mesh.GetBdrElement(i);
-      Array<int> vertices;
-      be->GetVertices(vertices);
-
-      double * coords1 = mesh.GetVertex(vertices[0]);
-      double * coords2 = mesh.GetVertex(vertices[1]);
-
-      Vector center(2);
-      center(0) = 0.5*(coords1[0] + coords2[0]);
-      center(1) = 0.5*(coords1[1] + coords2[1]);
-
-      if (abs(center(0) - 0.0) < 1e-10)
-      {
-         // the left edge
-         be->SetAttribute(1);
-      }
-      else
-      {
-         // all other boundaries
-         be->SetAttribute(2);
-      }
-   }
-   mesh.SetAttributes();
-
-   // 3. Refine the mesh.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   // 4. Define the necessary finite element spaces on the mesh.
-   H1_FECollection state_fec(order, dim); // space for u
-   H1_FECollection filter_fec(order, dim); // space for ρ̃
-   L2_FECollection control_fec(order-1, dim,
-                               BasisType::GaussLobatto); // space for ψ
-   FiniteElementSpace state_fes(&mesh, &state_fec,dim);
-   FiniteElementSpace filter_fes(&mesh, &filter_fec);
-   FiniteElementSpace control_fes(&mesh, &control_fec);
-
-   int state_size = state_fes.GetTrueVSize();
-   int control_size = control_fes.GetTrueVSize();
-   int filter_size = filter_fes.GetTrueVSize();
-   mfem::out << "Number of state unknowns: " << state_size << std::endl;
-   mfem::out << "Number of filter unknowns: " << filter_size << std::endl;
-   mfem::out << "Number of control unknowns: " << control_size << std::endl;
-
-   // 5. Set the initial guess for ρ.
-   GridFunction u(&state_fes);
-   GridFunction psi(&control_fes);
-   GridFunction psi_old(&control_fes);
-   GridFunction rho_filter(&filter_fes);
-   u = 0.0;
-   rho_filter = vol_fraction;
-   psi = inv_sigmoid(vol_fraction);
-   psi_old = inv_sigmoid(vol_fraction);
-
-   // ρ = sigmoid(ψ)
-   MappedGridFunctionCoefficient rho(&psi, sigmoid);
-   // Interpolation of ρ = sigmoid(ψ) in control fes (for ParaView output)
-   GridFunction rho_gf(&control_fes);
-   // ρ - ρ_old = sigmoid(ψ) - sigmoid(ψ_old)
-   DiffMappedGridFunctionCoefficient succ_diff_rho(&psi, &psi_old, sigmoid);
-
-   // 6. Set-up the physics solver.
-   int maxat = mesh.bdr_attributes.Max();
-   Array<int> ess_bdr(maxat);
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient lambda_cf(lambda);
-   ConstantCoefficient mu_cf(mu);
-   LinearElasticitySolver * ElasticitySolver = new LinearElasticitySolver();
-   ElasticitySolver->SetMesh(&mesh);
-   ElasticitySolver->SetOrder(state_fec.GetOrder());
-   ElasticitySolver->SetupFEM();
-   Vector center(2); center(0) = 2.9; center(1) = 0.5;
-   Vector force(2); force(0) = 0.0; force(1) = -1.0;
-   double r = 0.05;
-   VolumeForceCoefficient vforce_cf(r,center,force);
-   ElasticitySolver->SetRHSCoefficient(&vforce_cf);
-   ElasticitySolver->SetEssentialBoundary(ess_bdr);
-
-   // 7. Set-up the filter solver.
-   ConstantCoefficient eps2_cf(epsilon*epsilon);
-   DiffusionSolver * FilterSolver = new DiffusionSolver();
-   FilterSolver->SetMesh(&mesh);
-   FilterSolver->SetOrder(filter_fec.GetOrder());
-   FilterSolver->SetDiffusionCoefficient(&eps2_cf);
-   FilterSolver->SetMassCoefficient(&one);
-   Array<int> ess_bdr_filter;
-   if (mesh.bdr_attributes.Size())
-   {
-      ess_bdr_filter.SetSize(mesh.bdr_attributes.Max());
-      ess_bdr_filter = 0;
-   }
-   FilterSolver->SetEssentialBoundary(ess_bdr_filter);
-   FilterSolver->SetupFEM();
-
-   BilinearForm mass(&control_fes);
-   mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
-   mass.Assemble();
-   SparseMatrix M;
-   Array<int> empty;
-   mass.FormSystemMatrix(empty,M);
-
-   // 8. Define the Lagrange multiplier and gradient functions.
-   GridFunction grad(&control_fes);
-   GridFunction w_filter(&filter_fes);
-
-   // 9. Define some tools for later.
-   ConstantCoefficient zero(0.0);
-   GridFunction onegf(&control_fes);
-   onegf = 1.0;
-   GridFunction zerogf(&control_fes);
-   zerogf = 0.0;
-   LinearForm vol_form(&control_fes);
-   vol_form.AddDomainIntegrator(new DomainLFIntegrator(one));
-   vol_form.Assemble();
-   double domain_volume = vol_form(onegf);
-   const double target_volume = domain_volume * vol_fraction;
-
-   // 10. Connect to GLVis. Prepare for VisIt output.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sout_r;
-   if (glvis_visualization)
-   {
-      sout_r.open(vishost, visport);
-      sout_r.precision(8);
-   }
-
-   mfem::ParaViewDataCollection paraview_dc("ex37", &mesh);
-   if (paraview_output)
-   {
-      rho_gf.ProjectCoefficient(rho);
-      paraview_dc.SetPrefixPath("ParaView");
-      paraview_dc.SetLevelsOfDetail(order);
-      paraview_dc.SetDataFormat(VTKFormat::BINARY);
-      paraview_dc.SetHighOrderOutput(true);
-      paraview_dc.SetCycle(0);
-      paraview_dc.SetTime(0.0);
-      paraview_dc.RegisterField("displacement",&u);
-      paraview_dc.RegisterField("density",&rho_gf);
-      paraview_dc.RegisterField("filtered_density",&rho_filter);
-      paraview_dc.Save();
-   }
-
-   // 11. Iterate:
-   for (int k = 1; k <= max_it; k++)
-   {
-      if (k > 1) { alpha *= ((double) k) / ((double) k-1); }
-
-      mfem::out << "\nStep = " << k << std::endl;
-
-      // Step 1 - Filter solve
-      // Solve (ϵ^2 ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)
-      FilterSolver->SetRHSCoefficient(&rho);
-      FilterSolver->Solve();
-      rho_filter = *FilterSolver->GetFEMSolution();
-
-      // Step 2 - State solve
-      // Solve (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)
-      SIMPInterpolationCoefficient SIMP_cf(&rho_filter,rho_min, 1.0);
-      ProductCoefficient lambda_SIMP_cf(lambda_cf,SIMP_cf);
-      ProductCoefficient mu_SIMP_cf(mu_cf,SIMP_cf);
-      ElasticitySolver->SetLameCoefficients(&lambda_SIMP_cf,&mu_SIMP_cf);
-      ElasticitySolver->Solve();
-      u = *ElasticitySolver->GetFEMSolution();
-
-      // Step 3 - Adjoint filter solve
-      // Solve (ϵ² ∇ w̃, ∇ v) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)
-      StrainEnergyDensityCoefficient rhs_cf(&lambda_cf,&mu_cf,&u, &rho_filter,
-                                            rho_min);
-      FilterSolver->SetRHSCoefficient(&rhs_cf);
-      FilterSolver->Solve();
-      w_filter = *FilterSolver->GetFEMSolution();
-
-      // Step 4 - Compute gradient
-      // Solve G = M⁻¹w̃
-      GridFunctionCoefficient w_cf(&w_filter);
-      LinearForm w_rhs(&control_fes);
-      w_rhs.AddDomainIntegrator(new DomainLFIntegrator(w_cf));
-      w_rhs.Assemble();
-      M.Mult(w_rhs,grad);
-
-      // Step 5 - Update design variable ψ ← proj(ψ - αG)
-      psi.Add(-alpha, grad);
-      const double material_volume = proj(psi, target_volume);
-
-      // Compute ||ρ - ρ_old|| in control fes.
-      double norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
-      double norm_reduced_gradient = norm_increment/alpha;
-      psi_old = psi;
-
-      double compliance = (*(ElasticitySolver->GetLinearForm()))(u);
-      mfem::out << "norm of the reduced gradient = " << norm_reduced_gradient <<
-                std::endl;
-      mfem::out << "norm of the increment = " << norm_increment << endl;
-      mfem::out << "compliance = " << compliance << std::endl;
-      mfem::out << "volume fraction = " << material_volume / domain_volume <<
-                std::endl;
-
-      if (glvis_visualization)
-      {
-         GridFunction r_gf(&filter_fes);
-         r_gf.ProjectCoefficient(SIMP_cf);
-         sout_r << "solution\n" << mesh << r_gf
-                << "window_title 'Design density r(ρ̃)'" << flush;
-      }
-
-      if (paraview_output)
-      {
-         rho_gf.ProjectCoefficient(rho);
-         paraview_dc.SetCycle(k);
-         paraview_dc.SetTime((double)k);
-         paraview_dc.Save();
-      }
-
-      if (norm_reduced_gradient < ntol && norm_increment < itol)
-      {
-         break;
-      }
-   }
-
-   delete ElasticitySolver;
-   delete FilterSolver;
-
-   return 0;
-}

examples/ex37.hpp

@@ -1,748 +0,0 @@
-//                  MFEM Example 37 - Serial/Parallel Shared Code
-
-#include "mfem.hpp"
-#include <fstream>
-#include <iostream>
-#include <functional>
-
-namespace mfem
-{
-
-/// @brief Inverse sigmoid function
-double inv_sigmoid(double x)
-{
-   double tol = 1e-12;
-   x = std::min(std::max(tol,x),1.0-tol);
-   return std::log(x/(1.0-x));
-}
-
-/// @brief Sigmoid function
-double sigmoid(double x)
-{
-   if (x >= 0)
-   {
-      return 1.0/(1.0+std::exp(-x));
-   }
-   else
-   {
-      return std::exp(x)/(1.0+std::exp(x));
-   }
-}
-
-/// @brief Derivative of sigmoid function
-double der_sigmoid(double x)
-{
-   double tmp = sigmoid(-x);
-   return tmp - std::pow(tmp,2);
-}
-
-/// @brief Returns f(u(x)) where u is a scalar GridFunction and f:R → R
-class MappedGridFunctionCoefficient : public GridFunctionCoefficient
-{
-protected:
-   std::function<double(const double)> fun; // f:R → R
-public:
-   MappedGridFunctionCoefficient()
-      :GridFunctionCoefficient(),
-       fun([](double x) {return x;}) {}
-   MappedGridFunctionCoefficient(const GridFunction *gf,
-                                 std::function<double(const double)> fun_,
-                                 int comp=1)
-      :GridFunctionCoefficient(gf, comp),
-       fun(fun_) {}
-
-
-   virtual double Eval(ElementTransformation &T,
-                       const IntegrationPoint &ip)
-   {
-      return fun(GridFunctionCoefficient::Eval(T, ip));
-   }
-   void SetFunction(std::function<double(const double)> fun_) { fun = fun_; }
-};
-
-
-/// @brief Returns f(u(x)) - f(v(x)) where u, v are scalar GridFunctions and f:R → R
-class DiffMappedGridFunctionCoefficient : public GridFunctionCoefficient
-{
-protected:
-   const GridFunction *OtherGridF;
-   GridFunctionCoefficient OtherGridF_cf;
-   std::function<double(const double)> fun; // f:R → R
-public:
-   DiffMappedGridFunctionCoefficient()
-      :GridFunctionCoefficient(),
-       OtherGridF(nullptr),
-       OtherGridF_cf(),
-       fun([](double x) {return x;}) {}
-   DiffMappedGridFunctionCoefficient(const GridFunction *gf,
-                                     const GridFunction *other_gf,
-                                     std::function<double(const double)> fun_,
-                                     int comp=1)
-      :GridFunctionCoefficient(gf, comp),
-       OtherGridF(other_gf),
-       OtherGridF_cf(OtherGridF),
-       fun(fun_) {}
-
-   virtual double Eval(ElementTransformation &T,
-                       const IntegrationPoint &ip)
-   {
-      const double value1 = fun(GridFunctionCoefficient::Eval(T, ip));
-      const double value2 = fun(OtherGridF_cf.Eval(T, ip));
-      return value1 - value2;
-   }
-   void SetFunction(std::function<double(const double)> fun_) { fun = fun_; }
-};
-
-/// @brief Solid isotropic material penalization (SIMP) coefficient
-class SIMPInterpolationCoefficient : public Coefficient
-{
-protected:
-   GridFunction *rho_filter;
-   double min_val;
-   double max_val;
-   double exponent;
-
-public:
-   SIMPInterpolationCoefficient(GridFunction *rho_filter_, double min_val_= 1e-6,
-                                double max_val_ = 1.0, double exponent_ = 3)
-      : rho_filter(rho_filter_), min_val(min_val_), max_val(max_val_),
-        exponent(exponent_) { }
-
-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
-   {
-      double val = rho_filter->GetValue(T, ip);
-      double coeff = min_val + pow(val,exponent)*(max_val-min_val);
-      return coeff;
-   }
-};
-
-
-/// @brief Strain energy density coefficient
-class StrainEnergyDensityCoefficient : public Coefficient
-{
-protected:
-   Coefficient * lambda=nullptr;
-   Coefficient * mu=nullptr;
-   GridFunction *u = nullptr; // displacement
-   GridFunction *rho_filter = nullptr; // filter density
-   DenseMatrix grad; // auxiliary matrix, used in Eval
-   double exponent;
-   double rho_min;
-
-public:
-   StrainEnergyDensityCoefficient(Coefficient *lambda_, Coefficient *mu_,
-                                  GridFunction * u_, GridFunction * rho_filter_, double rho_min_=1e-6,
-                                  double exponent_ = 3.0)
-      : lambda(lambda_), mu(mu_),  u(u_), rho_filter(rho_filter_),
-        exponent(exponent_), rho_min(rho_min_)
-   {
-      MFEM_ASSERT(rho_min_ >= 0.0, "rho_min must be >= 0");
-      MFEM_ASSERT(rho_min_ < 1.0,  "rho_min must be > 1");
-      MFEM_ASSERT(u, "displacement field is not set");
-      MFEM_ASSERT(rho_filter, "density field is not set");
-   }
-
-   virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip)
-   {
-      double L = lambda->Eval(T, ip);
-      double M = mu->Eval(T, ip);
-      u->GetVectorGradient(T, grad);
-      double div_u = grad.Trace();
-      double density = L*div_u*div_u;
-      int dim = T.GetSpaceDim();
-      for (int i=0; i<dim; i++)
-      {
-         for (int j=0; j<dim; j++)
-         {
-            density += M*grad(i,j)*(grad(i,j)+grad(j,i));
-         }
-      }
-      double val = rho_filter->GetValue(T,ip);
-
-      return -exponent * pow(val, exponent-1.0) * (1-rho_min) * density;
-   }
-};
-
-/// @brief Volumetric force for linear elasticity
-class VolumeForceCoefficient : public VectorCoefficient
-{
-private:
-   double r;
-   Vector center;
-   Vector force;
-public:
-   VolumeForceCoefficient(double r_,Vector &  center_, Vector & force_) :
-      VectorCoefficient(center_.Size()), r(r_), center(center_), force(force_) { }
-
-   using VectorCoefficient::Eval;
-
-   virtual void Eval(Vector &V, ElementTransformation &T,
-                     const IntegrationPoint &ip)
-   {
-      Vector xx; xx.SetSize(T.GetDimension());
-      T.Transform(ip,xx);
-      for (int i=0; i<xx.Size(); i++)
-      {
-         xx[i]=xx[i]-center[i];
-      }
-
-      double cr=xx.Norml2();
-      V.SetSize(T.GetDimension());
-      if (cr <= r)
-      {
-         V = force;
-      }
-      else
-      {
-         V = 0.0;
-      }
-   }
-
-   void Set(double r_,Vector & center_, Vector & force_)
-   {
-      r=r_;
-      center = center_;
-      force = force_;
-   }
-};
-
-/**
- * @brief Class for solving Poisson's equation:
- *
- *       - ∇ ⋅(κ ∇ u) = f  in Ω
- *
- */
-class DiffusionSolver
-{
-private:
-   Mesh * mesh = nullptr;
-   int order = 1;
-   // diffusion coefficient
-   Coefficient * diffcf = nullptr;
-   // mass coefficient
-   Coefficient * masscf = nullptr;
-   Coefficient * rhscf = nullptr;
-   Coefficient * essbdr_cf = nullptr;
-   Coefficient * neumann_cf = nullptr;
-   VectorCoefficient * gradient_cf = nullptr;
-
-   // FEM solver
-   int dim;
-   FiniteElementCollection * fec = nullptr;
-   FiniteElementSpace * fes = nullptr;
-   Array<int> ess_bdr;
-   Array<int> neumann_bdr;
-   GridFunction * u = nullptr;
-   LinearForm * b = nullptr;
-   bool parallel;
-#ifdef MFEM_USE_MPI
-   ParMesh * pmesh = nullptr;
-   ParFiniteElementSpace * pfes = nullptr;
-#endif
-
-public:
-   DiffusionSolver() { }
-   DiffusionSolver(Mesh * mesh_, int order_, Coefficient * diffcf_,
-                   Coefficient * cf_);
-
-   void SetMesh(Mesh * mesh_)
-   {
-      mesh = mesh_;
-      parallel = false;
-#ifdef MFEM_USE_MPI
-      pmesh = dynamic_cast<ParMesh *>(mesh);
-      if (pmesh) { parallel = true; }
-#endif
-   }
-   void SetOrder(int order_) { order = order_ ; }
-   void SetDiffusionCoefficient(Coefficient * diffcf_) { diffcf = diffcf_; }
-   void SetMassCoefficient(Coefficient * masscf_) { masscf = masscf_; }
-   void SetRHSCoefficient(Coefficient * rhscf_) { rhscf = rhscf_; }
-   void SetEssentialBoundary(const Array<int> & ess_bdr_) { ess_bdr = ess_bdr_;};
-   void SetNeumannBoundary(const Array<int> & neumann_bdr_) { neumann_bdr = neumann_bdr_;};
-   void SetNeumannData(Coefficient * neumann_cf_) {neumann_cf = neumann_cf_;}
-   void SetEssBdrData(Coefficient * essbdr_cf_) {essbdr_cf = essbdr_cf_;}
-   void SetGradientData(VectorCoefficient * gradient_cf_) {gradient_cf = gradient_cf_;}
-
-   void ResetFEM();
-   void SetupFEM();
-
-   void Solve();
-   GridFunction * GetFEMSolution();
-   LinearForm * GetLinearForm() {return b;}
-#ifdef MFEM_USE_MPI
-   ParGridFunction * GetParFEMSolution();
-   ParLinearForm * GetParLinearForm()
-   {
-      if (parallel)
-      {
-         return dynamic_cast<ParLinearForm *>(b);
-      }
-      else
-      {
-         MFEM_ABORT("Wrong code path. Call GetLinearForm");
-         return nullptr;
-      }
-   }
-#endif
-
-   ~DiffusionSolver();
-
-};
-
-/**
- * @brief Class for solving linear elasticity:
- *
- *        -∇ ⋅ σ(u) = f  in Ω  + BCs
- *
- *  where
- *
- *        σ(u) = λ ∇⋅u I + μ (∇ u + ∇uᵀ)
- *
- */
-class LinearElasticitySolver
-{
-private:
-   Mesh * mesh = nullptr;
-   int order = 1;
-   Coefficient * lambda_cf = nullptr;
-   Coefficient * mu_cf = nullptr;
-   VectorCoefficient * essbdr_cf = nullptr;
-   VectorCoefficient * rhs_cf = nullptr;
-
-   // FEM solver
-   int dim;
-   FiniteElementCollection * fec = nullptr;
-   FiniteElementSpace * fes = nullptr;
-   Array<int> ess_bdr;
-   Array<int> neumann_bdr;
-   GridFunction * u = nullptr;
-   LinearForm * b = nullptr;
-   bool parallel;
-#ifdef MFEM_USE_MPI
-   ParMesh * pmesh = nullptr;
-   ParFiniteElementSpace * pfes = nullptr;
-#endif
-
-public:
-   LinearElasticitySolver() { }
-   LinearElasticitySolver(Mesh * mesh_, int order_,
-                          Coefficient * lambda_cf_, Coefficient * mu_cf_);
-
-   void SetMesh(Mesh * mesh_)
-   {
-      mesh = mesh_;
-      parallel = false;
-#ifdef MFEM_USE_MPI
-      pmesh = dynamic_cast<ParMesh *>(mesh);
-      if (pmesh) { parallel = true; }
-#endif
-   }
-   void SetOrder(int order_) { order = order_ ; }
-   void SetLameCoefficients(Coefficient * lambda_cf_, Coefficient * mu_cf_) { lambda_cf = lambda_cf_; mu_cf = mu_cf_;  }
-   void SetRHSCoefficient(VectorCoefficient * rhs_cf_) { rhs_cf = rhs_cf_; }
-   void SetEssentialBoundary(const Array<int> & ess_bdr_) { ess_bdr = ess_bdr_;};
-   void SetNeumannBoundary(const Array<int> & neumann_bdr_) { neumann_bdr = neumann_bdr_;};
-   void SetEssBdrData(VectorCoefficient * essbdr_cf_) {essbdr_cf = essbdr_cf_;}
-
-   void ResetFEM();
-   void SetupFEM();
-
-   void Solve();
-   GridFunction * GetFEMSolution();
-   LinearForm * GetLinearForm() {return b;}
-#ifdef MFEM_USE_MPI
-   ParGridFunction * GetParFEMSolution();
-   ParLinearForm * GetParLinearForm()
-   {
-      if (parallel)
-      {
-         return dynamic_cast<ParLinearForm *>(b);
-      }
-      else
-      {
-         MFEM_ABORT("Wrong code path. Call GetLinearForm");
-         return nullptr;
-      }
-   }
-#endif
-
-   ~LinearElasticitySolver();
-
-};
-
-
-// Poisson solver
-
-DiffusionSolver::DiffusionSolver(Mesh * mesh_, int order_,
-                                 Coefficient * diffcf_, Coefficient * rhscf_)
-   : mesh(mesh_), order(order_), diffcf(diffcf_), rhscf(rhscf_)
-{
-
-#ifdef MFEM_USE_MPI
-   pmesh = dynamic_cast<ParMesh *>(mesh);
-   if (pmesh) { parallel = true; }
-#endif
-
-   SetupFEM();
-}
-
-void DiffusionSolver::SetupFEM()
-{
-   dim = mesh->Dimension();
-   fec = new H1_FECollection(order, dim);
-
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      pfes = new ParFiniteElementSpace(pmesh, fec);
-      u = new ParGridFunction(pfes);
-      b = new ParLinearForm(pfes);
-   }
-   else
-   {
-      fes = new FiniteElementSpace(mesh, fec);
-      u = new GridFunction(fes);
-      b = new LinearForm(fes);
-   }
-#else
-   fes = new FiniteElementSpace(mesh, fec);
-   u = new GridFunction(fes);
-   b = new LinearForm(fes);
-#endif
-   *u=0.0;
-
-   if (!ess_bdr.Size())
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh->bdr_attributes.Max());
-         ess_bdr = 1;
-      }
-   }
-}
-
-void DiffusionSolver::Solve()
-{
-   OperatorPtr A;
-   Vector B, X;
-   Array<int> ess_tdof_list;
-
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      pfes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
-   }
-   else
-   {
-      fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
-   }
-#else
-   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
-#endif
-   *u=0.0;
-   if (b)
-   {
-      delete b;
-#ifdef MFEM_USE_MPI
-      if (parallel)
-      {
-         b = new ParLinearForm(pfes);
-      }
-      else
-      {
-         b = new LinearForm(fes);
-      }
-#else
-      b = new LinearForm(fes);
-#endif
-   }
-   if (rhscf)
-   {
-      b->AddDomainIntegrator(new DomainLFIntegrator(*rhscf));
-   }
-   if (neumann_cf)
-   {
-      MFEM_VERIFY(neumann_bdr.Size(), "neumann_bdr attributes not provided");
-      b->AddBoundaryIntegrator(new BoundaryLFIntegrator(*neumann_cf),neumann_bdr);
-   }
-   else if (gradient_cf)
-   {
-      MFEM_VERIFY(neumann_bdr.Size(), "neumann_bdr attributes not provided");
-      b->AddBoundaryIntegrator(new BoundaryNormalLFIntegrator(*gradient_cf),
-                               neumann_bdr);
-   }
-
-   b->Assemble();
-
-   BilinearForm * a = nullptr;
-
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      a = new ParBilinearForm(pfes);
-   }
-   else
-   {
-      a = new BilinearForm(fes);
-   }
-#else
-   a = new BilinearForm(fes);
-#endif
-   a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
-   if (masscf)
-   {
-      a->AddDomainIntegrator(new MassIntegrator(*masscf));
-   }
-   a->Assemble();
-   if (essbdr_cf)
-   {
-      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
-   }
-   a->FormLinearSystem(ess_tdof_list, *u, *b, A, X, B);
-
-   CGSolver * cg = nullptr;
-   Solver * M = nullptr;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      M = new HypreBoomerAMG;
-      dynamic_cast<HypreBoomerAMG*>(M)->SetPrintLevel(0);
-      cg = new CGSolver(pmesh->GetComm());
-   }
-   else
-   {
-      M = new GSSmoother((SparseMatrix&)(*A));
-      cg = new CGSolver;
-   }
-#else
-   M = new GSSmoother((SparseMatrix&)(*A));
-   cg = new CGSolver;
-#endif
-   cg->SetRelTol(1e-12);
-   cg->SetMaxIter(10000);
-   cg->SetPrintLevel(0);
-   cg->SetPreconditioner(*M);
-   cg->SetOperator(*A);
-   cg->Mult(B, X);
-   delete M;
-   delete cg;
-   a->RecoverFEMSolution(X, *b, *u);
-   delete a;
-}
-
-GridFunction * DiffusionSolver::GetFEMSolution()
-{
-   return u;
-}
-
-#ifdef MFEM_USE_MPI
-ParGridFunction * DiffusionSolver::GetParFEMSolution()
-{
-   if (parallel)
-   {
-      return dynamic_cast<ParGridFunction*>(u);
-   }
-   else
-   {
-      MFEM_ABORT("Wrong code path. Call GetFEMSolution");
-      return nullptr;
-   }
-}
-#endif
-
-DiffusionSolver::~DiffusionSolver()
-{
-   delete u; u = nullptr;
-   delete fes; fes = nullptr;
-#ifdef MFEM_USE_MPI
-   delete pfes; pfes=nullptr;
-#endif
-   delete fec; fec = nullptr;
-   delete b;
-}
-
-
-// Elasticity solver
-
-LinearElasticitySolver::LinearElasticitySolver(Mesh * mesh_, int order_,
-                                               Coefficient * lambda_cf_, Coefficient * mu_cf_)
-   : mesh(mesh_), order(order_), lambda_cf(lambda_cf_), mu_cf(mu_cf_)
-{
-#ifdef MFEM_USE_MPI
-   pmesh = dynamic_cast<ParMesh *>(mesh);
-   if (pmesh) { parallel = true; }
-#endif
-   SetupFEM();
-}
-
-void LinearElasticitySolver::SetupFEM()
-{
-   dim = mesh->Dimension();
-   fec = new H1_FECollection(order, dim,BasisType::Positive);
-
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      pfes = new ParFiniteElementSpace(pmesh, fec, dim);
-      u = new ParGridFunction(pfes);
-      b = new ParLinearForm(pfes);
-   }
-   else
-   {
-      fes = new FiniteElementSpace(mesh, fec,dim);
-      u = new GridFunction(fes);
-      b = new LinearForm(fes);
-   }
-#else
-   fes = new FiniteElementSpace(mesh, fec, dim);
-   u = new GridFunction(fes);
-   b = new LinearForm(fes);
-#endif
-   *u=0.0;
-
-   if (!ess_bdr.Size())
-   {
-      if (mesh->bdr_attributes.Size())
-      {
-         ess_bdr.SetSize(mesh->bdr_attributes.Max());
-         ess_bdr = 1;
-      }
-   }
-}
-
-void LinearElasticitySolver::Solve()
-{
-   GridFunction * x = nullptr;
-   OperatorPtr A;
-   Vector B, X;
-   Array<int> ess_tdof_list;
-
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      x = new ParGridFunction(pfes);
-      pfes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
-   }
-   else
-   {
-      x = new GridFunction(fes);
-      fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
-   }
-#else
-   x = new GridFunction(fes);
-   fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
-#endif
-   *u=0.0;
-   if (b)
-   {
-      delete b;
-#ifdef MFEM_USE_MPI
-      if (parallel)
-      {
-         b = new ParLinearForm(pfes);
-      }
-      else
-      {
-         b = new LinearForm(fes);
-      }
-#else
-      b = new LinearForm(fes);
-#endif
-   }
-   if (rhs_cf)
-   {
-      b->AddDomainIntegrator(new VectorDomainLFIntegrator(*rhs_cf));
-   }
-
-   b->Assemble();
-
-   *x = 0.0;
-
-   BilinearForm * a = nullptr;
-
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      a = new ParBilinearForm(pfes);
-   }
-   else
-   {
-      a = new BilinearForm(fes);
-   }
-#else
-   a = new BilinearForm(fes);
-#endif
-   a->AddDomainIntegrator(new ElasticityIntegrator(*lambda_cf, *mu_cf));
-   a->Assemble();
-   if (essbdr_cf)
-   {
-      u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
-   }
-   a->FormLinearSystem(ess_tdof_list, *x, *b, A, X, B);
-
-   CGSolver * cg = nullptr;
-   Solver * M = nullptr;
-#ifdef MFEM_USE_MPI
-   if (parallel)
-   {
-      M = new HypreBoomerAMG;
-      dynamic_cast<HypreBoomerAMG*>(M)->SetPrintLevel(0);
-      cg = new CGSolver(pmesh->GetComm());
-   }
-   else
-   {
-      M = new GSSmoother((SparseMatrix&)(*A));
-      cg = new CGSolver;
-   }
-#else
-   M = new GSSmoother((SparseMatrix&)(*A));
-   cg = new CGSolver;
-#endif
-   cg->SetRelTol(1e-10);
-   cg->SetMaxIter(10000);
-   cg->SetPrintLevel(0);
-   cg->SetPreconditioner(*M);
-   cg->SetOperator(*A);
-   cg->Mult(B, X);
-   delete M;
-   delete cg;
-   a->RecoverFEMSolution(X, *b, *x);
-   *u+=*x;
-   delete a;
-   delete x;
-}
-
-GridFunction * LinearElasticitySolver::GetFEMSolution()
-{
-   return u;
-}
-
-#ifdef MFEM_USE_MPI
-ParGridFunction * LinearElasticitySolver::GetParFEMSolution()
-{
-   if (parallel)
-   {
-      return dynamic_cast<ParGridFunction*>(u);
-   }
-   else
-   {
-      MFEM_ABORT("Wrong code path. Call GetFEMSolution");
-      return nullptr;
-   }
-}
-#endif
-
-LinearElasticitySolver::~LinearElasticitySolver()
-{
-   delete u; u = nullptr;
-   delete fes; fes = nullptr;
-#ifdef MFEM_USE_MPI
-   delete pfes; pfes=nullptr;
-#endif
-   delete fec; fec = nullptr;
-   delete b;
-}
-
-} // namespace mfem
-

examples/ex37p.cpp

@@ -1,497 +0,0 @@
-//                       MFEM Example 37 - Parallel Version
-//
-// Compile with: make ex37p
-//
-// Sample runs:
-//    mpirun -np 4 ex37p -alpha 10 -pv
-//    mpirun -np 4 ex37p -lambda 0.1 -mu 0.1
-//    mpirun -np 4 ex37p -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
-//    mpirun -np 4 ex37p -r 6 -o 2 -alpha 10.0 -epsilon 0.02 -mi 50 -ntol 1e-5
-//
-// Description: This example code demonstrates the use of MFEM to solve a
-//              density-filtered [3] topology optimization problem. The
-//              objective is to minimize the compliance
-//
-//                  minimize ∫_Ω f⋅u dx over u ∈ [H¹(Ω)]² and ρ ∈ L¹(Ω)
-//
-//                  subject to
-//
-//                    -Div(r(ρ̃)Cε(u)) = f       in Ω + BCs
-//                    -ϵ²Δρ̃ + ρ̃ = ρ             in Ω + Neumann BCs
-//                    0 ≤ ρ ≤ 1                 in Ω
-//                    ∫_Ω ρ dx = θ vol(Ω)
-//
-//              Here, r(ρ̃) = ρ₀ + ρ̃³ (1-ρ₀) is the solid isotropic material
-//              penalization (SIMP) law, C is the elasticity tensor for an
-//              isotropic linearly elastic material, ϵ > 0 is the design
-//              length scale, and 0 < θ < 1 is the volume fraction.
-//
-//              The problem is discretized and gradients are computing using
-//              finite elements [1]. The design is optimized using an entropic
-//              mirror descent algorithm introduced by Keith and Surowiec [2]
-//              that is tailored to the bound constraint 0 ≤ ρ ≤ 1.
-//
-//              This example highlights the ability of MFEM to deliver high-
-//              order solutions to inverse design problems and showcases how
-//              to set up and solve PDE-constrained optimization problems
-//              using the so-called reduced space approach.
-//
-// [1] Andreassen, E., Clausen, A., Schevenels, M., Lazarov, B. S., & Sigmund, O.
-//     (2011). Efficient topology optimization in MATLAB using 88 lines of
-//     code. Structural and Multidisciplinary Optimization, 43(1), 1-16.
-// [2] Keith, B. and Surowiec, T. (2023) Proximal Galerkin: A structure-
-//     preserving finite element method for pointwise bound constraints.
-//     arXiv:2307.12444 [math.NA]
-// [3] Lazarov, B. S., & Sigmund, O. (2011). Filters in topology optimization
-//     based on Helmholtz‐type differential equations. International Journal
-//     for Numerical Methods in Engineering, 86(6), 765-781.
-
-#include "mfem.hpp"
-#include <iostream>
-#include <fstream>
-#include "ex37.hpp"
-
-using namespace std;
-using namespace mfem;
-
-/**
- * @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
- *        ∫_Ω ρ dx = θ vol(Ω) as follows:
- *
- *        1. Compute the root of the R → R function
- *            f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
- *        2. Set ψ ← ψ + c.
- *
- * @param psi a GridFunction to be updated
- * @param target_volume θ vol(Ω)
- * @param tol Newton iteration tolerance
- * @param max_its Newton maximum iteration number
- * @return double Final volume, ∫_Ω sigmoid(ψ)
- */
-double proj(ParGridFunction &psi, double target_volume, double tol=1e-12,
-            int max_its=10)
-{
-   MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
-   MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
-
-   ParLinearForm int_sigmoid_psi(psi.ParFESpace());
-   int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
-   ParLinearForm int_der_sigmoid_psi(psi.ParFESpace());
-   int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
-                                              der_sigmoid_psi));
-   bool done = false;
-   for (int k=0; k<max_its; k++) // Newton iteration
-   {
-      int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
-      double f = int_sigmoid_psi.Sum();
-      MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-      f -= target_volume;
-
-      int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
-      double df = int_der_sigmoid_psi.Sum();
-      MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
-
-      const double dc = -f/df;
-      psi += dc;
-      if (abs(dc) < tol) { done = true; break; }
-   }
-   if (!done)
-   {
-      mfem_warning("Projection reached maximum iteration without converging. "
-                   "Result may not be accurate.");
-   }
-   int_sigmoid_psi.Assemble();
-   double material_volume = int_sigmoid_psi.Sum();
-   MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1, MPI_DOUBLE, MPI_SUM,
-                 MPI_COMM_WORLD);
-   return material_volume;
-}
-
-/**
- * ---------------------------------------------------------------
- *                      ALGORITHM PREAMBLE
- * ---------------------------------------------------------------
- *
- *  The Lagrangian for this problem is
- *
- *          L(u,ρ,ρ̃,w,w̃) = (f,u) - (r(ρ̃) C ε(u),ε(w)) + (f,w)
- *                       - (ϵ² ∇ρ̃,∇w̃) - (ρ̃,w̃) + (ρ,w̃)
- *
- *  where
- *
- *    r(ρ̃) = ρ₀ + ρ̃³ (1 - ρ₀)       (SIMP rule)
- *
- *    ε(u) = (∇u + ∇uᵀ)/2           (symmetric gradient)
- *
- *    C e = λtr(e)I + 2μe           (isotropic material)
- *
- *  NOTE: The Lame parameters can be computed from Young's modulus E
- *        and Poisson's ratio ν as follows:
- *
- *             λ = E ν/((1+ν)(1-2ν)),      μ = E/(2(1+ν))
- *
- * ---------------------------------------------------------------
- *
- *  Discretization choices:
- *
- *     u ∈ V ⊂ (H¹)ᵈ (order p)
- *     ψ ∈ L² (order p - 1), ρ = sigmoid(ψ)
- *     ρ̃ ∈ H¹ (order p)
- *     w ∈ V  (order p)
- *     w̃ ∈ H¹ (order p)
- *
- * ---------------------------------------------------------------
- *                          ALGORITHM
- * ---------------------------------------------------------------
- *
- *  Update ρ with projected mirror descent via the following algorithm.
- *
- *  1. Initialize ψ = inv_sigmoid(vol_fraction) so that ∫ sigmoid(ψ) = θ vol(Ω)
- *
- *  While not converged:
- *
- *     2. Solve filter equation ∂_w̃ L = 0; i.e.,
- *
- *           (ϵ² ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)   ∀ v ∈ H¹.
- *
- *     3. Solve primal problem ∂_w L = 0; i.e.,
- *
- *      (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)   ∀ v ∈ V.
- *
- *     NB. The dual problem ∂_u L = 0 is the negative of the primal problem due to symmetry.
- *
- *     4. Solve for filtered gradient ∂_ρ̃ L = 0; i.e.,
- *
- *      (ϵ² ∇ w̃ , ∇ v ) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)   ∀ v ∈ H¹.
- *
- *     5. Project the gradient onto the discrete latent space; i.e., solve
- *
- *                         (G,v) = (w̃,v)   ∀ v ∈ L².
- *
- *     6. Bregman proximal gradient update; i.e.,
- *
- *                            ψ ← ψ - αG + c,
- *
- *     where α > 0 is a step size parameter and c ∈ R is a constant ensuring
- *
- *                     ∫_Ω sigmoid(ψ - αG + c) dx = θ vol(Ω).
- *
- *  end
- */
-
-int main(int argc, char *argv[])
-{
-   // 0. Initialize MPI and HYPRE.
-   Mpi::Init();
-   int num_procs = Mpi::WorldSize();
-   int myid = Mpi::WorldRank();
-   Hypre::Init();
-
-   // 1. Parse command-line options.
-   int ref_levels = 5;
-   int order = 2;
-   double alpha = 1.0;
-   double epsilon = 0.01;
-   double vol_fraction = 0.5;
-   int max_it = 1e3;
-   double itol = 1e-1;
-   double ntol = 1e-4;
-   double rho_min = 1e-6;
-   double lambda = 1.0;
-   double mu = 1.0;
-   bool glvis_visualization = true;
-   bool paraview_output = false;
-
-   OptionsParser args(argc, argv);
-   args.AddOption(&ref_levels, "-r", "--refine",
-                  "Number of times to refine the mesh uniformly.");
-   args.AddOption(&order, "-o", "--order",
-                  "Order (degree) of the finite elements.");
-   args.AddOption(&alpha, "-alpha", "--alpha-step-length",
-                  "Step length for gradient descent.");
-   args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
-                  "Length scale for ρ.");
-   args.AddOption(&max_it, "-mi", "--max-it",
-                  "Maximum number of gradient descent iterations.");
-   args.AddOption(&ntol, "-ntol", "--rel-tol",
-                  "Normalized exit tolerance.");
-   args.AddOption(&itol, "-itol", "--abs-tol",
-                  "Increment exit tolerance.");
-   args.AddOption(&vol_fraction, "-vf", "--volume-fraction",
-                  "Volume fraction for the material density.");
-   args.AddOption(&lambda, "-lambda", "--lambda",
-                  "Lamé constant λ.");
-   args.AddOption(&mu, "-mu", "--mu",
-                  "Lamé constant μ.");
-   args.AddOption(&rho_min, "-rmin", "--psi-min",
-                  "Minimum of density coefficient.");
-   args.AddOption(&glvis_visualization, "-vis", "--visualization", "-no-vis",
-                  "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&paraview_output, "-pv", "--paraview", "-no-pv",
-                  "--no-paraview",
-                  "Enable or disable ParaView output.");
-   args.Parse();
-   if (!args.Good())
-   {
-      if (myid == 0)
-      {
-         args.PrintUsage(cout);
-      }
-      MPI_Finalize();
-      return 1;
-   }
-   if (myid == 0)
-   {
-      mfem::out << num_procs << " number of process created.\n";
-      args.PrintOptions(cout);
-   }
-
-   Mesh mesh = Mesh::MakeCartesian2D(3, 1, mfem::Element::Type::QUADRILATERAL,
-                                     true, 3.0, 1.0);
-   int dim = mesh.Dimension();
-
-   // 2. Set BCs.
-   for (int i = 0; i<mesh.GetNBE(); i++)
-   {
-      Element * be = mesh.GetBdrElement(i);
-      Array<int> vertices;
-      be->GetVertices(vertices);
-
-      double * coords1 = mesh.GetVertex(vertices[0]);
-      double * coords2 = mesh.GetVertex(vertices[1]);
-
-      Vector center(2);
-      center(0) = 0.5*(coords1[0] + coords2[0]);
-      center(1) = 0.5*(coords1[1] + coords2[1]);
-
-      if (abs(center(0) - 0.0) < 1e-10)
-      {
-         // the left edge
-         be->SetAttribute(1);
-      }
-      else
-      {
-         // all other boundaries
-         be->SetAttribute(2);
-      }
-   }
-   mesh.SetAttributes();
-
-   // 3. Refine the mesh.
-   for (int lev = 0; lev < ref_levels; lev++)
-   {
-      mesh.UniformRefinement();
-   }
-
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
-   mesh.Clear();
-
-   // 4. Define the necessary finite element spaces on the mesh.
-   H1_FECollection state_fec(order, dim); // space for u
-   H1_FECollection filter_fec(order, dim); // space for ρ̃
-   L2_FECollection control_fec(order-1, dim,
-                               BasisType::GaussLobatto); // space for ψ
-   ParFiniteElementSpace state_fes(&pmesh, &state_fec,dim);
-   ParFiniteElementSpace filter_fes(&pmesh, &filter_fec);
-   ParFiniteElementSpace control_fes(&pmesh, &control_fec);
-
-   HYPRE_BigInt state_size = state_fes.GlobalTrueVSize();
-   HYPRE_BigInt control_size = control_fes.GlobalTrueVSize();
-   HYPRE_BigInt filter_size = filter_fes.GlobalTrueVSize();
-   if (myid==0)
-   {
-      cout << "Number of state unknowns: " << state_size << endl;
-      cout << "Number of filter unknowns: " << filter_size << endl;
-      cout << "Number of control unknowns: " << control_size << endl;
-   }
-
-   // 5. Set the initial guess for ρ.
-   ParGridFunction u(&state_fes);
-   ParGridFunction psi(&control_fes);
-   ParGridFunction psi_old(&control_fes);
-   ParGridFunction rho_filter(&filter_fes);
-   u = 0.0;
-   rho_filter = vol_fraction;
-   psi = inv_sigmoid(vol_fraction);
-   psi_old = inv_sigmoid(vol_fraction);
-
-   // ρ = sigmoid(ψ)
-   MappedGridFunctionCoefficient rho(&psi, sigmoid);
-   // Interpolation of ρ = sigmoid(ψ) in control fes (for ParaView output)
-   ParGridFunction rho_gf(&control_fes);
-   // ρ - ρ_old = sigmoid(ψ) - sigmoid(ψ_old)
-   DiffMappedGridFunctionCoefficient succ_diff_rho(&psi, &psi_old, sigmoid);
-
-   // 6. Set-up the physics solver.
-   int maxat = pmesh.bdr_attributes.Max();
-   Array<int> ess_bdr(maxat);
-   ess_bdr = 0;
-   ess_bdr[0] = 1;
-   ConstantCoefficient one(1.0);
-   ConstantCoefficient lambda_cf(lambda);
-   ConstantCoefficient mu_cf(mu);
-   LinearElasticitySolver * ElasticitySolver = new LinearElasticitySolver();
-   ElasticitySolver->SetMesh(&pmesh);
-   ElasticitySolver->SetOrder(state_fec.GetOrder());
-   ElasticitySolver->SetupFEM();
-   Vector center(2); center(0) = 2.9; center(1) = 0.5;
-   Vector force(2); force(0) = 0.0; force(1) = -1.0;
-   double r = 0.05;
-   VolumeForceCoefficient vforce_cf(r,center,force);
-   ElasticitySolver->SetRHSCoefficient(&vforce_cf);
-   ElasticitySolver->SetEssentialBoundary(ess_bdr);
-
-   // 7. Set-up the filter solver.
-   ConstantCoefficient eps2_cf(epsilon*epsilon);
-   DiffusionSolver * FilterSolver = new DiffusionSolver();
-   FilterSolver->SetMesh(&pmesh);
-   FilterSolver->SetOrder(filter_fec.GetOrder());
-   FilterSolver->SetDiffusionCoefficient(&eps2_cf);
-   FilterSolver->SetMassCoefficient(&one);
-   Array<int> ess_bdr_filter;
-   if (pmesh.bdr_attributes.Size())
-   {
-      ess_bdr_filter.SetSize(pmesh.bdr_attributes.Max());
-      ess_bdr_filter = 0;
-   }
-   FilterSolver->SetEssentialBoundary(ess_bdr_filter);
-   FilterSolver->SetupFEM();
-
-   ParBilinearForm mass(&control_fes);
-   mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
-   mass.Assemble();
-   HypreParMatrix M;
-   Array<int> empty;
-   mass.FormSystemMatrix(empty,M);
-
-   // 8. Define the Lagrange multiplier and gradient functions.
-   ParGridFunction grad(&control_fes);
-   ParGridFunction w_filter(&filter_fes);
-
-   // 9. Define some tools for later.
-   ConstantCoefficient zero(0.0);
-   ParGridFunction onegf(&control_fes);
-   onegf = 1.0;
-   ParGridFunction zerogf(&control_fes);
-   zerogf = 0.0;
-   ParLinearForm vol_form(&control_fes);
-   vol_form.AddDomainIntegrator(new DomainLFIntegrator(one));
-   vol_form.Assemble();
-   double domain_volume = vol_form(onegf);
-   const double target_volume = domain_volume * vol_fraction;
-
-   // 10. Connect to GLVis. Prepare for VisIt output.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
-   socketstream sout_r;
-   if (glvis_visualization)
-   {
-      sout_r.open(vishost, visport);
-      sout_r.precision(8);
-   }
-
-   mfem::ParaViewDataCollection paraview_dc("ex37p", &pmesh);
-   if (paraview_output)
-   {
-      rho_gf.ProjectCoefficient(rho);
-      paraview_dc.SetPrefixPath("ParaView");
-      paraview_dc.SetLevelsOfDetail(order);
-      paraview_dc.SetDataFormat(VTKFormat::BINARY);
-      paraview_dc.SetHighOrderOutput(true);
-      paraview_dc.SetCycle(0);
-      paraview_dc.SetTime(0.0);
-      paraview_dc.RegisterField("displacement",&u);
-      paraview_dc.RegisterField("density",&rho_gf);
-      paraview_dc.RegisterField("filtered_density",&rho_filter);
-      paraview_dc.Save();
-   }
-
-   // 11. Iterate:
-   for (int k = 1; k <= max_it; k++)
-   {
-      if (k > 1) { alpha *= ((double) k) / ((double) k-1); }
-
-      if (myid == 0)
-      {
-         cout << "\nStep = " << k << endl;
-      }
-
-      // Step 1 - Filter solve
-      // Solve (ϵ^2 ∇ ρ̃, ∇ v ) + (ρ̃,v) = (ρ,v)
-      FilterSolver->SetRHSCoefficient(&rho);
-      FilterSolver->Solve();
-      rho_filter = *FilterSolver->GetFEMSolution();
-
-      // Step 2 - State solve
-      // Solve (λ r(ρ̃) ∇⋅u, ∇⋅v) + (2 μ r(ρ̃) ε(u), ε(v)) = (f,v)
-      SIMPInterpolationCoefficient SIMP_cf(&rho_filter,rho_min, 1.0);
-      ProductCoefficient lambda_SIMP_cf(lambda_cf,SIMP_cf);
-      ProductCoefficient mu_SIMP_cf(mu_cf,SIMP_cf);
-      ElasticitySolver->SetLameCoefficients(&lambda_SIMP_cf,&mu_SIMP_cf);
-      ElasticitySolver->Solve();
-      u = *ElasticitySolver->GetFEMSolution();
-
-      // Step 3 - Adjoint filter solve
-      // Solve (ϵ² ∇ w̃, ∇ v) + (w̃ ,v) = (-r'(ρ̃) ( λ |∇⋅u|² + 2 μ |ε(u)|²),v)
-      StrainEnergyDensityCoefficient rhs_cf(&lambda_cf,&mu_cf,&u, &rho_filter,
-                                            rho_min);
-      FilterSolver->SetRHSCoefficient(&rhs_cf);
-      FilterSolver->Solve();
-      w_filter = *FilterSolver->GetFEMSolution();
-
-      // Step 4 - Compute gradient
-      // Solve G = M⁻¹w̃
-      GridFunctionCoefficient w_cf(&w_filter);
-      ParLinearForm w_rhs(&control_fes);
-      w_rhs.AddDomainIntegrator(new DomainLFIntegrator(w_cf));
-      w_rhs.Assemble();
-      M.Mult(w_rhs,grad);
-
-      // Step 5 - Update design variable ψ ← proj(ψ - αG)
-      psi.Add(-alpha, grad);
-      const double material_volume = proj(psi, target_volume);
-
-      // Compute ||ρ - ρ_old|| in control fes.
-      double norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
-      double norm_reduced_gradient = norm_increment/alpha;
-      psi_old = psi;
-
-      double compliance = (*(ElasticitySolver->GetLinearForm()))(u);
-      MPI_Allreduce(MPI_IN_PLACE,&compliance,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
-      if (myid == 0)
-      {
-         mfem::out << "norm of the reduced gradient = " << norm_reduced_gradient << endl;
-         mfem::out << "norm of the increment = " << norm_increment << endl;
-         mfem::out << "compliance = " << compliance << endl;
-         mfem::out << "volume fraction = " << material_volume / domain_volume << endl;
-      }
-
-      if (glvis_visualization)
-      {
-         ParGridFunction r_gf(&filter_fes);
-         r_gf.ProjectCoefficient(SIMP_cf);
-         sout_r << "parallel " << num_procs << " " << myid << "\n";
-         sout_r << "solution\n" << pmesh << r_gf
-                << "window_title 'Design density r(ρ̃)'" << flush;
-      }
-
-      if (paraview_output)
-      {
-         rho_gf.ProjectCoefficient(rho);
-         paraview_dc.SetCycle(k);
-         paraview_dc.SetTime((double)k);
-         paraview_dc.Save();
-      }
-
-      if (norm_reduced_gradient < ntol && norm_increment < itol)
-      {
-         break;
-      }
-   }
-
-   delete ElasticitySolver;
-   delete FilterSolver;
-
-   return 0;
-}

examples/makefile

@@ -23,11 +23,10 @@ MFEM_LIB_FILE = mfem_is_not_built
 
 SEQ_EXAMPLES = ex0 ex1 ex2 ex3 ex4 ex5 ex6 ex7 ex8 ex9 ex10 ex14 ex15 ex16 \
   ex17 ex18 ex19 ex20 ex21 ex22 ex23 ex24 ex25 ex26 ex27 ex28 ex29 ex30 \
-  ex31 ex33 ex34 ex36 ex37
+  ex31 ex33 ex34 ex36
 PAR_EXAMPLES = ex0p ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex8p ex9p ex10p ex11p \
   ex12p ex13p ex14p ex15p ex16p ex17p ex18p ex19p ex20p ex21p ex22p ex24p \
-  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p \
-	ex37p
+  ex25p ex26p ex27p ex28p ex29p ex30p ex31p ex32p ex33p ex34p ex35p ex36p
 SEQ_DEVICE_EXAMPLES = ex1 ex3 ex4 ex5 ex6 ex9 ex22 ex24 ex25 ex26 ex34
 PAR_DEVICE_EXAMPLES = ex1p ex2p ex3p ex4p ex5p ex6p ex7p ex9p ex13p ex22p \
   ex24p ex25p ex26p ex34p ex35p
@@ -93,12 +92,10 @@ $(SUBDIRS_TPRINT):
 # Additional dependencies
 ex18: $(SRC)ex18.hpp
 ex33: $(SRC)ex33.hpp
-ex37: $(SRC)ex37.hpp
 
 ifeq ($(MFEM_USE_MPI),YES)
 ex18p: $(SRC)ex18.hpp
 ex33p: $(SRC)ex33.hpp
-ex37p: $(SRC)ex37.hpp
 endif
 
 MFEM_TESTS = EXAMPLES
@@ -142,10 +139,6 @@ ex27-test-seq: ex27
 	@$(call mfem-test,$<,, Serial example,-dg)
 ex27p-test-par: ex27p
 	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-dg)
-ex37-test-seq: ex37
-	@$(call mfem-test,$<,, Serial example,-mi 3)
-ex37p-test-par: ex37p
-	@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-mi 3)
 # Testing: optional tests
 ifeq ($(MFEM_USE_STRUMPACK),YES)
 ex11p-test-strumpack: ex11p

examples/petsc/CMakeLists.txt

@@ -24,13 +24,13 @@ if (MFEM_USE_MPI)
     ex10p.cpp
     )
   list(APPEND PETSC_RC_FILES
-    rc_ex1p rc_ex1p_device rc_ex1p_deviceamg
-    rc_ex2p rc_ex2p_bddc rc_ex2p_asm
+    rc_ex1p
+    rc_ex2p
     rc_ex3p rc_ex3p_bddc
     rc_ex4p rc_ex4p_bddc
     rc_ex5p_bddc rc_ex5p_fieldsplit
-    rc_ex9p_expl rc_ex9p_expl_device rc_ex9p_impl
-    rc_ex10p rc_ex10p_mf rc_ex10p_mfop rc_ex10p_jfnk
+    rc_ex9p_expl rc_ex9p_impl
+    rc_ex10p
     )
 endif()
 
@@ -39,7 +39,7 @@ if (MFEM_USE_SLEPC)
     ex11p.cpp
     )
   list(APPEND PETSC_RC_FILES
-    rc_ex11p_lobpcg rc_ex11p_lobpcg_device rc_ex11p_gd
+    rc_ex11p_lobpcg rc_ex11p_gd
     )
 endif()
 
@@ -74,13 +74,7 @@ add_mfem_examples(PETSC_EXAMPLES_SRCS ${PFX} copy_petsc_rc_files test_petsc)
 # Command line options for the tests.
 set(EX1_ARGS_W       -m ../../data/amr-quad.mesh --usepetsc)
 set(EX1_ARGS_P       -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p)
-set(EX1_ARGS_CUDA    -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_device)
-set(EX1_ARGS_CUDAAMG -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_deviceamg)
-set(EX1_ARGS_HIP     -m ../../data/star.mesh --usepetsc --partial-assembly --device hip --petscopts rc_ex1p_device)
-set(EX1_ARGS_HIPAMG  -m ../../data/star.mesh --usepetsc --device hip --petscopts rc_ex1p_deviceamg)
 set(EX2_ARGS         -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p)
-set(EX2_ARGS_BDDC    -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc)
-set(EX2_ARGS_ASM     -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm)
 set(EX3_ARGS         -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping)
 set(EX4_ARGS         -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping)
 set(EX4_HYB_ARGS     -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization)
@@ -91,46 +85,22 @@ set(EX6_ARGS         -m ../../data/amr-quad.mesh --usepetsc)
 set(EX6_NONOVL_ARGS  -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping)
 set(EX9_E_ARGS       -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1)
 set(EX9_ES_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step)
-set(EX9_ES_ARGS_CUDA -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device cuda)
-set(EX9_ES_ARGS_HIP  -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device hip)
 set(EX9_IS_ARGS      -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5)
 set(EX10_ARGS        -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3)
-set(EX10_MF_ARGS     -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3)
-set(EX10_MFOP_ARGS   -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mfop -tf 6 -s 3 -rs 0 -dt 3)
-set(EX10_JFNK_ARGS   -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3)
 if (MFEM_USE_SLEPC)
   set(EX11_ARGS_SINV   -m ../../data/star.mesh --useslepc)
   set(EX11_ARGS_LOBPCG -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg)
-  set(EX11_ARGS_LOBPCG_CUDA -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device cuda)
-  set(EX11_ARGS_LOBPCG_HIP  -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device hip)
   set(EX11_ARGS_GD -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd)
 endif()
 
 # Add the tests: one test per command-line-variable.
 if (MFEM_ENABLE_TESTING)
   set(TEST_OPTIONS_VARS
-      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX2_ARGS_BDDC EX2_ARGS_ASM EX3_ARGS
-      EX4_ARGS EX4_HYB_ARGS EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS
-      EX6_ARGS EX6_NONOVL_ARGS EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS
-      EX10_MF_ARGS EX10_MFOP_ARGS EX10_JFNK_ARGS)
+      EX1_ARGS_W EX1_ARGS_P EX2_ARGS EX3_ARGS EX4_ARGS EX4_HYB_ARGS
+      EX5_BDDC_LB_ARGS EX5_BDDC_GB_ARGS EX5_FSPL_ARGS EX6_ARGS EX6_NONOVL_ARGS
+      EX9_E_ARGS EX9_ES_ARGS EX9_IS_ARGS EX10_ARGS)
   if (MFEM_USE_SLEPC)
-    list(APPEND TEST_OPTIONS_VARS
-      EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
-  endif()
-  # CUDA/HIP tests
-  if (MFEM_USE_CUDA)
-    list(APPEND TEST_OPTIONS_VARS
-      EX1_ARGS_CUDA EX1_ARGS_CUDAAMG EX9_ES_ARGS_CUDA)
-    if (MFEM_USE_SLEPC)
-      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_LOBPCG_CUDA)
-    endif()
-  elseif (MFEM_USE_HIP)
-    list(APPEND TEST_OPTIONS_VARS
-      EX1_ARGS_HIP EX1_ARGS_HIPAMG EX9_ES_ARGS_HIP)
-    if (MFEM_USE_SLEPC)
-      # SLEPc does not support BVSVEC with HIP
-      # list(APPEND TEST_OPTIONS_VARS EX11_ARGS_LOBPCG_HIP)
-    endif()
+      list(APPEND TEST_OPTIONS_VARS EX11_ARGS_SINV EX11_ARGS_LOBPCG EX11_ARGS_GD)
   endif()
 
   foreach(TEST_OPTIONS_VAR ${TEST_OPTIONS_VARS})
@@ -145,7 +115,7 @@ if (MFEM_ENABLE_TESTING)
 
     # All PETSC tests are parallel.
     if (MFEM_USE_MPI)
-      add_test(NAME ${TEST_NAME_FULL}_np=${MFEM_MPI_NP}
+      add_test(NAME ${TEST_NAME_FULL}_np=4
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
         ${MPIEXEC_PREFLAGS}
         $<TARGET_FILE:${TEST_NAME}> ${TEST_OPTIONS}

examples/petsc/ex1p.cpp

@@ -7,7 +7,7 @@
 //               mpirun -np 4 ex1p -m ../../data/amr-quad.mesh --petscopts rc_ex1p
 //
 // Device sample runs:
-//               mpirun -np 4 ex1p -pa -d cuda --petscopts rc_ex1p_device
+//               mpirun -np 4 ex1p -pa -d cuda --petscopts rc_ex1p_cuda
 //
 // Description:  This example code demonstrates the use of MFEM to define a
 //               simple finite element discretization of the Laplace problem

examples/petsc/makefile

@@ -66,9 +66,7 @@ include $(MFEM_TEST_MK)
 
 # Testing: Parallel runs
 RUN_MPI = $(MFEM_MPIEXEC) $(MFEM_MPIEXEC_NP) $(MFEM_MPI_NP)
-TESTNAME      = Parallel PETSc example
-TESTNAME_CUDA = Parallel CUDA PETSc example
-TESTNAME_HIP  = Parallel HIP PETSc example
+TESTNAME = Parallel PETSc example
 %-test-par: %
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME))
 
@@ -76,10 +74,8 @@ TESTNAME_HIP  = Parallel HIP PETSc example
 # Testing PETSc execution options.
 EX1_ARGS_W            := -m ../../data/amr-quad.mesh --usepetsc
 EX1_ARGS_P            := -m ../../data/amr-quad.mesh --usepetsc --petscopts rc_ex1p
-EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_device
-EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_deviceamg
-EX1_ARGS_HIP          := -m ../../data/star.mesh --usepetsc --partial-assembly --device hip --petscopts rc_ex1p_device
-EX1_ARGS_HIPAMG       := -m ../../data/star.mesh --usepetsc --device hip --petscopts rc_ex1p_deviceamg
+EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly --device cuda --petscopts rc_ex1p_cuda
+EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
 EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
@@ -93,8 +89,7 @@ EX6_ARGS              := -m ../../data/amr-quad.mesh --usepetsc
 EX6_NONOVL_ARGS       := -m ../../data/amr-quad.mesh --usepetsc --nonoverlapping
 EX9_E_ARGS            := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl -dt 0.1
 EX9_ES_ARGS           := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl --no-step
-EX9_ES_ARGS_CUDA      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device cuda
-EX9_ES_ARGS_HIP       := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_device --no-step --partial-assembly --device hip
+EX9_ES_ARGS_CUDA      := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_expl_cuda --no-step --partial-assembly --device cuda
 EX9_IS_ARGS           := -m ../../data/periodic-hexagon.mesh --usepetsc --petscopts rc_ex9p_impl --implicit -tf 0.5
 EX10_ARGS             := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p -tf 30 -s 3 -rs 2 -dt 3
 EX10_MF_ARGS          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_mf -tf 6 -s 3 -rs 0 -dt 3
@@ -102,20 +97,15 @@ EX10_MFOP_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_
 EX10_JFNK_ARGS        := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex10p_jfnk --jfnk -tf 6 -s 3 -rs 0 -dt 3
 EX11_ARGS_SINV        := -m ../../data/star.mesh --useslepc
 EX11_ARGS_LOBPCG      := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg
-EX11_ARGS_LOBPCG_CUDA := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device cuda
-EX11_ARGS_LOBPCG_HIP  := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_device --device hip
+EX11_ARGS_LOBPCG_CUDA := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_lobpcg_cuda --device cuda
 EX11_ARGS_GD          := -m ../../data/star.mesh --useslepc --slepcopts rc_ex11p_gd
 
 ex1p-test-par: ex1p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_W))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_P))
 ifeq ($(MFEM_USE_CUDA),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX1_ARGS_CUDA))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX1_ARGS_CUDAAMG))
-endif
-ifeq ($(MFEM_USE_HIP),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX1_ARGS_HIP))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX1_ARGS_HIPAMG))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_CUDA))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX1_ARGS_CUDAAMG))
 endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
@@ -138,10 +128,7 @@ ex9p-test-par: ex9p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_IS_ARGS))
 ifeq ($(MFEM_USE_CUDA),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX9_ES_ARGS_CUDA))
-endif
-ifeq ($(MFEM_USE_HIP),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX9_ES_ARGS_HIP))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX9_ES_ARGS_CUDA))
 endif
 ex10p-test-par: ex10p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX10_ARGS))
@@ -153,12 +140,8 @@ ex11p-test-par: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_SINV))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG))
 ifeq ($(MFEM_USE_CUDA),YES)
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_CUDA),$(EX11_ARGS_LOBPCG_CUDA))
+	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_LOBPCG_CUDA))
 endif
-# SLEPc does not support BVSVEC with HIP
-#ifeq ($(MFEM_USE_HIP),YES)
-#	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME_HIP),$(EX11_ARGS_LOBPCG_HIP))
-#endif
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX11_ARGS_GD))
 endif
 
@@ -173,9 +156,6 @@ clean: clean-build clean-exec
 clean-build:
 	rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
 	rm -rf *.dSYM *.TVD.*breakpoints
-ifneq ($(SRC),)
-	rm -f $(RC_FILES)
-endif
 
 clean-exec:
 	@rm -rf mesh.* sol.* sol_p.* sol_u.* Example5*

fem/CMakeLists.txt

@@ -23,7 +23,6 @@ set(SRCS
   integ/bilininteg_diffusion_mf.cpp
   integ/bilininteg_diffusion_pa.cpp
   integ/bilininteg_diffusion_ea.cpp
-  integ/bilininteg_diffusion_patch.cpp
   integ/bilininteg_divdiv_pa.cpp
   integ/bilininteg_gradient_pa.cpp
   integ/bilininteg_interp_pa.cpp
@@ -88,7 +87,6 @@ set(SRCS
   ceed/solvers/algebraic.cpp
   ceed/solvers/full-assembly.cpp
   ceed/solvers/solvers-atpmg.cpp
-  kdtree.cpp
   linearform.cpp
   linearform_ext.cpp
   lininteg.cpp
@@ -200,7 +198,6 @@ set(HDRS
   ceed/solvers/algebraic.hpp
   ceed/solvers/full-assembly.hpp
   ceed/solvers/solvers-atpmg.hpp
-  kdtree.hpp
   linearform.hpp
   linearform_ext.hpp
   lininteg.hpp

fem/bilinearform.cpp

@@ -13,7 +13,6 @@
 
 #include "fem.hpp"
 #include "../general/device.hpp"
-#include "../mesh/nurbs.hpp"
 #include <cmath>
 
 namespace mfem
@@ -422,17 +421,11 @@ void BilinearForm::Assemble(int skip_zeros)
                         "invalid element marker for domain integrator #"
                         << k << ", counting from zero");
          }
-
-         if (domain_integs[k]->Patchwise())
-         {
-            MFEM_VERIFY(fes->GetNURBSext(), "Patchwise integration requires a "
-                        << "NURBS FE space");
-         }
       }
 
-      // Element-wise integration
       for (int i = 0; i < fes -> GetNE(); i++)
       {
+         int elem_attr = fes->GetMesh()->GetAttribute(i);
          doftrans = fes->GetElementVDofs(i, vdofs);
          if (element_matrices)
          {
@@ -440,13 +433,11 @@ void BilinearForm::Assemble(int skip_zeros)
          }
          else
          {
-            const int elem_attr = fes->GetMesh()->GetAttribute(i);
             elmat.SetSize(0);
             for (int k = 0; k < domain_integs.Size(); k++)
             {
-               if ((domain_integs_marker[k] == NULL ||
+               if ( domain_integs_marker[k] == NULL ||
                     (*(domain_integs_marker[k]))[elem_attr-1] == 1)
-                   && !domain_integs[k]->Patchwise())
                {
                   const FiniteElement &fe = *fes->GetFE(i);
                   eltrans = fes->GetElementTransformation(i);
@@ -488,43 +479,6 @@ void BilinearForm::Assemble(int skip_zeros)
             }
          }
       }
-
-      // Patch-wise integration
-      if (fes->GetNURBSext())
-      {
-         for (int p=0; p<mesh->NURBSext->GetNP(); ++p)
-         {
-            bool vdofsSet = false;
-            for (int k = 0; k < domain_integs.Size(); k++)
-            {
-               if (domain_integs[k]->Patchwise())
-               {
-                  if (!vdofsSet)
-                  {
-                     fes->GetPatchVDofs(p, vdofs);
-                     vdofsSet = true;
-                  }
-
-                  SparseMatrix* spmat = nullptr;
-                  domain_integs[k]->AssemblePatchMatrix(p, *fes, spmat);
-                  Array<int> cols;
-                  Vector srow;
-
-                  for (int r=0; r<spmat->Height(); ++r)
-                  {
-                     spmat->GetRow(r, cols, srow);
-                     for (int i=0; i<cols.Size(); ++i)
-                     {
-                        cols[i] = vdofs[cols[i]];
-                     }
-                     mat->AddRow(vdofs[r], cols, srow);
-                  }
-
-                  delete spmat;
-               }
-            }
-         }
-      }
    }
 
    if (boundary_integs.Size())

fem/bilinearform.hpp

@@ -254,12 +254,6 @@ public:
    /// Access all the integrators added with AddDomainIntegrator().
    Array<BilinearFormIntegrator*> *GetDBFI() { return &domain_integs; }
 
-   /// @brief Access all boundary markers added with AddDomainIntegrator().
-   ///
-   /// If no marker was specified when the integrator was added, the
-   /// corresponding pointer (to Array<int>) will be NULL. */
-   Array<Array<int>*> *GetDBFI_Marker() { return &domain_integs_marker; }
-
    /// Access all the integrators added with AddBoundaryIntegrator().
    Array<BilinearFormIntegrator*> *GetBBFI() { return &boundary_integs; }
    /** @brief Access all boundary markers added with AddBoundaryIntegrator().
@@ -458,7 +452,7 @@ public:
        practice it is convenient to have it in transposed form for
        construction of RAP operators in matrix-free methods. */
    virtual const Operator *GetOutputRestrictionTranspose() const
-   { return fes->GetRestrictionTransposeOperator(); }
+   { return GetOutputProlongation(); }
    /// Get the output finite element space restriction matrix
    virtual const Operator *GetOutputRestriction() const
    { return GetRestriction(); }

fem/bilinearform_ext.cpp

@@ -264,14 +264,6 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
       localX.SetSize(elem_restrict->Height(), Device::GetDeviceMemoryType());
       localY.SetSize(elem_restrict->Height(), Device::GetDeviceMemoryType());
       localY.UseDevice(true); // ensure 'localY = 0.0' is done on device
-
-      // Gather the attributes on the host from all the elements
-      const Mesh &mesh = *trial_fes->GetMesh();
-      elem_attributes.SetSize(mesh.GetNE());
-      for (int i = 0; i < mesh.GetNE(); ++i)
-      {
-         elem_attributes[i] = mesh.GetAttribute(i);
-      }
    }
 
    // Construct face restriction operators only if the bilinear form has
@@ -297,46 +289,6 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
       bdr_face_X.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
       bdr_face_Y.SetSize(bdr_face_restrict_lex->Height(), Device::GetMemoryType());
       bdr_face_Y.UseDevice(true); // ensure 'faceBoundY = 0.0' is done on device
-
-      const Mesh &mesh = *trial_fes->GetMesh();
-      // See LinearFormExtension::Update for explanation of f_to_be logic.
-      std::unordered_map<int,int> f_to_be;
-      for (int i = 0; i < mesh.GetNBE(); ++i)
-      {
-         const int f = mesh.GetBdrElementEdgeIndex(i);
-         f_to_be[f] = i;
-      }
-      const int nf_bdr = trial_fes->GetNFbyType(FaceType::Boundary);
-      bdr_attributes.SetSize(nf_bdr);
-      int f_ind = 0;
-      int missing_bdr_elems = 0;
-      for (int f = 0; f < mesh.GetNumFaces(); ++f)
-      {
-         if (!mesh.GetFaceInformation(f).IsOfFaceType(FaceType::Boundary))
-         {
-            continue;
-         }
-         int attribute = 1; // default value
-         if (f_to_be.find(f) != f_to_be.end())
-         {
-            const int be = f_to_be[f];
-            attribute = mesh.GetBdrAttribute(be);
-         }
-         else
-         {
-            // If a boundary face does not correspond to the a boundary element,
-            // we assign it the default attribute of 1. We also generate a
-            // warning at runtime with the number of such missing elements.
-            ++missing_bdr_elems;
-         }
-         bdr_attributes[f_ind] = attribute;
-         ++f_ind;
-      }
-      if (missing_bdr_elems)
-      {
-         MFEM_WARNING("Missing " << missing_bdr_elems << " boundary elements "
-                      "for boundary faces.");
-      }
    }
 }
 
@@ -347,16 +299,7 @@ void PABilinearFormExtension::Assemble()
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
    for (BilinearFormIntegrator *integ : integrators)
    {
-      if (integ->Patchwise())
-      {
-         MFEM_VERIFY(a->FESpace()->GetNURBSext(),
-                     "Patchwise integration requires a NURBS FE space");
-         integ->AssembleNURBSPA(*a->FESpace());
-      }
-      else
-      {
-         integ->AssemblePA(*a->FESpace());
-      }
+      integ->AssemblePA(*a->FESpace());
    }
 
    Array<BilinearFormIntegrator*> &bdr_integrators = *a->GetBBFI();
@@ -467,52 +410,24 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
    const int iSz = integrators.Size();
-
-   bool allPatchwise = true;
-   bool somePatchwise = false;
-
-   for (int i = 0; i < iSz; ++i)
-   {
-      if (integrators[i]->Patchwise())
-      {
-         somePatchwise = true;
-      }
-      else
-      {
-         allPatchwise = false;
-      }
-   }
-
-   MFEM_VERIFY(!(somePatchwise && !allPatchwise),
-               "All or none of the integrators should be patchwise");
-
-   if (DeviceCanUseCeed() || !elem_restrict || allPatchwise)
+   if (DeviceCanUseCeed() || !elem_restrict)
    {
       y.UseDevice(true); // typically this is a large vector, so store on device
       y = 0.0;
       for (int i = 0; i < iSz; ++i)
       {
-         if (integrators[i]->Patchwise())
-         {
-            integrators[i]->AddMultNURBSPA(x, y);
-         }
-         else
-         {
-            integrators[i]->AddMultPA(x, y);
-         }
+         integrators[i]->AddMultPA(x, y);
       }
    }
    else
    {
       if (iSz)
       {
-         Array<Array<int>*> &elem_markers = *a->GetDBFI_Marker();
          elem_restrict->Mult(x, localX);
          localY = 0.0;
          for (int i = 0; i < iSz; ++i)
          {
-            AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
-                               false, localY);
+            integrators[i]->AddMultPA(localX, localY);
          }
          elem_restrict->MultTranspose(localY, y);
       }
@@ -545,21 +460,17 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
    const bool has_bdr_integs = (n_bdr_face_integs > 0 || n_bdr_integs > 0);
    if (bdr_face_restrict_lex && has_bdr_integs)
    {
-      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
-      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
       if (bdr_face_X.Size()>0)
       {
          bdr_face_Y = 0.0;
          for (int i = 0; i < n_bdr_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
-                               false, bdr_face_Y);
+            bdr_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
          }
          for (int i = 0; i < n_bdr_face_integs; ++i)
          {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                               bdr_attributes, false, bdr_face_Y);
+            bdr_face_integs[i]->AddMultPA(bdr_face_X, bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
@@ -572,13 +483,11 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
    const int iSz = integrators.Size();
    if (elem_restrict)
    {
-      Array<Array<int>*> &elem_markers = *a->GetDBFI_Marker();
       elem_restrict->Mult(x, localX);
       localY = 0.0;
       for (int i = 0; i < iSz; ++i)
       {
-         AddMultWithMarkers(*integrators[i], localX, elem_markers[i], elem_attributes,
-                            true, localY);
+         integrators[i]->AddMultTransposePA(localX, localY);
       }
       elem_restrict->MultTranspose(localY, y);
    }
@@ -608,85 +517,23 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
       }
    }
 
-   Array<BilinearFormIntegrator*> &bdr_integs = *a->GetBBFI();
-   Array<BilinearFormIntegrator*> &bdr_face_integs = *a->GetBFBFI();
-   const int n_bdr_integs = bdr_integs.Size();
-   const int n_bdr_face_integs = bdr_face_integs.Size();
-   const bool has_bdr_integs = (n_bdr_face_integs > 0 || n_bdr_integs > 0);
-   if (bdr_face_restrict_lex && has_bdr_integs)
+   Array<BilinearFormIntegrator*> &bdrFaceIntegrators = *a->GetBFBFI();
+   const int bFISz = bdrFaceIntegrators.Size();
+   if (bdr_face_restrict_lex && bFISz>0)
    {
-      Array<Array<int>*> &bdr_markers = *a->GetBBFI_Marker();
-      Array<Array<int>*> &bdr_face_markers = *a->GetBFBFI_Marker();
-
       bdr_face_restrict_lex->Mult(x, bdr_face_X);
-      if (bdr_face_X.Size() > 0)
+      if (bdr_face_X.Size()>0)
       {
          bdr_face_Y = 0.0;
-         for (int i = 0; i < n_bdr_integs; ++i)
+         for (int i = 0; i < bFISz; ++i)
          {
-            AddMultWithMarkers(*bdr_integs[i], bdr_face_X, bdr_markers[i], bdr_attributes,
-                               true, bdr_face_Y);
-         }
-         for (int i = 0; i < n_bdr_face_integs; ++i)
-         {
-            AddMultWithMarkers(*bdr_face_integs[i], bdr_face_X, bdr_face_markers[i],
-                               bdr_attributes, true, bdr_face_Y);
+            bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
          }
          bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
       }
    }
 }
 
-// Compute kernels for PABilinearFormExtension::AddMultWithMarkers.
-// Cannot be in member function with non-public visibility.
-static void AddWithMarkers_(
-   const int ne,
-   const int nd,
-   const Vector &x,
-   const Array<int> &markers,
-   const Array<int> &attributes,
-   Vector &y)
-{
-   const auto d_x = Reshape(x.Read(), nd, ne);
-   const auto d_m = Reshape(markers.Read(), markers.Size());
-   const auto d_attr = Reshape(attributes.Read(), ne);
-   auto d_y = Reshape(y.ReadWrite(), nd, ne);
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int attr = d_attr[e];
-      if (d_m[attr - 1] == 0) { return; }
-      for (int i = 0; i < nd; ++i)
-      {
-         d_y(i, e) += d_x(i, e);
-      }
-   });
-}
-
-void PABilinearFormExtension::AddMultWithMarkers(
-   const BilinearFormIntegrator &integ,
-   const Vector &x,
-   const Array<int> *markers,
-   const Array<int> &attributes,
-   const bool transpose,
-   Vector &y) const
-{
-   if (markers)
-   {
-      tmp_evec.SetSize(y.Size());
-      tmp_evec = 0.0;
-      if (transpose) { integ.AddMultTransposePA(x, tmp_evec); }
-      else { integ.AddMultPA(x, tmp_evec); }
-      const int ne = attributes.Size();
-      const int nd = x.Size() / ne;
-      AddWithMarkers_(ne, nd, tmp_evec, *markers, attributes, y);
-   }
-   else
-   {
-      if (transpose) { integ.AddMultTransposePA(x, y); }
-      else { integ.AddMultPA(x, y); }
-   }
-}
-
 // Data and methods for element-assembled bilinear forms
 EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
    : PABilinearFormExtension(form),

fem/bilinearform_ext.hpp

@@ -68,9 +68,6 @@ class PABilinearFormExtension : public BilinearFormExtension
 {
 protected:
    const FiniteElementSpace *trial_fes, *test_fes; // Not owned
-   /// Attributes of all mesh elements.
-   Array<int> elem_attributes, bdr_attributes;
-   mutable Vector tmp_evec; // Work array
    mutable Vector localX, localY;
    mutable Vector int_face_X, int_face_Y;
    mutable Vector bdr_face_X, bdr_face_Y;
@@ -94,25 +91,6 @@ public:
 
 protected:
    void SetupRestrictionOperators(const L2FaceValues m);
-
-   /// @brief Accumulate the action (or transpose) of the integrator on @a x
-   /// into @a y, taking into account the (possibly null) @a markers array.
-   ///
-   /// If @a markers is non-null, then only those elements or boundary elements
-   /// whose attribute is marked in the markers array will be added to @a y.
-   ///
-   /// @param integ The integrator (domain, boundary, or boundary face).
-   /// @param x Input E-vector.
-   /// @param markers Marked attributes (possibly null, meaning all attributes).
-   /// @param attributes Array of element or boundary element attributes.
-   /// @param transpose Compute the action or transpose of the integrator .
-   /// @param y Output E-vector
-   void AddMultWithMarkers(const BilinearFormIntegrator &integ,
-                           const Vector &x,
-                           const Array<int> *markers,
-                           const Array<int> &attributes,
-                           const bool transpose,
-                           Vector &y) const;
 };
 
 /// Data and methods for element-assembled bilinear forms

fem/bilininteg.cpp

@@ -26,12 +26,6 @@ void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&)
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleNURBSPA(const FiniteElementSpace&)
-{
-   mfem_error ("BilinearFormIntegrator::AssembleNURBSPA(fes)\n"
-               "   is not implemented for this class.");
-}
-
 void BilinearFormIntegrator::AssemblePA(const FiniteElementSpace&,
                                         const FiniteElementSpace&)
 {
@@ -98,13 +92,7 @@ void BilinearFormIntegrator::AssembleDiagonalPA_ADAt(const Vector &, Vector &)
 
 void BilinearFormIntegrator::AddMultPA(const Vector &, Vector &) const
 {
-   MFEM_ABORT("BilinearFormIntegrator:AddMultPA:(...)\n"
-              "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AddMultNURBSPA(const Vector &, Vector &) const
-{
-   MFEM_ABORT("BilinearFormIntegrator::AddMultNURBSPA(...)\n"
+   MFEM_ABORT("BilinearFormIntegrator::MultAssembled(...)\n"
               "   is not implemented for this class.");
 }
 
@@ -138,30 +126,23 @@ void BilinearFormIntegrator::AssembleDiagonalMF(Vector &)
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleElementMatrix(
+void BilinearFormIntegrator::AssembleElementMatrix (
    const FiniteElement &el, ElementTransformation &Trans,
-   DenseMatrix &elmat)
+   DenseMatrix &elmat )
 {
    MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix(...)\n"
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssembleElementMatrix2(
+void BilinearFormIntegrator::AssembleElementMatrix2 (
    const FiniteElement &el1, const FiniteElement &el2,
-   ElementTransformation &Trans, DenseMatrix &elmat)
+   ElementTransformation &Trans, DenseMatrix &elmat )
 {
    MFEM_ABORT("BilinearFormIntegrator::AssembleElementMatrix2(...)\n"
               "   is not implemented for this class.");
 }
 
-void BilinearFormIntegrator::AssemblePatchMatrix(
-   const int patch, const FiniteElementSpace &fes, SparseMatrix*& smat)
-{
-   mfem_error ("BilinearFormIntegrator::AssemblePatchMatrix(...)\n"
-               "   is not implemented for this class.");
-}
-
-void BilinearFormIntegrator::AssembleFaceMatrix(
+void BilinearFormIntegrator::AssembleFaceMatrix (
    const FiniteElement &el1, const FiniteElement &el2,
    FaceElementTransformations &Trans, DenseMatrix &elmat)
 {
@@ -867,19 +848,6 @@ void DiffusionIntegrator::AssembleElementMatrix
 
    const IntegrationRule *ir = IntRule ? IntRule : &GetRule(el, el);
 
-   const NURBSFiniteElement *NURBSFE =
-      dynamic_cast<const NURBSFiniteElement *>(&el);
-
-   bool deleteRule = false;
-   if (NURBSFE && patchRules)
-   {
-      const int patch = NURBSFE->GetPatch();
-      const int* ijk = NURBSFE->GetIJK();
-      Array<const KnotVector*>& kv = NURBSFE->KnotVectors();
-      ir = &patchRules->GetElementRule(NURBSFE->GetElement(), patch, ijk, kv,
-                                       deleteRule);
-   }
-
    elmat = 0.0;
    for (int i = 0; i < ir->GetNPoints(); i++)
    {
@@ -914,11 +882,6 @@ void DiffusionIntegrator::AssembleElementMatrix
          AddMult_a_AAt(w, dshapedxt, elmat);
       }
    }
-
-   if (deleteRule)
-   {
-      delete ir;
-   }
 }
 
 void DiffusionIntegrator::AssembleElementMatrix2(
@@ -2456,7 +2419,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix(
 {
    int dof = el.GetDof();
    int spaceDim = Trans.GetSpaceDim();
-   int vdim = std::max(spaceDim, el.GetRangeDim());
+   int vdim = std::max(spaceDim, el.GetVDim());
 
    double w;
 
@@ -2524,7 +2487,7 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
    {
       // assume test_fe is scalar FE and trial_fe is vector FE
       int spaceDim = Trans.GetSpaceDim();
-      int vdim = std::max(spaceDim, trial_fe.GetRangeDim());
+      int vdim = std::max(spaceDim, trial_fe.GetVDim());
       int trial_dof = trial_fe.GetDof();
       int test_dof = test_fe.GetDof();
       double w;
@@ -2622,8 +2585,8 @@ void VectorFEMassIntegrator::AssembleElementMatrix2(
    {
       // assume both test_fe and trial_fe are vector FE
       int spaceDim = Trans.GetSpaceDim();
-      int trial_vdim = std::max(spaceDim, trial_fe.GetRangeDim());
-      int test_vdim = std::max(spaceDim, test_fe.GetRangeDim());
+      int trial_vdim = std::max(spaceDim, trial_fe.GetVDim());
+      int test_vdim = std::max(spaceDim, test_fe.GetVDim());
       int trial_dof = trial_fe.GetDof();
       int test_dof = test_fe.GetDof();
       double w;

fem/bilininteg.hpp

@@ -20,6 +20,17 @@
 namespace mfem
 {
 
+// Local maximum size of dofs and quads in 1D
+constexpr int HCURL_MAX_D1D = 5;
+#ifdef MFEM_USE_HIP
+constexpr int HCURL_MAX_Q1D = 5;
+#else
+constexpr int HCURL_MAX_Q1D = 6;
+#endif
+
+constexpr int HDIV_MAX_D1D = 5;
+constexpr int HDIV_MAX_Q1D = 6;
+
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
 {
@@ -50,11 +61,6 @@ public:
    virtual void AssemblePA(const FiniteElementSpace &trial_fes,
                            const FiniteElementSpace &test_fes);
 
-   /// Method defining partial assembly on NURBS patches.
-   /** The result of the partial assembly is stored internally so that it can be
-       used later in the method AddMultNURBSPA(). */
-   virtual void AssembleNURBSPA(const FiniteElementSpace &fes);
-
    virtual void AssemblePABoundary(const FiniteElementSpace &fes);
 
    virtual void AssemblePAInteriorFaces(const FiniteElementSpace &fes);
@@ -76,9 +82,6 @@ public:
        called. */
    virtual void AddMultPA(const Vector &x, Vector &y) const;
 
-   /// Method for partially assembled action on NURBS patches.
-   virtual void AddMultNURBSPA(const Vector&x, Vector&y) const;
-
    /// Method for partially assembled transposed action.
    /** Perform the transpose action of integrator on the input @a x and add the
        result to the output @a y. Both @a x and @a y are E-vectors, i.e. they
@@ -145,13 +148,6 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 
-   /** Given a particular NURBS patch, computes the patch matrix as a
-       SparseMatrix @a smat.
-    */
-   virtual void AssemblePatchMatrix(const int patch,
-                                    const FiniteElementSpace &fes,
-                                    SparseMatrix*& smat);
-
    virtual void AssembleFaceMatrix(const FiniteElement &el1,
                                    const FiniteElement &el2,
                                    FaceElementTransformations &Trans,
@@ -580,7 +576,7 @@ protected:
 
 
    inline virtual int GetTestVDim(const FiniteElement & test_fe)
-   { return std::max(space_dim, test_fe.GetRangeDim()); }
+   { return std::max(space_dim, test_fe.GetVDim()); }
 
    inline virtual void CalcTestShape(const FiniteElement & test_fe,
                                      ElementTransformation &Trans,
@@ -588,7 +584,7 @@ protected:
    { test_fe.CalcVShape(Trans, shape); }
 
    inline virtual int GetTrialVDim(const FiniteElement & trial_fe)
-   { return std::max(space_dim, trial_fe.GetRangeDim()); }
+   { return std::max(space_dim, trial_fe.GetVDim()); }
 
    inline virtual void CalcTrialShape(const FiniteElement & trial_fe,
                                       ElementTransformation &Trans,
@@ -678,7 +674,7 @@ protected:
 
 
    inline virtual int GetVDim(const FiniteElement & vector_fe)
-   { return std::max(space_dim, vector_fe.GetRangeDim()); }
+   { return std::max(space_dim, vector_fe.GetVDim()); }
 
    inline virtual void CalcVShape(const FiniteElement & vector_fe,
                                   ElementTransformation &Trans,
@@ -1105,7 +1101,7 @@ public:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetRangeDim() == 3 &&
+      return (trial_fe.GetVDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               test_fe.GetRangeType()  == mfem::FiniteElement::SCALAR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::GRAD );
@@ -1288,8 +1284,8 @@ public:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetCurlDim() == 3 && trial_fe.GetRangeDim() == 3 &&
-              test_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
+      return (trial_fe.GetCurlDim() == 3 && trial_fe.GetVDim() == 3 &&
+              test_fe.GetCurlDim() == 3 && test_fe.GetVDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               trial_fe.GetDerivType() == mfem::FiniteElement::CURL &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR &&
@@ -1419,7 +1415,7 @@ public:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetRangeDim() == 3 && test_fe.GetCurlDim() == 3 &&
+      return (trial_fe.GetVDim() == 3 && test_fe.GetCurlDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::CURL );
@@ -1489,7 +1485,7 @@ public:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (test_fe.GetRangeDim() == 3 &&
+      return (test_fe.GetVDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::SCALAR &&
               trial_fe.GetDerivType() == mfem::FiniteElement::GRAD &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
@@ -1529,7 +1525,7 @@ public:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
+      return (trial_fe.GetCurlDim() == 3 && test_fe.GetVDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               trial_fe.GetDerivType() == mfem::FiniteElement::CURL   &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
@@ -1900,7 +1896,7 @@ protected:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetCurlDim() == 3 && test_fe.GetRangeDim() == 3 &&
+      return (trial_fe.GetCurlDim() == 3 && test_fe.GetVDim() == 3 &&
               trial_fe.GetDerivType() == mfem::FiniteElement::CURL  &&
               test_fe.GetRangeType()  == mfem::FiniteElement::VECTOR );
    }
@@ -1959,7 +1955,7 @@ protected:
       const FiniteElement & trial_fe,
       const FiniteElement & test_fe) const
    {
-      return (trial_fe.GetRangeDim() == 3 && test_fe.GetCurlDim() == 3 &&
+      return (trial_fe.GetVDim() == 3 && test_fe.GetCurlDim() == 3 &&
               trial_fe.GetRangeType() == mfem::FiniteElement::VECTOR &&
               test_fe.GetDerivType()  == mfem::FiniteElement::CURL );
    }
@@ -2115,59 +2111,6 @@ private:
    Vector pa_data;
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
-   // Data for NURBS patch PA
-
-   // Type for a variable-row-length 2D array, used for data related to 1D
-   // quadrature rules in each dimension.
-   typedef std::vector<std::vector<int>> IntArrayVar2D;
-
-   int numPatches = 0;
-   static constexpr int numTypes = 2;  // Number of rule types
-
-   // In the case integrationMode == Mode::PATCHWISE_REDUCED, an approximate
-   // integration rule with sparse nonzero weights is computed by NNLSSolver,
-   // for each 1D basis function on each patch, in each spatial dimension. For a
-   // fixed 1D basis function b_i with DOF index i, in the tensor product basis
-   // of patch p, the prescribed exact 1D rule is of the form
-   // \sum_k a_{i,j,k} w_k for some integration points indexed by k, with
-   // weights w_k and coefficients a_{i,j,k} depending on Q(x), an element
-   // transformation, b_i, and b_j, for all 1D basis functions b_j whose support
-   // overlaps that of b_i. Define the constraint matrix G = [g_{j,k}] with
-   // g_{j,k} = a_{i,j,k} and the vector of exact weights w = [w_k]. A reduced
-   // rule should have different weights w_r, many of them zero, and should
-   // approximately satisfy Gw_r = Gw. A sparse approximate solution to this
-   // underdetermined system is computed by NNLSSolver, and its data is stored
-   // in the following members.
-
-   // For each patch p, spatial dimension d (total dim), and rule type t (total
-   // numTypes), an std::vector<Vector> of reduced quadrature weights for all
-   // basis functions is stored in reducedWeights[t + numTypes * (d + dim * p)],
-   // reshaped as rw(t,d,p). Note that nd may vary with respect to the patch and
-   // spatial dimension. Array reducedIDs is treated similarly.
-   std::vector<std::vector<Vector>> reducedWeights;
-   std::vector<IntArrayVar2D> reducedIDs;
-   std::vector<Array<int>> pQ1D, pD1D;
-   std::vector<std::vector<Array2D<double>>> pB, pG;
-   std::vector<IntArrayVar2D> pminD, pmaxD, pminQ, pmaxQ, pminDD, pmaxDD;
-
-   std::vector<Array<const IntegrationRule*>> pir1d;
-
-   void SetupPatchPA(const int patch, Mesh *mesh, bool unitWeights=false);
-
-   void SetupPatchBasisData(Mesh *mesh, unsigned int patch);
-
-   /** Called by AssemblePatchMatrix for sparse matrix assembly on a NURBS patch
-    with full 1D quadrature rules. */
-   void AssemblePatchMatrix_fullQuadrature(const int patch,
-                                           const FiniteElementSpace &fes,
-                                           SparseMatrix*& smat);
-
-   /** Called by AssemblePatchMatrix for sparse matrix assembly on a NURBS patch
-    with reduced 1D quadrature rules. */
-   void AssemblePatchMatrix_reducedQuadrature(const int patch,
-                                              const FiniteElementSpace &fes,
-                                              SparseMatrix*& smat);
-
 public:
    /// Construct a diffusion integrator with coefficient Q = 1
    DiffusionIntegrator(const IntegrationRule *ir = nullptr)
@@ -2203,14 +2146,6 @@ public:
                                        ElementTransformation &Trans,
                                        DenseMatrix &elmat);
 
-   virtual void AssemblePatchMatrix(const int patch,
-                                    const FiniteElementSpace &fes,
-                                    SparseMatrix*& smat);
-
-   virtual void AssembleNURBSPA(const FiniteElementSpace &fes);
-
-   void AssemblePatchPA(const int patch, const FiniteElementSpace &fes);
-
    /// Perform the local action of the BilinearFormIntegrator
    virtual void AssembleElementVector(const FiniteElement &el,
                                       ElementTransformation &Tr,
@@ -2245,10 +2180,6 @@ public:
 
    virtual void AddMultTransposePA(const Vector&, Vector&) const;
 
-   virtual void AddMultNURBSPA(const Vector&, Vector&) const;
-
-   void AddMultPatchPA(const int patch, const Vector &x, Vector &y) const;
-
    static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
                                          const FiniteElement &test_fe);
 
@@ -3440,7 +3371,7 @@ private:
    void cross_product(const Vector & x, const DenseMatrix & Y, DenseMatrix & Z)
    {
       int dim = x.Size();
-      MFEM_VERIFY(Y.Width() == dim, "Size mismatch");
+      MFEM_VERIFY(Y.Width() == dim, "Size missmatch");
       int dimc = dim == 3 ? dim : 1;
       int h = Y.Height();
       Z.SetSize(h,dimc);

fem/coefficient.cpp

@@ -1591,21 +1591,14 @@ void VectorQuadratureFunctionCoefficient::Eval(Vector &V,
 {
    QuadF.HostRead();
 
-   const int el_idx = QuadF.GetSpace()->GetEntityIndex(T);
-   // Handle the case of "interior boundary elements" and FaceQuadratureSpace
-   // with FaceType::Boundary.
-   if (el_idx < 0) { V = 0.0; return; }
-
-   const int ip_idx = QuadF.GetSpace()->GetPermutedIndex(el_idx, ip.index);
-
    if (index == 0 && vdim == QuadF.GetVDim())
    {
-      QuadF.GetValues(el_idx, ip_idx, V);
+      QuadF.GetValues(T.ElementNo, ip.index, V);
    }
    else
    {
       Vector temp;
-      QuadF.GetValues(el_idx, ip_idx, temp);
+      QuadF.GetValues(T.ElementNo, ip.index, temp);
       V.SetSize(vdim);
       for (int i = 0; i < vdim; i++)
       {
@@ -1632,12 +1625,7 @@ double QuadratureFunctionCoefficient::Eval(ElementTransformation &T,
 {
    QuadF.HostRead();
    Vector temp(1);
-   const int el_idx = QuadF.GetSpace()->GetEntityIndex(T);
-   // Handle the case of "interior boundary elements" and FaceQuadratureSpace
-   // with FaceType::Boundary.
-   if (el_idx < 0) { return 0.0; }
-   const int ip_idx = QuadF.GetSpace()->GetPermutedIndex(el_idx, ip.index);
-   QuadF.GetValues(el_idx, ip_idx, temp);
+   QuadF.GetValues(T.ElementNo, ip.index, temp);
    return temp[0];
 }
 

fem/complex_fem.cpp

@@ -1243,28 +1243,25 @@ ParSesquilinearForm::FormLinearSystem(const Array<int> &ess_tdof_list,
          HypreParMatrix * Ah;
          A_i.Get(Ah);
          hypre_ParCSRMatrix *Aih = *Ah;
-         if (!HypreUsingGPU())
+#if !defined(HYPRE_USING_GPU)
+         ess_tdof_list.HostRead();
+         for (int k = 0; k < n; k++)
          {
-            ess_tdof_list.HostRead();
-            for (int k = 0; k < n; k++)
-            {
-               const int j = ess_tdof_list[k];
-               Aih->diag->data[Aih->diag->i[j]] = 0.0;
-            }
+            const int j = ess_tdof_list[k];
+            Aih->diag->data[Aih->diag->i[j]] = 0.0;
          }
-         else
+#else
+         Ah->HypreReadWrite();
+         const int *d_ess_tdof_list =
+            ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
+         const int *d_diag_i = Aih->diag->i;
+         double *d_diag_data = Aih->diag->data;
+         MFEM_GPU_FORALL(k, n,
          {
-            Ah->HypreReadWrite();
-            const int *d_ess_tdof_list =
-               ess_tdof_list.GetMemory().Read(MemoryClass::DEVICE, n);
-            const int *d_diag_i = Aih->diag->i;
-            double *d_diag_data = Aih->diag->data;
-            MFEM_GPU_FORALL(k, n,
-            {
-               const int j = d_ess_tdof_list[k];
-               d_diag_data[d_diag_i[j]] = 0.0;
-            });
-         }
+            const int j = d_ess_tdof_list[k];
+            d_diag_data[d_diag_i[j]] = 0.0;
+         });
+#endif
       }
       else
       {

fem/datacollection.cpp

@@ -922,7 +922,7 @@ void ParaViewDataCollection::Save()
    {
       const std::string &field_name = qfield.first;
       std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
-      qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name);
+      qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel());
    }
 
    // MPI rank 0 also creates a "PVTU" file that points to all of the separately

fem/dgmassinv.cpp

@@ -166,19 +166,21 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
       b = b_.Read();
    }
 
-   static constexpr int NB = Q1D ? Q1D : 1; // block size
+   constexpr int NB = Q1D ? Q1D : 1; // block size
 
    mfem::forall_2D(NE, NB, NB, [=] MFEM_HOST_DEVICE (int e)
    {
+      constexpr int NB = Q1D ? Q1D : 1; // redefine here for some compilers
+
       // Perform change of basis if needed
       if (CHANGE_BASIS)
       {
          // Transform RHS
-         DGMassBasis<DIM,D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
+         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_Bt, b_orig, b2, d1d);
          if (IT_MODE)
          {
             // Transform initial guess
-            DGMassBasis<DIM,D1D>(e, NE, d2q_B, u, u, d1d);
+            DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, d2q_B, u, u, d1d);
          }
       }
 
@@ -255,7 +257,7 @@ void DGMassInverse::DGMassCGIteration(const Vector &b_, Vector &u_) const
 
       if (CHANGE_BASIS)
       {
-         DGMassBasis<DIM,D1D>(e, NE, q2d_B, u, u, d1d);
+         DGMassBasis<DIM,D1D,MAX_D1D>(e, NE, q2d_B, u, u, d1d);
       }
    });
 }

fem/dgmassinv_kernels.hpp

@@ -172,7 +172,7 @@ double DGMassDot(const int e,
    return s_dot[0];
 }
 
-template<int T_D1D = 0>
+template<int T_D1D = 0, int MAX_D1D = 0>
 MFEM_HOST_DEVICE inline
 void DGMassBasis2D(const int e,
                    const int NE,
@@ -181,7 +181,7 @@ void DGMassBasis2D(const int e,
                    double *y_,
                    const int d1d = 0)
 {
-   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
    const int D1D = T_D1D ? T_D1D : d1d;
 
    const auto b = Reshape(b_, D1D, D1D);
@@ -213,7 +213,7 @@ void DGMassBasis2D(const int e,
    MFEM_SYNC_THREAD;
 }
 
-template<int T_D1D = 0>
+template<int T_D1D = 0, int MAX_D1D = 0>
 MFEM_HOST_DEVICE inline
 void DGMassBasis3D(const int e,
                    const int NE,
@@ -228,7 +228,7 @@ void DGMassBasis3D(const int e,
    const auto x = Reshape(x_, D1D, D1D, D1D, NE);
    auto y = Reshape(y_, D1D, D1D, D1D, NE);
 
-   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
 
    MFEM_SHARED double sB[MD1*MD1];
    MFEM_SHARED double sm0[MD1*MD1*MD1];
@@ -260,7 +260,7 @@ void DGMassBasis3D(const int e,
    MFEM_SYNC_THREAD;
 }
 
-template<int DIM, int T_D1D = 0>
+template<int DIM, int T_D1D = 0, int MAX_D1D = 0>
 MFEM_HOST_DEVICE inline
 void DGMassBasis(const int e,
                  const int NE,
@@ -271,11 +271,11 @@ void DGMassBasis(const int e,
 {
    if (DIM == 2)
    {
-      DGMassBasis2D<T_D1D>(e, NE, b_, x_, y_, d1d);
+      DGMassBasis2D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
    }
    else if (DIM == 3)
    {
-      DGMassBasis3D<T_D1D>(e, NE, b_, x_, y_, d1d);
+      DGMassBasis3D<T_D1D, MAX_D1D>(e, NE, b_, x_, y_, d1d);
    }
    else
    {

fem/doftrans.hpp

@@ -125,7 +125,7 @@ public:
     DofTransformation objects are provided by the FiniteElementSpace which has
     access to the mesh and can therefore provide the face orientations. This is
     convenient when working with GridFunction, LinearForm, or BilinearForm
-    objects or their parallel counterparts.
+    obejcts or their parallel counterparts.
 
     StatelessDofTransformation objects are provided by FiniteElement or
     FiniteElementCollection objects which do not have access to face

fem/fe/fe_base.cpp

@@ -807,7 +807,7 @@ void NodalFiniteElement::Project(
    else
    {
       DenseMatrix vshape(fe.GetDof(), std::max(Trans.GetSpaceDim(),
-                                               fe.GetRangeDim()));
+                                               fe.GetVDim()));
 
       I.SetSize(vshape.Width()*dof, fe.GetDof());
       for (int k = 0; k < dof; k++)

fem/fe/fe_base.hpp

@@ -307,20 +307,19 @@ public:
    FiniteElement(int D, Geometry::Type G, int Do, int O,
                  int F = FunctionSpace::Pk);
 
-   /// Returns the reference space dimension for the finite element.
+   /// Returns the reference space dimension for the finite element
    int GetDim() const { return dim; }
 
-   /** @brief Returns the vector dimension for vector-valued finite elements,
-       which is also the dimension of the interpolation operatrion. */
-   int GetRangeDim() const { return vdim; }
+   /// Returns the vector dimension for vector-valued finite elements
+   int GetVDim() const { return vdim; }
 
-   /// Returns the dimension of the curl for vector-valued finite elements.
+   /// Returns the dimension of the curl for vector-valued finite elements
    int GetCurlDim() const { return cdim; }
 
-   /// Returns the Geometry::Type of the reference element.
+   /// Returns the Geometry::Type of the reference element
    Geometry::Type GetGeomType() const { return geom_type; }
 
-   /// Returns the number of degrees of freedom in the finite element.
+   /// Returns the number of degrees of freedom in the finite element
    int GetDof() const { return dof; }
 
    /** @brief Returns the order of the finite element. In the case of

fem/fe/fe_nd.cpp

@@ -1852,7 +1852,7 @@ void ND_R1D_SegmentElement::Project(const FiniteElement &fe,
    else
    {
       double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
 
       double * tk_ptr = const_cast<double*>(tk);
 
@@ -2293,7 +2293,7 @@ void ND_R2D_FiniteElement::Project(const FiniteElement &fe,
    else
    {
       double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
 
       double * tk_ptr = const_cast<double*>(tk);
 

fem/fe/fe_nurbs.hpp

@@ -56,10 +56,6 @@ public:
    Vector              &Weights    ()         const { return weights; }
    /// Update the NURBSFiniteElement according to the currently set knot vectors
    virtual void         SetOrder   ()         const { }
-
-   /// Returns the indices (i,j) in 2D or (i,j,k) in 3D of this element in the
-   /// tensor product ordering of the patch.
-   const int* GetIJK() const { return ijk; }
 };
 
 

fem/fe/fe_rt.cpp

@@ -1486,7 +1486,7 @@ void RT_R1D_SegmentElement::Project(const FiniteElement &fe,
    else
    {
       double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
 
       double * nk_ptr = const_cast<double*>(nk);
 
@@ -1523,7 +1523,7 @@ void RT_R1D_SegmentElement::ProjectCurl(const FiniteElement &fe,
                                         ElementTransformation &Trans,
                                         DenseMatrix &curl) const
 {
-   DenseMatrix curl_shape(fe.GetDof(), fe.GetRangeDim());
+   DenseMatrix curl_shape(fe.GetDof(), fe.GetVDim());
    Vector curl_k(fe.GetDof());
 
    double * nk_ptr = const_cast<double*>(nk);
@@ -1849,7 +1849,7 @@ void RT_R2D_FiniteElement::Project(const FiniteElement &fe,
    else
    {
       double vk[Geometry::MaxDim];
-      DenseMatrix vshape(fe.GetDof(), fe.GetRangeDim());
+      DenseMatrix vshape(fe.GetDof(), fe.GetVDim());
 
       double * nk_ptr = const_cast<double*>(nk);
 
@@ -1888,7 +1888,7 @@ void RT_R2D_FiniteElement::ProjectCurl(const FiniteElement &fe,
                                        ElementTransformation &Trans,
                                        DenseMatrix &curl) const
 {
-   DenseMatrix curl_shape(fe.GetDof(), fe.GetRangeDim());
+   DenseMatrix curl_shape(fe.GetDof(), fe.GetVDim());
    Vector curl_k(fe.GetDof());
 
    double * nk_ptr = const_cast<double*>(nk);

fem/fe_coll.cpp

@@ -87,16 +87,6 @@ int FiniteElementCollection::GetDerivMapType(int dim) const
    return FiniteElement::UNKNOWN_MAP_TYPE;
 }
 
-int FiniteElementCollection::GetRangeDim(int dim) const
-{
-   const FiniteElement *fe = FiniteElementForDim(dim);
-   if (fe)
-   {
-      return fe->GetRangeDim();
-   }
-   return 0;
-}
-
 int FiniteElementCollection::HasFaceDofs(Geometry::Type geom, int p) const
 {
    switch (geom)
@@ -1723,7 +1713,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_Elements[Geometry::SEGMENT] = new H1_SegmentElement(p, btype);
       }
 
-      SegDofOrd[0] = (pm1 > 0) ? new int[2*pm1] : nullptr;
+      SegDofOrd[0] = new int[2*pm1];
       SegDofOrd[1] = SegDofOrd[0] + pm1;
       for (int i = 0; i < pm1; i++)
       {
@@ -1761,7 +1751,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
 
       const int &TriDof = H1_dof[Geometry::TRIANGLE];
       const int &QuadDof = H1_dof[Geometry::SQUARE];
-      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
+      TriDofOrd[0] = new int[6*TriDof];
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -1782,7 +1772,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          }
       }
 
-      QuadDofOrd[0] = (QuadDof > 0) ? new int[8*QuadDof] : nullptr;
+      QuadDofOrd[0] = new int[8*QuadDof];
       for (int i = 1; i < 8; i++)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
@@ -1865,7 +1855,7 @@ H1_FECollection::H1_FECollection(const int p, const int dim, const int btype)
          H1_Elements[Geometry::PYRAMID] = new LinearPyramidFiniteElement;
 
          const int &TetDof = H1_dof[Geometry::TETRAHEDRON];
-         TetDofOrd[0] = (TetDof > 0) ? new int[24*TetDof] : nullptr;
+         TetDofOrd[0] = new int[24*TetDof];
          for (int i = 1; i < 24; i++)
          {
             TetDofOrd[i] = TetDofOrd[i-1] + TetDof;
@@ -2137,7 +2127,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
       // No need to set the map_type for Tr_Elements.
 
       const int pp1 = p + 1;
-      SegDofOrd[0] = (pp1 > 0) ? new int[2*pp1] : nullptr;
+      SegDofOrd[0] = new int[2*pp1];
       SegDofOrd[1] = SegDofOrd[0] + pp1;
       for (int i = 0; i <= p; i++)
       {
@@ -2170,7 +2160,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
       }
 
       const int TriDof = L2_Elements[Geometry::TRIANGLE]->GetDof();
-      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
+      TriDofOrd[0] = new int[6*TriDof];
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -2191,7 +2181,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
          }
       }
       const int QuadDof = L2_Elements[Geometry::SQUARE]->GetDof();
-      OtherDofOrd = (QuadDof > 0) ? new int[QuadDof] : nullptr;
+      OtherDofOrd = new int[QuadDof];
       for (int j = 0; j < QuadDof; j++)
       {
          OtherDofOrd[j] = j; // for Or == 0
@@ -2235,7 +2225,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
       const int PriDof = L2_Elements[Geometry::PRISM]->GetDof();
       const int MaxDof = std::max(TetDof, std::max(PriDof, HexDof));
 
-      TetDofOrd[0] = (TetDof > 0) ? new int[24*TetDof] : nullptr;
+      TetDofOrd[0] = new int[24*TetDof];
       for (int i = 1; i < 24; i++)
       {
          TetDofOrd[i] = TetDofOrd[i-1] + TetDof;
@@ -2324,7 +2314,7 @@ L2_FECollection::L2_FECollection(const int p, const int dim, const int btype,
             }
          }
       }
-      OtherDofOrd = (MaxDof > 0) ? new int[MaxDof] : nullptr;
+      OtherDofOrd = new int[MaxDof];
       for (int j = 0; j < MaxDof; j++)
       {
          OtherDofOrd[j] = j; // for Or == 0
@@ -2512,7 +2502,7 @@ void RT_FECollection::InitFaces(const int p, const int dim_,
       RT_Elements[Geometry::SEGMENT] = l2_seg;
       RT_dof[Geometry::SEGMENT] = pp1;
 
-      SegDofOrd[0] = (pp1 > 0) ? new int[2*pp1] : nullptr;
+      SegDofOrd[0] = new int[2*pp1];
       SegDofOrd[1] = SegDofOrd[0] + pp1;
       for (int i = 0; i <= p; i++)
       {
@@ -2533,7 +2523,7 @@ void RT_FECollection::InitFaces(const int p, const int dim_,
       RT_dof[Geometry::SQUARE] = pp1*pp1;
 
       int TriDof = RT_dof[Geometry::TRIANGLE];
-      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
+      TriDofOrd[0] = new int[6*TriDof];
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -2563,7 +2553,7 @@ void RT_FECollection::InitFaces(const int p, const int dim_,
       }
 
       int QuadDof = RT_dof[Geometry::SQUARE];
-      QuadDofOrd[0] = (QuadDof > 0) ? new int[8*QuadDof] : nullptr;
+      QuadDofOrd[0] = new int[8*QuadDof];
       for (int i = 1; i < 8; i++)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
@@ -2759,7 +2749,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       ND_Elements[Geometry::SEGMENT] = new ND_SegmentElement(p, ob_type);
       ND_dof[Geometry::SEGMENT] = p;
 
-      SegDofOrd[0] = (p > 0) ? new int[2*p] : nullptr;
+      SegDofOrd[0] = new int[2*p];
       SegDofOrd[1] = SegDofOrd[0] + p;
       for (int i = 0; i < p; i++)
       {
@@ -2779,7 +2769,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       ND_dof[Geometry::TRIANGLE] = p*pm1;
 
       int QuadDof = ND_dof[Geometry::SQUARE];
-      QuadDofOrd[0] = (QuadDof > 0) ? new int[8*QuadDof] : nullptr;
+      QuadDofOrd[0] = new int[8*QuadDof];
       for (int i = 1; i < 8; i++)
       {
          QuadDofOrd[i] = QuadDofOrd[i-1] + QuadDof;
@@ -2823,7 +2813,7 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
       }
 
       int TriDof = ND_dof[Geometry::TRIANGLE];
-      TriDofOrd[0] = (TriDof > 0) ? new int[6*TriDof] : nullptr;
+      TriDofOrd[0] = new int[6*TriDof];
       for (int i = 1; i < 6; i++)
       {
          TriDofOrd[i] = TriDofOrd[i-1] + TriDof;
@@ -3173,7 +3163,7 @@ ND_R2D_FECollection::ND_R2D_FECollection(const int p, const int dim,
                                                                  ob_type);
       ND_dof[Geometry::SEGMENT] = 2 * p - 1;
 
-      SegDofOrd[0] = (4*p > 2) ? new int[4 * p - 2] : nullptr;
+      SegDofOrd[0] = new int[4 * p - 2];
       SegDofOrd[1] = SegDofOrd[0] + 2 * p - 1;
       for (int i = 0; i < p; i++)
       {
@@ -3357,7 +3347,7 @@ void RT_R2D_FECollection::InitFaces(const int p, const int dim,
       RT_Elements[Geometry::SEGMENT] = l2_seg;
       RT_dof[Geometry::SEGMENT] = pp1;
 
-      SegDofOrd[0] = (pp1 > 0) ? new int[2*pp1] : nullptr;
+      SegDofOrd[0] = new int[2*pp1];
       SegDofOrd[1] = SegDofOrd[0] + pp1;
       for (int i = 0; i <= p; i++)
       {

fem/fem.hpp

@@ -26,7 +26,6 @@
 #include "bilininteg.hpp"
 #include "fespace.hpp"
 #include "gridfunc.hpp"
-#include "kdtree.hpp"
 #include "linearform.hpp"
 #include "nonlinearform.hpp"
 #include "bilinearform.hpp"

fem/fespace.cpp

@@ -64,7 +64,7 @@ FiniteElementSpace::FiniteElementSpace()
      face_dof(NULL),
      NURBSext(NULL), own_ext(false),
      DoFTrans(0), VDoFTrans(vdim, ordering),
-     cP_is_set(false),
+     cP(NULL), cR(NULL), cR_hp(NULL), cP_is_set(false),
      Th(Operator::ANY_TYPE),
      sequence(0), mesh_sequence(0), orders_changed(false), relaxed_hp(false)
 { }
@@ -123,24 +123,24 @@ void FiniteElementSpace::CopyProlongationAndRestriction(
 
    if (fes.GetConformingProlongation() != NULL)
    {
-      if (perm) { cP.reset(Mult(*perm_mat, *fes.GetConformingProlongation())); }
-      else { cP.reset(new SparseMatrix(*fes.GetConformingProlongation())); }
+      if (perm) { cP = Mult(*perm_mat, *fes.GetConformingProlongation()); }
+      else { cP = new SparseMatrix(*fes.GetConformingProlongation()); }
       cP_is_set = true;
    }
    else if (perm != NULL)
    {
-      cP.reset(perm_mat);
+      cP = perm_mat;
       cP_is_set = true;
       perm_mat = NULL;
    }
    if (fes.GetConformingRestriction() != NULL)
    {
-      if (perm) { cR.reset(Mult(*fes.GetConformingRestriction(), *perm_mat_tr)); }
-      else { cR.reset(new SparseMatrix(*fes.GetConformingRestriction())); }
+      if (perm) { cR = Mult(*fes.GetConformingRestriction(), *perm_mat_tr); }
+      else { cR = new SparseMatrix(*fes.GetConformingRestriction()); }
    }
    else if (perm != NULL)
    {
-      cR.reset(perm_mat_tr);
+      cR = perm_mat_tr;
       perm_mat_tr = NULL;
    }
 
@@ -309,12 +309,6 @@ FiniteElementSpace::GetBdrElementVDofs(int i, Array<int> &vdofs) const
    }
 }
 
-void FiniteElementSpace::GetPatchVDofs(int i, Array<int> &vdofs) const
-{
-   GetPatchDofs(i, vdofs);
-   DofsToVDofs(vdofs);
-}
-
 void FiniteElementSpace::GetFaceVDofs(int i, Array<int> &vdofs) const
 {
    GetFaceDofs(i, vdofs);
@@ -960,10 +954,7 @@ void FiniteElementSpace::BuildConformingInterpolation() const
 
    if (FEColl()->GetContType() == FiniteElementCollection::DISCONTINUOUS)
    {
-      cP.reset();
-      cR.reset();
-      cR_hp.reset();
-      R_transpose.reset();
+      cP = cR = cR_hp = NULL; // will be treated as identities
       return;
    }
 
@@ -1117,15 +1108,12 @@ void FiniteElementSpace::BuildConformingInterpolation() const
    // if all dofs are true dofs leave cP and cR NULL
    if (n_true_dofs == ndofs)
    {
-      cP.reset();
-      cR.reset();
-      cR_hp.reset();
-      R_transpose.reset();
+      cP = cR = cR_hp = NULL; // will be treated as identities
       return;
    }
 
    // create the conforming prolongation matrix cP
-   cP.reset(new SparseMatrix(ndofs, n_true_dofs));
+   cP = new SparseMatrix(ndofs, n_true_dofs);
 
    // create the conforming restriction matrix cR
    int *cR_J;
@@ -1139,19 +1127,12 @@ void FiniteElementSpace::BuildConformingInterpolation() const
          cR_A[i] = 1.0;
       }
       cR_I[n_true_dofs] = n_true_dofs;
-      cR.reset(new SparseMatrix(cR_I, cR_J, cR_A, n_true_dofs, ndofs));
+      cR = new SparseMatrix(cR_I, cR_J, cR_A, n_true_dofs, ndofs);
    }
 
    // In var. order spaces, create the restriction matrix cR_hp which is similar
    // to cR, but has interpolation in the extra master edge/face DOFs.
-   if (IsVariableOrder())
-   {
-      cR_hp.reset(new SparseMatrix(n_true_dofs, ndofs));
-   }
-   else
-   {
-      cR_hp.reset();
-   }
+   cR_hp = IsVariableOrder() ? new SparseMatrix(n_true_dofs, ndofs) : NULL;
 
    Array<bool> finalized(ndofs);
    finalized = false;
@@ -1269,28 +1250,21 @@ const SparseMatrix* FiniteElementSpace::GetConformingProlongation() const
 {
    if (Conforming()) { return NULL; }
    if (!cP_is_set) { BuildConformingInterpolation(); }
-   return cP.get();
+   return cP;
 }
 
 const SparseMatrix* FiniteElementSpace::GetConformingRestriction() const
 {
    if (Conforming()) { return NULL; }
    if (!cP_is_set) { BuildConformingInterpolation(); }
-   if (cR && !R_transpose) { R_transpose.reset(new TransposeOperator(*cR)); }
-   return cR.get();
+   return cR;
 }
 
 const SparseMatrix* FiniteElementSpace::GetHpConformingRestriction() const
 {
    if (Conforming()) { return NULL; }
    if (!cP_is_set) { BuildConformingInterpolation(); }
-   return IsVariableOrder() ? cR_hp.get() : cR.get();
-}
-
-const Operator *FiniteElementSpace::GetRestrictionTransposeOperator() const
-{
-   GetRestrictionOperator(); // Ensure that R_transpose is built
-   return R_transpose.get();
+   return IsVariableOrder() ? cR_hp : cR;
 }
 
 int FiniteElementSpace::GetNConformingDofs() const
@@ -2221,10 +2195,7 @@ void FiniteElementSpace::Constructor(Mesh *mesh_, NURBSExtension *NURBSext_,
          own_ext = 1;
       }
       UpdateNURBS();
-      cP.reset();
-      cR.reset();
-      cR_hp.reset();
-      R_transpose.reset();
+      cP = cR = cR_hp = NULL;
       cP_is_set = false;
 
       ConstructDoFTrans();
@@ -2386,7 +2357,6 @@ void FiniteElementSpace::Construct()
    cR = NULL;
    cR_hp = NULL;
    cP_is_set = false;
-   R_transpose = NULL;
    // 'Th' is initialized/destroyed before this method is called.
 
    int dim = mesh->Dimension();
@@ -2831,24 +2801,11 @@ FiniteElementSpace::GetElementDofs(int elem, Array<int> &dofs) const
    return DoFTrans[mesh->GetElementBaseGeometry(elem)];
 }
 
-void FiniteElementSpace::GetPatchDofs(int patch, Array<int> &dofs) const
-{
-   MFEM_ASSERT(NURBSext,
-               "FiniteElementSpace::GetPatchDofs needs a NURBSExtension");
-   NURBSext->GetPatchDofs(patch, dofs);
-}
-
 const FiniteElement *FiniteElementSpace::GetFE(int i) const
 {
-   if (i < 0 || i >= mesh->GetNE())
-   {
-      if (mesh->GetNE() == 0)
-      {
-         MFEM_ABORT("Empty MPI partitions are not permitted!");
-      }
-      MFEM_ABORT("Invalid element id:" << i << "; minimum allowed:" << 0 <<
-                 ", maximum allowed:" << mesh->GetNE()-1);
-   }
+   if (i < 0 || !mesh->GetNE()) { return NULL; }
+   MFEM_VERIFY(i < mesh->GetNE(),
+               "Invalid element id " << i << ", maximum allowed " << mesh->GetNE()-1);
 
    const FiniteElement *FE =
       fec->GetFE(mesh->GetElementGeometry(i), GetElementOrderImpl(i));
@@ -3248,10 +3205,9 @@ FiniteElementSpace::~FiniteElementSpace()
 
 void FiniteElementSpace::Destroy()
 {
-   R_transpose.reset();
-   cR.reset();
-   cR_hp.reset();
-   cP.reset();
+   delete cR;
+   delete cR_hp;
+   delete cP;
    Th.Clear();
    L2E_nat.Clear();
    L2E_lex.Clear();
@@ -3264,7 +3220,6 @@ void FiniteElementSpace::Destroy()
    {
       delete x.second;
    }
-   L2F.clear();
    for (int i = 0; i < E2IFQ_array.Size(); i++)
    {
       delete E2IFQ_array[i];
@@ -3364,14 +3319,14 @@ void FiniteElementSpace::GetTrueTransferOperator(
       switch (RP_case)
       {
          case 1:
-            T.Reset(new ProductOperator(cR.get(), T.Ptr(), false, owner));
+            T.Reset(new ProductOperator(cR, T.Ptr(), false, owner));
             break;
          case 2:
             T.Reset(new ProductOperator(T.Ptr(), coarse_P, owner, false));
             break;
          case 3:
             T.Reset(new TripleProductOperator(
-                       cR.get(), T.Ptr(), coarse_P, false, owner, false));
+                       cR, T.Ptr(), coarse_P, false, owner, false));
             break;
       }
    }
@@ -3489,7 +3444,7 @@ void FiniteElementSpace::Update(bool want_transform)
             if (cP && cR)
             {
                Th.SetOperatorOwner(false);
-               Th.Reset(new TripleProductOperator(cP.get(), cR.get(), Th.Ptr(),
+               Th.Reset(new TripleProductOperator(cP, cR, Th.Ptr(),
                                                   false, false, true));
             }
             break;

fem/fespace.hpp

@@ -214,7 +214,7 @@ class FaceQuadratureInterpolator;
     @par
     Clearly the notion of a @b vdof is relevant in each of the three contexts
     mentioned above so extra care must be taken whenever @b vdim != 1 to ensure
-    that the @b edof, @b ldof, or @b tdof is being interpreted correctly.
+    that the @b edof, @b ldof, or @b tdof is being interpretted correctly.
  */
 class FiniteElementSpace
 {
@@ -277,14 +277,12 @@ protected:
    /** Matrix representing the prolongation from the global conforming dofs to
        a set of intermediate partially conforming dofs, e.g. the dofs associated
        with a "cut" space on a non-conforming mesh. */
-   mutable std::unique_ptr<SparseMatrix> cP;
+   mutable SparseMatrix *cP; // owned
    /// Conforming restriction matrix such that cR.cP=I.
-   mutable std::unique_ptr<SparseMatrix> cR;
+   mutable SparseMatrix *cR; // owned
    /// A version of the conforming restriction matrix for variable-order spaces.
-   mutable std::unique_ptr<SparseMatrix> cR_hp;
+   mutable SparseMatrix *cR_hp; // owned
    mutable bool cP_is_set;
-   /// Operator computing the action of the transpose of the restriction.
-   mutable std::unique_ptr<Operator> R_transpose;
 
    /// Transformation to apply to GridFunctions after space Update().
    OperatorHandle Th;
@@ -594,17 +592,10 @@ public:
    { return GetConformingProlongation(); }
 
    /// Return an operator that performs the transpose of GetRestrictionOperator
-   /** The returned operator is owned by the FiniteElementSpace.
-
-       For a serial conforming space, this returns NULL, indicating the identity
-       operator.
-
-       For a parallel conforming space, this will return a matrix-free
-       (Device)ConformingProlongationOperator.
-
-       For a non-conforming mesh this will return a TransposeOperator wrapping
-       the restriction matrix. */
-   const Operator *GetRestrictionTransposeOperator() const;
+   /** The returned operator is owned by the FiniteElementSpace. In serial this
+       is the same as GetProlongationMatrix() */
+   virtual const Operator *GetRestrictionTransposeOperator() const
+   { return GetConformingProlongation(); }
 
    /// An abstract operator that performs the same action as GetRestrictionMatrix
    /** In some cases this is an optimized matrix-free implementation. The
@@ -820,11 +811,6 @@ public:
    virtual DofTransformation *GetBdrElementDofs(int bel,
                                                 Array<int> &dofs) const;
 
-   /** @brief Returns indices of degrees of freedom for NURBS patch index
-    @a patch. Cartesian ordering is used, for the tensor-product degrees of
-    freedom. */
-   void GetPatchDofs(int patch, Array<int> &dofs) const;
-
    /// @brief Returns the indices of the degrees of freedom for the specified
    /// face, including the DOFs for the edges and the vertices of the face.
    ///
@@ -907,7 +893,7 @@ public:
    /// changed in the forward mappings by passing a value for @a ndofs which
    /// differs from that returned by GetNDofs().
    ///
-   /// @note These methods, with the exception of VDofToDof(), are designed to
+   /// @note Thse methods, with the exception of VDofToDof(), are designed to
    /// produce the correctly encoded values when dof entries are negative,
    /// see @ref ldof for more on negative dof indices.
    ///
@@ -1009,7 +995,7 @@ public:
 
    /// @brief Returns indices of degrees of freedom for the @a i'th element.
    /// The returned indices are offsets into an @ref ldof vector with @b vdim
-   /// not necessarily equal to 1. The returned indices are always ordered
+   /// not necessarily equal to 1. The returned indexes are always ordered
    /// byNODES, irrespective of whether the space is byNODES or byVDIM.
    /// See also GetElementDofs().
    ///
@@ -1038,9 +1024,6 @@ public:
    /// @note The returned object should NOT be deleted by the caller.
    DofTransformation *GetBdrElementVDofs(int i, Array<int> &vdofs) const;
 
-   /// Returns indices of degrees of freedom in @a vdofs for NURBS patch @a i.
-   void GetPatchVDofs(int i, Array<int> &vdofs) const;
-
    /// @brief Returns the indices of the degrees of freedom for the specified
    /// face, including the DOFs for the edges and the vertices of the face.
    ///
@@ -1124,9 +1107,7 @@ public:
    int GetLocalDofForDof(int i) const { return dof_ldof_array[i]; }
 
    /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection
-        associated with i'th element in the mesh object.
-        Note: The method has been updated to abort instead of returning NULL for
-        an empty partition. */
+        associated with i'th element in the mesh object. */
    virtual const FiniteElement *GetFE(int i) const;
 
    /** @brief Returns pointer to the FiniteElement in the FiniteElementCollection

fem/gridfunc.cpp

@@ -27,6 +27,7 @@
 #include <iostream>
 #include <algorithm>
 
+
 namespace mfem
 {
 
@@ -340,7 +341,7 @@ int GridFunction::VectorDim() const
       return fes->GetVDim();
    }
    return fes->GetVDim()*std::max(fes->GetMesh()->SpaceDimension(),
-                                  fe->GetRangeDim());
+                                  fe->GetVDim());
 }
 
 int GridFunction::CurlDim() const
@@ -1041,7 +1042,7 @@ void GridFunction::GetVectorValue(ElementTransformation &T,
    else
    {
       int spaceDim = fes->GetMesh()->SpaceDimension();
-      int vdim = std::max(spaceDim, fe->GetRangeDim());
+      int vdim = std::max(spaceDim, fe->GetVDim());
       DenseMatrix vshape(dof, vdim);
       fe->CalcVShape(T, vshape);
       val.SetSize(vdim);
@@ -1093,7 +1094,7 @@ void GridFunction::GetVectorValues(ElementTransformation &T,
    else
    {
       int spaceDim = fes->GetMesh()->SpaceDimension();
-      int vdim = std::max(spaceDim, FElem->GetRangeDim());
+      int vdim = std::max(spaceDim, FElem->GetVDim());
       DenseMatrix vshape(dof, vdim);
 
       vals.SetSize(vdim, nip);

fem/gslib.cpp

@@ -1236,7 +1236,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
    gsl_ref.SetSize(points_cnt * dim);
    gsl_dist.SetSize(points_cnt);
 
-   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[])
+   auto xvFill = [&](const double *xv_base[], unsigned xv_stride[], int dim)
    {
       for (int d = 0; d < dim; d++)
       {
@@ -1256,7 +1256,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[2];
       unsigned xv_stride[2];
-      xvFill(xv_base, xv_stride);
+      xvFill(xv_base, xv_stride, dim);
       findptsms_2(gsl_code.GetData(), sizeof(unsigned int),
                   gsl_proc.GetData(), sizeof(unsigned int),
                   gsl_elem.GetData(), sizeof(unsigned int),
@@ -1270,7 +1270,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
    {
       const double *xv_base[3];
       unsigned xv_stride[3];
-      xvFill(xv_base, xv_stride);
+      xvFill(xv_base, xv_stride, dim);
       findptsms_3(gsl_code.GetData(), sizeof(unsigned int),
                   gsl_proc.GetData(), sizeof(unsigned int),
                   gsl_elem.GetData(), sizeof(unsigned int),
@@ -1308,6 +1308,298 @@ void OversetFindPointsGSLIB::Interpolate(const Vector &point_pos,
    Interpolate(field_in, field_out);
 }
 
+#ifdef MFEM_USE_MPI
+
+GSLIBCommunicator::GSLIBCommunicator(MPI_Comm comm_)
+   : cr(NULL), gsl_comm(NULL)
+{
+   gsl_comm = new gslib::comm;
+   cr      = new gslib::crystal;
+   comm_init(gsl_comm, comm_);
+   crystal_init(cr, gsl_comm);
+}
+
+void GSLIBCommunicator::SendData(int dim, const Array<unsigned int> & gsl_proc,
+                                 const Array<unsigned int> & elem_send,
+                                 const Vector &ref_send,
+                                 const Vector &coords_send,
+                                 const Array<int> &s_conn_send,
+                                 Array<unsigned int> & proc_recv,
+                                 Array<unsigned int> & index_recv,
+                                 Array<unsigned int> & elem_recv,
+                                 Vector &ref_recv,
+                                 Vector &coords_recv,
+                                 Array<int> &s_conn_recv)
+{
+   int nptsend = gsl_proc.Size();
+   int nptElem   = elem_send.Size();
+   int nptRST    = ref_send.Size();
+
+   MFEM_VERIFY(nptElem == nptsend,
+               "Incompatible Elem size.");
+   MFEM_VERIFY(nptsend*dim == nptRST,
+               "Incompatible nptRST size.");
+   MFEM_VERIFY(dim <= 3,
+               "Incompatible dimension.");
+
+   // Pack data to send via crystal router
+   struct gslib::array *outpt = new gslib::array;
+
+   struct out_pt { double rst[3], coords[3]; int s_conn; uint index, elem, proc; };
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->index = index;
+      pt->elem = elem_send[index];
+      pt->proc  = gsl_proc[index];
+      pt->s_conn = s_conn_send[index];
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->rst[d]= ref_send(index*dim + d);
+         pt->coords[d]= coords_send(index + d*nptsend);
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+   // unpack
+   int npt = outpt->n;
+   proc_recv.SetSize(npt);
+   elem_recv.SetSize(npt);
+   index_recv.SetSize(npt);
+   ref_recv.SetSize(npt*dim);
+   coords_recv.SetSize(npt*dim);
+   s_conn_recv.SetSize(npt);
+
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      index_recv[index] = pt->index;
+      elem_recv[index] = pt->elem;
+      proc_recv[index] = pt->proc;
+      s_conn_recv[index] = pt->s_conn;
+      for (int d = 0; d < dim; ++d)
+      {
+         ref_recv(index*dim + d)= pt->rst[d]; // by VDIM
+         coords_recv(index + d*npt)= pt->coords[d]; // by NODES
+      }
+      ++pt;
+   }
+
+   array_free(outpt);
+   delete outpt;
+}
+
+void GSLIBCommunicator::SendData2(int dim,
+                                  const Array<unsigned int> & gsl_proc,
+                                  const Vector &xyz_send,
+                                  const Vector &xi_send,
+                                  const Array<int> &s_conn_send,
+                                  const Array<int> &conn_send,
+                                  const DenseMatrix &coords_send,
+                                  Vector &xyz_recv,
+                                  Vector &xi_recv,
+                                  Array<int> &s_conn_recv,
+                                  Array<int> &conn_recv,
+                                  DenseMatrix &coords_recv)
+{
+   int nptsend = gsl_proc.Size();
+
+   struct gslib::array *outpt = new gslib::array;
+   struct out_pt {double xyz[3], xi[2], coords[12]; int s_conn; int conn[4]; uint proc;};
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->proc  = gsl_proc[index];
+      pt->s_conn  = s_conn_send[index];
+      for (int d = 0; d < dim-1; ++d)
+      {
+         pt->xi[d]= xi_send(index*(dim-1) + d);
+      }
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->xyz[d]= xyz_send(index + d*nptsend);
+      }
+      for (int j = 0; j<4; j++)
+      {
+         pt->conn[j] = conn_send[index*4+j];
+         for (int d = 0; d < dim; ++d)
+         {
+            pt->coords[j*dim+d]= coords_send(index*4+j,d);
+         }
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+   // unpack
+   int npt = outpt->n;
+   xi_recv.SetSize(npt*(dim-1));
+   xyz_recv.SetSize(npt*dim);
+   s_conn_recv.SetSize(npt);
+   conn_recv.SetSize(npt*4);
+   coords_recv.SetSize(npt*4,dim);
+
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      s_conn_recv[index] = pt->s_conn;
+      for (int d = 0; d < dim-1; ++d)
+      {
+         xi_recv(index*(dim-1) + d) = pt->xi[d];
+      }
+      for (int d = 0; d < dim; ++d)
+      {
+         xyz_recv(index + d*npt)= pt->xyz[d]; // by NODES
+      }
+      for (int j = 0; j<4; j++)
+      {
+         conn_recv[index*4+j] = pt->conn[j];
+         for (int d = 0; d < dim; ++d)
+         {
+            coords_recv(index*4+j,d) = pt->coords[j*dim+d];
+         }
+      }
+      ++pt;
+   }
+   array_free(outpt);
+   delete outpt;
+}
+
+
+void GSLIBCommunicator::ExchangeNormal(Mesh & mesh,
+                                       const Array<unsigned int> &gsl_proc,
+                                       const Array<unsigned int> &gsl_mfem_elem,
+                                       const Vector &gsl_mfem_ref,
+                                       Vector &recv_normals)
+{
+   int dim = mesh.Dimension();
+   int nptsend = gsl_proc.Size();
+   int nptElem   = gsl_mfem_elem.Size();
+   int nptRST    = gsl_mfem_ref.Size();
+
+   recv_normals.SetSize(nptRST);
+   int nptNormal    = recv_normals.Size();
+
+   MFEM_VERIFY(nptElem == nptsend,
+               "Incompatible Elem size.");
+   MFEM_VERIFY(nptsend*dim == nptRST,
+               "Incompatible nptRST size.");
+   MFEM_VERIFY(dim <= 3,
+               "Incompatible dimension.");
+
+   // Pack data to send via crystal router
+   struct gslib::array *outpt = new gslib::array;
+
+   struct out_pt { double rst[3]; uint index, elem, proc; };
+   struct out_pt *pt;
+   array_init(struct out_pt, outpt, nptsend);
+   outpt->n=nptsend;
+   pt = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < nptsend; index++)
+   {
+      pt->index = index;
+      pt->elem = gsl_mfem_elem[index];
+      pt->proc  = gsl_proc[index];
+      for (int d = 0; d < dim; ++d)
+      {
+         pt->rst[d]= gsl_mfem_ref(index*dim + d);
+      }
+      ++pt;
+   }
+
+   // Transfer data to target MPI ranks
+   sarray_transfer(struct out_pt, outpt, proc, 1, cr);
+
+   // Get normal vector
+   int npt = outpt->n;
+   pt = (struct out_pt *)outpt->ptr;
+   Vector normal(npt*dim);
+   for (int index = 0; index < npt; index++)
+   {
+      IntegrationPoint ip;
+      ip.Set3(&pt->rst[0]);
+      Vector localval(normal.GetData()+index*dim, dim);
+      // get the normal at this integration point here
+      // for now I just put back this proc's rank + the input rst coordinates
+      for (int d = 0; d < dim; d++)
+      {
+         localval(d) = gsl_comm->id + pt->rst[d];
+      }
+      ++pt;
+   }
+
+   // Save index and proc data in a struct
+   struct gslib::array *savpt = new gslib::array;
+   struct sav_pt { uint index, proc; };
+   struct sav_pt *spt;
+   array_init(struct sav_pt, savpt, npt);
+   savpt->n=npt;
+   spt = (struct sav_pt *)savpt->ptr;
+   pt  = (struct out_pt *)outpt->ptr;
+   for (int index = 0; index < npt; index++)
+   {
+      spt->index = pt->index;
+      spt->proc  = pt->proc;
+      ++pt; ++spt;
+   }
+
+   array_free(outpt);
+   delete outpt;
+
+   // Copy data from save struct to send struct and send component wise
+   struct gslib::array *sendpt = new gslib::array;
+   struct send_pt { double ival; uint index, proc; };
+   struct send_pt *sdpt;
+   for (int j = 0; j < dim; j++)
+   {
+      array_init(struct send_pt, sendpt, npt);
+      sendpt->n=npt;
+      spt  = (struct sav_pt *)savpt->ptr;
+      sdpt = (struct send_pt *)sendpt->ptr;
+      for (int index = 0; index < npt; index++)
+      {
+         sdpt->index = spt->index;
+         sdpt->proc  = spt->proc;
+         sdpt->ival  = normal(j + index*dim);
+         ++sdpt; ++spt;
+      }
+
+      sarray_transfer(struct send_pt, sendpt, proc, 1, cr);
+      sdpt = (struct send_pt *)sendpt->ptr;
+      for (int index = 0; index < static_cast<int>(sendpt->n); index++)
+      {
+         int idx = sdpt->index*dim + j;
+         recv_normals(idx) = sdpt->ival;
+         ++sdpt;
+      }
+      array_free(sendpt);
+   }
+   array_free(savpt);
+   delete sendpt;
+   delete savpt;
+}
+
+void GSLIBCommunicator::FreeData()
+{
+   crystal_free(cr);
+}
+
+GSLIBCommunicator::~GSLIBCommunicator()
+{
+   delete gsl_comm;
+   delete cr;
+}
+#endif
 
 } // namespace mfem
 

fem/gslib.hpp

@@ -290,6 +290,55 @@ public:
    using FindPointsGSLIB::Interpolate;
 };
 
+#ifdef MFEM_USE_MPI
+// Use to send info to certain processes
+class GSLIBCommunicator
+{
+protected:
+   struct gslib::crystal *cr;             // gslib's internal data
+   struct gslib::comm *gsl_comm;          // gslib's internal data
+
+public:
+   GSLIBCommunicator(MPI_Comm comm_);
+
+   virtual ~GSLIBCommunicator();
+
+   void ExchangeNormal(Mesh& mesh,
+                       const Array<unsigned int> &gsl_proc,
+                       const Array<unsigned int> &gsl_mfem_elem,
+                       const Vector &gsl_mfem_ref,
+                       Vector &recv_normals); //npt*dim
+
+   void SendData(int dim,
+                 const Array<unsigned int> & gsl_proc,
+                 const Array<unsigned int> & elem_send,
+                 const Vector &ref_send,
+                 const Vector &coords_send,
+                 const Array<int> &s_conn_send,
+                 Array<unsigned int> & proc_recv,
+                 Array<unsigned int> & index_recv,
+                 Array<unsigned int> & elem_recv,
+                 Vector &ref_recv,
+                 Vector &coords_recv,
+                 Array<int> & s_conn_recv);
+
+   void SendData2(int dim,
+                  const Array<unsigned int> & gsl_proc,
+                  const Vector &xyz_send,
+                  const Vector &xi_send,
+                  const Array<int> &s_conn_send,
+                  const Array<int> &conn_send,
+                  const DenseMatrix &coords_send,
+                  Vector &xyz_recv,
+                  Vector &ref_recv,
+                  Array<int> &s_conn_recv,
+                  Array<int> &conn_recv,
+                  DenseMatrix &coords_recv);
+
+   virtual void FreeData();
+};
+#endif
+
 } // namespace mfem
 
 #endif // MFEM_USE_GSLIB

fem/integ/bilininteg_convection_ea.cpp

@@ -28,8 +28,8 @@ static void EAConvectionAssemble1D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
@@ -38,7 +38,7 @@ static void EAConvectionAssemble1D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_Gi[MQ1];
       double r_Bj[MQ1];
       for (int q = 0; q < Q1D; q++)
@@ -80,8 +80,8 @@ static void EAConvectionAssemble2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, NE);
@@ -90,8 +90,8 @@ static void EAConvectionAssemble2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       double r_G[MQ1][MD1];
       for (int d = 0; d < D1D; d++)
@@ -157,8 +157,8 @@ static void EAConvectionAssemble3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 3, NE);
@@ -167,8 +167,8 @@ static void EAConvectionAssemble3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       double r_G[MQ1][MD1];
       for (int d = 0; d < D1D; d++)

fem/integ/bilininteg_convection_pa.cpp

@@ -203,8 +203,8 @@ void PAConvectionApply2D(const int ne,
    const int NE = ne;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -216,8 +216,8 @@ void PAConvectionApply2D(const int ne,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double u[max_D1D][max_D1D];
       for (int dy = 0; dy < D1D; ++dy)
@@ -323,8 +323,8 @@ void SmemPAConvectionApply2D(const int ne,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -338,8 +338,8 @@ void SmemPAConvectionApply2D(const int ne,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       // constexpr int MDQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D;
       MFEM_SHARED double u[NBZ][max_D1D][max_D1D];
       MFEM_FOREACH_THREAD(dy,y,D1D)
@@ -450,8 +450,8 @@ void PAConvectionApply3D(const int ne,
    const int NE = ne;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -463,8 +463,8 @@ void PAConvectionApply3D(const int ne,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double u[max_D1D][max_D1D][max_D1D];
       for (int dz = 0; dz < D1D; ++dz)
@@ -631,8 +631,8 @@ void SmemPAConvectionApply3D(const int ne,
    const int NE = ne;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -644,8 +644,8 @@ void SmemPAConvectionApply3D(const int ne,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D;
       MFEM_SHARED double sm0[max_DQ*max_DQ*max_DQ];
       MFEM_SHARED double sm1[max_DQ*max_DQ*max_DQ];
@@ -835,8 +835,8 @@ void PAConvectionApplyT2D(const int ne,
    const int NE = ne;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto Gt = Reshape(gt.Read(), D1D, Q1D);
@@ -848,8 +848,8 @@ void PAConvectionApplyT2D(const int ne,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double u[max_D1D][max_D1D];
       for (int dy = 0; dy < D1D; ++dy)
@@ -951,8 +951,8 @@ void SmemPAConvectionApplyT2D(const int ne,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto Gt = Reshape(gt.Read(), D1D, Q1D);
@@ -966,8 +966,8 @@ void SmemPAConvectionApplyT2D(const int ne,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       MFEM_SHARED double u[NBZ][max_D1D][max_D1D];
       MFEM_FOREACH_THREAD(dy,y,D1D)
       {
@@ -1073,8 +1073,8 @@ void PAConvectionApplyT3D(const int ne,
    const int NE = ne;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto Gt = Reshape(gt.Read(), D1D, Q1D);
@@ -1086,8 +1086,8 @@ void PAConvectionApplyT3D(const int ne,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double u[max_D1D][max_D1D][max_D1D];
       for (int dz = 0; dz < D1D; ++dz)
@@ -1249,8 +1249,8 @@ void SmemPAConvectionApplyT3D(const int ne,
    const int NE = ne;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto Gt = Reshape(gt.Read(), D1D, Q1D);
@@ -1262,8 +1262,8 @@ void SmemPAConvectionApplyT3D(const int ne,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       constexpr int max_DQ = (max_Q1D > max_D1D) ? max_Q1D : max_D1D;
       MFEM_SHARED double sm0[3*max_DQ*max_DQ*max_DQ];
       MFEM_SHARED double sm1[3*max_DQ*max_DQ*max_DQ];

fem/integ/bilininteg_dgtrace_ea.cpp

@@ -83,8 +83,8 @@ static void EADGTraceAssemble2DInt(const int NF,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), D1D, D1D, 2, NF);
@@ -138,8 +138,8 @@ static void EADGTraceAssemble2DBdr(const int NF,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), D1D, D1D, NF);
@@ -181,8 +181,8 @@ static void EADGTraceAssemble3DInt(const int NF,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, 2, NF);
    auto A_int = Reshape(eadata_int.ReadWrite(), D1D, D1D, D1D, D1D, 2, NF);
@@ -191,8 +191,8 @@ static void EADGTraceAssemble3DInt(const int NF,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       for (int d = 0; d < D1D; d++)
       {
@@ -278,8 +278,8 @@ static void EADGTraceAssemble3DBdr(const int NF,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 2, 2, NF);
    auto A_bdr = Reshape(eadata_bdr.ReadWrite(), D1D, D1D, D1D, D1D, NF);
@@ -287,8 +287,8 @@ static void EADGTraceAssemble3DBdr(const int NF,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       for (int d = 0; d < D1D; d++)
       {

fem/integ/bilininteg_dgtrace_pa.cpp

@@ -258,8 +258,8 @@ void PADGTraceApply2D(const int NF,
    const int VDIM = 1;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
@@ -272,8 +272,8 @@ void PADGTraceApply2D(const int NF,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double u0[max_D1D][VDIM];
       double u1[max_D1D][VDIM];
       for (int d = 0; d < D1D; d++)
@@ -349,8 +349,8 @@ void PADGTraceApply3D(const int NF,
    const int VDIM = 1;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
@@ -363,8 +363,8 @@ void PADGTraceApply3D(const int NF,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double u0[max_D1D][max_D1D][VDIM];
       double u1[max_D1D][max_D1D][VDIM];
       for (int d1 = 0; d1 < D1D; d1++)
@@ -494,8 +494,8 @@ void SmemPADGTraceApply3D(const int NF,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
@@ -509,8 +509,8 @@ void SmemPADGTraceApply3D(const int NF,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       MFEM_SHARED double u0[NBZ][max_D1D][max_D1D];
       MFEM_SHARED double u1[NBZ][max_D1D][max_D1D];
       MFEM_FOREACH_THREAD(d1,x,D1D)
@@ -659,8 +659,8 @@ void PADGTraceApplyTranspose2D(const int NF,
    const int VDIM = 1;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
@@ -673,8 +673,8 @@ void PADGTraceApplyTranspose2D(const int NF,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double u0[max_D1D][VDIM];
       double u1[max_D1D][VDIM];
       for (int d = 0; d < D1D; d++)
@@ -755,8 +755,8 @@ void PADGTraceApplyTranspose3D(const int NF,
    const int VDIM = 1;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
@@ -769,8 +769,8 @@ void PADGTraceApplyTranspose3D(const int NF,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double u0[max_D1D][max_D1D][VDIM];
       double u1[max_D1D][max_D1D][VDIM];
       for (int d1 = 0; d1 < D1D; d1++)
@@ -911,8 +911,8 @@ void SmemPADGTraceApplyTranspose3D(const int NF,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
@@ -926,8 +926,8 @@ void SmemPADGTraceApplyTranspose3D(const int NF,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       MFEM_SHARED double u0[NBZ][max_D1D][max_D1D];
       MFEM_SHARED double u1[NBZ][max_D1D][max_D1D];
       MFEM_FOREACH_THREAD(d1,x,D1D)

fem/integ/bilininteg_diffusion_ea.cpp

@@ -28,8 +28,8 @@ static void EADiffusionAssemble1D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
    auto A = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
@@ -37,7 +37,7 @@ static void EADiffusionAssemble1D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_Gi[MQ1];
       double r_Gj[MQ1];
       for (int q = 0; q < Q1D; q++)
@@ -79,8 +79,8 @@ static void EADiffusionAssemble2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, 3, NE);
@@ -89,8 +89,8 @@ static void EADiffusionAssemble2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       double r_G[MQ1][MD1];
       for (int d = 0; d < D1D; d++)
@@ -156,8 +156,8 @@ static void EADiffusionAssemble3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
@@ -166,8 +166,8 @@ static void EADiffusionAssemble3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       double r_G[MQ1][MD1];
       for (int d = 0; d < D1D; d++)

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -98,8 +98,8 @@ inline void PADiffusionDiagonal2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    // note the different shape for D, if this is a symmetric matrix we only
@@ -110,8 +110,8 @@ inline void PADiffusionDiagonal2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       // gradphi \cdot Q \gradphi has four terms
       double QD0[MQ1][MD1];
       double QD1[MQ1][MD1];
@@ -165,10 +165,10 @@ inline void SmemPADiffusionDiagonal2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
@@ -179,8 +179,8 @@ inline void SmemPADiffusionDiagonal2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
       MFEM_SHARED double BG[2][MQ1*MD1];
       double (*B)[MD1] = (double (*)[MD1]) (BG+0);
       double (*G)[MD1] = (double (*)[MD1]) (BG+1);
@@ -260,10 +260,10 @@ inline void PADiffusionDiagonal3D(const int NE,
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
@@ -272,8 +272,8 @@ inline void PADiffusionDiagonal3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double QQD[MQ1][MQ1][MD1];
       double QDD[MQ1][MD1][MD1];
       for (int i = 0; i < DIM; ++i)
@@ -361,10 +361,10 @@ inline void SmemPADiffusionDiagonal3D(const int NE,
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D*Q1D*Q1D, symmetric ? 6 : 9, NE);
@@ -374,8 +374,8 @@ inline void SmemPADiffusionDiagonal3D(const int NE,
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
       MFEM_SHARED double BG[2][MQ1*MD1];
       double (*B)[MD1] = (double (*)[MD1]) (BG+0);
       double (*G)[MD1] = (double (*)[MD1]) (BG+1);
@@ -521,8 +521,8 @@ inline void PADiffusionApply2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b_.Read(), Q1D, D1D);
    auto G = Reshape(g_.Read(), Q1D, D1D);
    auto Bt = Reshape(bt_.Read(), D1D, Q1D);
@@ -535,8 +535,8 @@ inline void PADiffusionApply2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double grad[max_Q1D][max_Q1D][2];
       for (int qy = 0; qy < Q1D; ++qy)
@@ -642,10 +642,10 @@ inline void SmemPADiffusionApply2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D*Q1D, symmetric ? 3 : 4, NE);
@@ -657,8 +657,8 @@ inline void SmemPADiffusionApply2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
       MFEM_SHARED double sBG[2][MQ1*MD1];
       double (*B)[MD1] = (double (*)[MD1]) (sBG+0);
       double (*G)[MD1] = (double (*)[MD1]) (sBG+1);
@@ -800,8 +800,8 @@ inline void PADiffusionApply3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -813,8 +813,8 @@ inline void PADiffusionApply3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double grad[max_Q1D][max_Q1D][max_Q1D][3];
       for (int qz = 0; qz < Q1D; ++qz)
       {
@@ -992,10 +992,10 @@ inline void SmemPADiffusionApply3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= M1D, "");
+   MFEM_VERIFY(Q1D <= M1Q, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto g = Reshape(g_.Read(), Q1D, D1D);
    auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
@@ -1005,8 +1005,8 @@ inline void SmemPADiffusionApply3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
       constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
       MFEM_SHARED double sBG[2][MQ1*MD1];
       double (*B)[MD1] = (double (*)[MD1]) (sBG+0);

fem/integ/bilininteg_diffusion_pa.cpp

@@ -12,7 +12,6 @@
 #include "../bilininteg.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
-#include "../../mesh/nurbs.hpp"
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 #include "bilininteg_diffusion_kernels.hpp"
 
@@ -75,29 +74,6 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
                               ir->GetWeights(), geom->J, coeff, pa_data);
 }
 
-void DiffusionIntegrator::AssembleNURBSPA(const FiniteElementSpace &fes)
-{
-   fespace = &fes;
-   Mesh *mesh = fes.GetMesh();
-   dim = mesh->Dimension();
-   MFEM_VERIFY(3 == dim, "Only 3D so far");
-
-   numPatches = mesh->NURBSext->GetNP();
-   for (int p=0; p<numPatches; ++p)
-   {
-      AssemblePatchPA(p, fes);
-   }
-}
-
-void DiffusionIntegrator::AssemblePatchPA(const int patch,
-                                          const FiniteElementSpace &fes)
-{
-   Mesh *mesh = fes.GetMesh();
-   SetupPatchBasisData(mesh, patch);
-
-   SetupPatchPA(patch, mesh);  // For full quadrature, unitWeights = false
-}
-
 void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 {
    if (DeviceCanUseCeed())
@@ -139,221 +115,4 @@ void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
    }
 }
 
-// This version uses full 1D quadrature rules, taking into account the
-// minimum interaction between basis functions and integration points.
-void DiffusionIntegrator::AddMultPatchPA(const int patch, const Vector &x,
-                                         Vector &y) const
-{
-   MFEM_VERIFY(3 == dim, "Only 3D so far");
-
-   const Array<int>& Q1D = pQ1D[patch];
-   const Array<int>& D1D = pD1D[patch];
-
-   const std::vector<Array2D<double>>& B = pB[patch];
-   const std::vector<Array2D<double>>& G = pG[patch];
-
-   const IntArrayVar2D& minD = pminD[patch];
-   const IntArrayVar2D& maxD = pmaxD[patch];
-   const IntArrayVar2D& minQ = pminQ[patch];
-   const IntArrayVar2D& maxQ = pmaxQ[patch];
-
-   auto X = Reshape(x.Read(), D1D[0], D1D[1], D1D[2]);
-   auto Y = Reshape(y.ReadWrite(), D1D[0], D1D[1], D1D[2]);
-
-   const auto qd = Reshape(pa_data.Read(), Q1D[0]*Q1D[1]*Q1D[2],
-                           (symmetric ? 6 : 9));
-
-   // NOTE: the following is adapted from AssemblePatchMatrix_fullQuadrature
-   std::vector<Array3D<double>> grad(dim);
-   // TODO: Can an optimal order of dimensions be determined, for each patch?
-   Array3D<double> gradXY(3, std::max(Q1D[0], D1D[0]), std::max(Q1D[1], D1D[1]));
-   Array2D<double> gradX(3, std::max(Q1D[0], D1D[0]));
-
-   for (int d=0; d<dim; ++d)
-   {
-      grad[d].SetSize(Q1D[0], Q1D[1], Q1D[2]);
-
-      for (int qz = 0; qz < Q1D[2]; ++qz)
-      {
-         for (int qy = 0; qy < Q1D[1]; ++qy)
-         {
-            for (int qx = 0; qx < Q1D[0]; ++qx)
-            {
-               grad[d](qx,qy,qz) = 0.0;
-            }
-         }
-      }
-   }
-
-   for (int dz = 0; dz < D1D[2]; ++dz)
-   {
-      for (int qy = 0; qy < Q1D[1]; ++qy)
-      {
-         for (int qx = 0; qx < Q1D[0]; ++qx)
-         {
-            for (int d=0; d<dim; ++d)
-            {
-               gradXY(d,qx,qy) = 0.0;
-            }
-         }
-      }
-      for (int dy = 0; dy < D1D[1]; ++dy)
-      {
-         for (int qx = 0; qx < Q1D[0]; ++qx)
-         {
-            gradX(0,qx) = 0.0;
-            gradX(1,qx) = 0.0;
-         }
-         for (int dx = 0; dx < D1D[0]; ++dx)
-         {
-            const double s = X(dx,dy,dz);
-            for (int qx = minD[0][dx]; qx <= maxD[0][dx]; ++qx)
-            {
-               gradX(0,qx) += s * B[0](qx,dx);
-               gradX(1,qx) += s * G[0](qx,dx);
-            }
-         }
-         for (int qy = minD[1][dy]; qy <= maxD[1][dy]; ++qy)
-         {
-            const double wy  = B[1](qy,dy);
-            const double wDy = G[1](qy,dy);
-            // This full range of qx values is generally necessary.
-            for (int qx = 0; qx < Q1D[0]; ++qx)
-            {
-               const double wx  = gradX(0,qx);
-               const double wDx = gradX(1,qx);
-               gradXY(0,qx,qy) += wDx * wy;
-               gradXY(1,qx,qy) += wx  * wDy;
-               gradXY(2,qx,qy) += wx  * wy;
-            }
-         }
-      }
-      for (int qz = minD[2][dz]; qz <= maxD[2][dz]; ++qz)
-      {
-         const double wz  = B[2](qz,dz);
-         const double wDz = G[2](qz,dz);
-         for (int qy = 0; qy < Q1D[1]; ++qy)
-         {
-            for (int qx = 0; qx < Q1D[0]; ++qx)
-            {
-               grad[0](qx,qy,qz) += gradXY(0,qx,qy) * wz;
-               grad[1](qx,qy,qz) += gradXY(1,qx,qy) * wz;
-               grad[2](qx,qy,qz) += gradXY(2,qx,qy) * wDz;
-            }
-         }
-      }
-   }
-
-   for (int qz = 0; qz < Q1D[2]; ++qz)
-   {
-      for (int qy = 0; qy < Q1D[1]; ++qy)
-      {
-         for (int qx = 0; qx < Q1D[0]; ++qx)
-         {
-            const int q = qx + ((qy + (qz * Q1D[1])) * Q1D[0]);
-            const double O00 = qd(q,0);
-            const double O01 = qd(q,1);
-            const double O02 = qd(q,2);
-            const double O10 = symmetric ? O01 : qd(q,3);
-            const double O11 = symmetric ? qd(q,3) : qd(q,4);
-            const double O12 = symmetric ? qd(q,4) : qd(q,5);
-            const double O20 = symmetric ? O02 : qd(q,6);
-            const double O21 = symmetric ? O12 : qd(q,7);
-            const double O22 = symmetric ? qd(q,5) : qd(q,8);
-
-            const double grad0 = grad[0](qx,qy,qz);
-            const double grad1 = grad[1](qx,qy,qz);
-            const double grad2 = grad[2](qx,qy,qz);
-
-            grad[0](qx,qy,qz) = (O00*grad0)+(O01*grad1)+(O02*grad2);
-            grad[1](qx,qy,qz) = (O10*grad0)+(O11*grad1)+(O12*grad2);
-            grad[2](qx,qy,qz) = (O20*grad0)+(O21*grad1)+(O22*grad2);
-         } // qx
-      } // qy
-   } // qz
-
-   for (int qz = 0; qz < Q1D[2]; ++qz)
-   {
-      for (int dy = 0; dy < D1D[1]; ++dy)
-      {
-         for (int dx = 0; dx < D1D[0]; ++dx)
-         {
-            for (int d=0; d<3; ++d)
-            {
-               gradXY(d,dx,dy) = 0.0;
-            }
-         }
-      }
-      for (int qy = 0; qy < Q1D[1]; ++qy)
-      {
-         for (int dx = 0; dx < D1D[0]; ++dx)
-         {
-            for (int d=0; d<3; ++d)
-            {
-               gradX(d,dx) = 0.0;
-            }
-         }
-         for (int qx = 0; qx < Q1D[0]; ++qx)
-         {
-            const double gX = grad[0](qx,qy,qz);
-            const double gY = grad[1](qx,qy,qz);
-            const double gZ = grad[2](qx,qy,qz);
-            for (int dx = minQ[0][qx]; dx <= maxQ[0][qx]; ++dx)
-            {
-               const double wx  = B[0](qx,dx);
-               const double wDx = G[0](qx,dx);
-               gradX(0,dx) += gX * wDx;
-               gradX(1,dx) += gY * wx;
-               gradX(2,dx) += gZ * wx;
-            }
-         }
-         for (int dy = minQ[1][qy]; dy <= maxQ[1][qy]; ++dy)
-         {
-            const double wy  = B[1](qy,dy);
-            const double wDy = G[1](qy,dy);
-            for (int dx = 0; dx < D1D[0]; ++dx)
-            {
-               gradXY(0,dx,dy) += gradX(0,dx) * wy;
-               gradXY(1,dx,dy) += gradX(1,dx) * wDy;
-               gradXY(2,dx,dy) += gradX(2,dx) * wy;
-            }
-         }
-      }
-      for (int dz = minQ[2][qz]; dz <= maxQ[2][qz]; ++dz)
-      {
-         const double wz  = B[2](qz,dz);
-         const double wDz = G[2](qz,dz);
-         for (int dy = 0; dy < D1D[1]; ++dy)
-         {
-            for (int dx = 0; dx < D1D[0]; ++dx)
-            {
-               Y(dx,dy,dz) +=
-                  ((gradXY(0,dx,dy) * wz) +
-                   (gradXY(1,dx,dy) * wz) +
-                   (gradXY(2,dx,dy) * wDz));
-            }
-         }
-      } // dz
-   } // qz
-}
-
-void DiffusionIntegrator::AddMultNURBSPA(const Vector &x, Vector &y) const
-{
-   Vector xp, yp;
-
-   for (int p=0; p<numPatches; ++p)
-   {
-      Array<int> vdofs;
-      fespace->GetPatchVDofs(p, vdofs);
-
-      x.GetSubVector(vdofs, xp);
-      yp.SetSize(vdofs.Size());
-      yp = 0.0;
-
-      AddMultPatchPA(p, xp, yp);
-
-      y.AddElementVector(vdofs, yp);
-   }
-}
-
 } // namespace mfem

fem/integ/bilininteg_gradient_pa.cpp

@@ -229,9 +229,9 @@ static void PAGradientApply2D(const int NE,
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, TR_D1D);
    auto G = Reshape(g.Read(), Q1D, TR_D1D);
    auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
@@ -245,8 +245,8 @@ static void PAGradientApply2D(const int NE,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = 2;
       // the following variables are evaluated at compile time
-      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double grad[max_Q1D][max_Q1D][VDIM];
       for (int qy = 0; qy < Q1D; ++qy)
@@ -359,9 +359,9 @@ static void PAGradientApply3D(const int NE,
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, TR_D1D);
    auto G = Reshape(g.Read(), Q1D, TR_D1D);
    auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
@@ -375,8 +375,8 @@ static void PAGradientApply3D(const int NE,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = 3;
       // the following variables are evaluated at compile time
-      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double grad[max_Q1D][max_Q1D][max_Q1D][VDIM];
       for (int qz = 0; qz < Q1D; ++qz)
@@ -555,11 +555,11 @@ static void SmemPAGradientApply3D(const int NE,
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
    MFEM_VERIFY(TR_D1D <= Q1D, "");
    MFEM_VERIFY(TE_D1D <= Q1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
 
    auto b = Reshape(b_.Read(), Q1D, TR_D1D);
    auto g = Reshape(g_.Read(), Q1D, TR_D1D);
@@ -575,9 +575,9 @@ static void SmemPAGradientApply3D(const int NE,
       const int D1DR = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int D1DE = T_TE_D1D ? T_TE_D1D : te_d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1R = T_TR_D1D ? T_TR_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MD1E = T_TE_D1D ? T_TE_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1R = T_TR_D1D ? T_TR_D1D : MAX_D1D;
+      constexpr int MD1E = T_TE_D1D ? T_TE_D1D : MAX_D1D;
       constexpr int MD1 = MD1E > MD1R ? MD1E : MD1R;
       constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
       MFEM_SHARED double sBG[2][MQ1*MD1];

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -26,6 +26,9 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
                                    const Vector &pa_data,
                                    Vector &diag)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto op = Reshape(pa_data.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
@@ -33,9 +36,6 @@ void PAHcurlMassAssembleDiagonal2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       int osc = 0;
 
       for (int c = 0; c < VDIM; ++c)  // loop over x, y components
@@ -83,10 +83,11 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
                                    const Vector &pa_data,
                                    Vector &diag)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: Q1D > MAX_Q1D");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
@@ -96,8 +97,6 @@ void PAHcurlMassAssembleDiagonal3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       int osc = 0;
 
       for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
@@ -159,6 +158,10 @@ void PAHcurlMassApply2D(const int D1D,
                         const Vector &x,
                         Vector &y)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -169,10 +172,6 @@ void PAHcurlMassApply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -289,10 +288,11 @@ void PAHcurlMassApply3D(const int D1D,
                         const Vector &x,
                         Vector &y)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: Q1D > MAX_Q1D");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
@@ -305,9 +305,6 @@ void PAHcurlMassApply3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qz = 0; qz < Q1D; ++qz)
@@ -607,6 +604,9 @@ void PACurlCurlAssembleDiagonal2D(const int D1D,
                                   const Vector &pa_data,
                                   Vector &diag)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Gc = Reshape(gc.Read(), Q1D, D1D);
    auto op = Reshape(pa_data.Read(), Q1D, Q1D, NE);
@@ -614,9 +614,6 @@ void PACurlCurlAssembleDiagonal2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       int osc = 0;
 
       for (int c = 0; c < VDIM; ++c)  // loop over x, y components
@@ -664,6 +661,9 @@ void PACurlCurlApply2D(const int D1D,
                        const Vector &x,
                        Vector &y)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
 
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -675,10 +675,6 @@ void PACurlCurlApply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double curl[MAX_Q1D][MAX_Q1D];
 
       // curl[qy][qx] will be computed as du_y/dx - du_x/dy
@@ -828,6 +824,9 @@ void PAHcurlL2Apply2D(const int D1D,
                       const Vector &x, // trial = H(curl)
                       Vector &y)  // test = L2 or H1
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
    const int H1 = (D1Dtest == D1D);
 
    MFEM_VERIFY(y.Size() == NE*D1Dtest*D1Dtest, "Test vector of wrong dimension");
@@ -842,10 +841,6 @@ void PAHcurlL2Apply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double curl[MAX_Q1D][MAX_Q1D];
 
       // curl[qy][qx] will be computed as du_y/dx - du_x/dy
@@ -944,6 +939,9 @@ void PAHcurlL2ApplyTranspose2D(const int D1D,
                                const Vector &x, // trial = H(curl)
                                Vector &y)  // test = L2 or H1
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
    const int H1 = (D1Dtest == D1D);
 
    MFEM_VERIFY(x.Size() == NE*D1Dtest*D1Dtest, "Test vector of wrong dimension");
@@ -958,10 +956,6 @@ void PAHcurlL2ApplyTranspose2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D];
 
       // Zero-order term in L2 or H1 test space

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -59,10 +59,8 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
                                               const Vector &pa_data,
                                               Vector &diag)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -74,8 +72,8 @@ inline void SmemPAHcurlMassAssembleDiagonal3D(const int d1d,
    mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -220,10 +218,8 @@ inline void SmemPAHcurlMassApply3D(const int d1d,
                                    const Vector &x,
                                    Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -238,8 +234,8 @@ inline void SmemPAHcurlMassApply3D(const int d1d,
    mfem::forall_3D(NE, Q1D, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int e)
    {
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -445,10 +441,8 @@ inline void PACurlCurlAssembleDiagonal3D(const int d1d,
                                          const Vector &pa_data,
                                          Vector &diag)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -482,8 +476,8 @@ inline void PACurlCurlAssembleDiagonal3D(const int d1d,
       // which may be non-symmetric depending on a possibly non-symmetric matrix coefficient.
 
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -641,10 +635,8 @@ inline void SmemPACurlCurlAssembleDiagonal3D(const int d1d,
                                              const Vector &pa_data,
                                              Vector &diag)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -675,8 +667,8 @@ inline void SmemPACurlCurlAssembleDiagonal3D(const int d1d,
       // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
 
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -856,10 +848,8 @@ inline void PACurlCurlApply3D(const int d1d,
                               const Vector &x,
                               Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -884,8 +874,8 @@ inline void PACurlCurlApply3D(const int d1d,
       // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
 
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -1379,10 +1369,8 @@ inline void SmemPACurlCurlApply3D(const int d1d,
                                   const Vector &x,
                                   Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -1404,8 +1392,8 @@ inline void SmemPACurlCurlApply3D(const int d1d,
    auto device_kernel = [=] MFEM_DEVICE (int e)
    {
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -1750,10 +1738,8 @@ inline void PAHcurlL2Apply3D(const int d1d,
                              const Vector &x,
                              Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -1778,8 +1764,8 @@ inline void PAHcurlL2Apply3D(const int d1d,
       // If c = 2, \hat{\nabla}\times\hat{u} reduces to [(u_2)_{x_1}, -(u_2)_{x_0}, 0]
 
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -2121,10 +2107,8 @@ inline void SmemPAHcurlL2Apply3D(const int d1d,
                                  const Vector &x,
                                  Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -2139,8 +2123,8 @@ inline void SmemPAHcurlL2Apply3D(const int d1d,
    {
       constexpr int VDIM = 3;
       constexpr int maxCoeffDim = 9;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -2441,10 +2425,8 @@ inline void PAHcurlL2ApplyTranspose3D(const int d1d,
                                       Vector &y)
 {
    // See PAHcurlL2Apply3D for comments.
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -2460,8 +2442,8 @@ inline void PAHcurlL2ApplyTranspose3D(const int d1d,
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
       constexpr int VDIM = 3;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -2809,10 +2791,8 @@ inline void SmemPAHcurlL2ApplyTranspose3D(const int d1d,
                                           const Vector &x,
                                           Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
@@ -2827,8 +2807,8 @@ inline void SmemPAHcurlL2ApplyTranspose3D(const int d1d,
    {
       constexpr int VDIM = 3;
       constexpr int maxCoeffDim = 9;
-      constexpr int MD1D = T_D1D ? T_D1D : DofQuadLimits::HCURL_MAX_D1D;
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+      constexpr int MD1D = T_D1D ? T_D1D : HCURL_MAX_D1D;
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 

fem/integ/bilininteg_hcurlhdiv_kernels.cpp

@@ -224,10 +224,11 @@ void PAHcurlHdivMassApply2D(const int D1D,
                             const Vector &x_,
                             Vector &y_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: Q1D > MAX_Q1D");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
    constexpr static int VDIM = 2;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -243,8 +244,6 @@ void PAHcurlHdivMassApply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -324,7 +323,7 @@ void PAHcurlHdivMassApply2D(const int D1D,
 
          for (int qy = 0; qy < Q1D; ++qy)
          {
-            double massX[DofQuadLimits::HDIV_MAX_D1D];
+            double massX[HDIV_MAX_D1D];
             for (int dx = 0; dx < D1Dx; ++dx)
             {
                massX[dx] = 0.0;
@@ -371,10 +370,11 @@ void PAHcurlHdivMassApply3D(const int D1D,
                             const Vector &x_,
                             Vector &y_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: Q1D > MAX_Q1D");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -395,8 +395,6 @@ void PAHcurlHdivMassApply3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qz = 0; qz < Q1D; ++qz)
@@ -509,7 +507,7 @@ void PAHcurlHdivMassApply3D(const int D1D,
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
-         double massXY[DofQuadLimits::HDIV_MAX_D1D][DofQuadLimits::HDIV_MAX_D1D];
+         double massXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
 
          osc = 0;
          for (int c = 0; c < VDIM; ++c)  // loop over x, y, z test components
@@ -530,7 +528,7 @@ void PAHcurlHdivMassApply3D(const int D1D,
             }
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               double massX[DofQuadLimits::HDIV_MAX_D1D];
+               double massX[HDIV_MAX_D1D];
                for (int dx = 0; dx < D1Dx; ++dx)
                {
                   massX[dx] = 0.0;

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -92,12 +92,10 @@ inline void PAHcurlHdivApply3D(const int d1d,
                                const Vector &x,
                                Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_D1D_TEST || d1dtest <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1dtest > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_D1D_TEST ||
+               d1dtest <= HCURL_MAX_D1D, "Error: d1dtest > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -122,8 +120,8 @@ inline void PAHcurlHdivApply3D(const int d1d,
 
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D :
-                           DofQuadLimits::HCURL_MAX_D1D;  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+                           HCURL_MAX_D1D;  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -461,12 +459,10 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
                                         const Vector &x,
                                         Vector &y)
 {
-   MFEM_VERIFY(T_D1D || d1d <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1d > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_D1D_TEST || d1dtest <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: d1dtest > HCURL_MAX_D1D");
-   MFEM_VERIFY(T_Q1D || q1d <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: q1d > HCURL_MAX_Q1D");
+   MFEM_VERIFY(T_D1D || d1d <= HCURL_MAX_D1D, "Error: d1d > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_D1D_TEST ||
+               d1dtest <= HCURL_MAX_D1D, "Error: d1dtest > HCURL_MAX_D1D");
+   MFEM_VERIFY(T_Q1D || q1d <= HCURL_MAX_Q1D, "Error: q1d > HCURL_MAX_Q1D");
    const int D1D = T_D1D ? T_D1D : d1d;
    const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
@@ -491,8 +487,8 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
 
       constexpr int VDIM = 3;
       constexpr int MD1D = T_D1D ? T_D1D :
-                           DofQuadLimits::HCURL_MAX_D1D;  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
-      constexpr int MQ1D = T_Q1D ? T_Q1D : DofQuadLimits::HCURL_MAX_Q1D;
+                           HCURL_MAX_D1D;  // Assuming HDIV_MAX_D1D <= HCURL_MAX_D1D
+      constexpr int MQ1D = T_Q1D ? T_Q1D : HCURL_MAX_Q1D;
       const int D1D = T_D1D ? T_D1D : d1d;
       const int D1Dtest = T_D1D_TEST ? T_D1D_TEST : d1dtest;
       const int Q1D = T_Q1D ? T_Q1D : q1d;

fem/integ/bilininteg_hdiv_kernels.cpp

@@ -176,6 +176,9 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
+
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
@@ -183,9 +186,6 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_Q1D = DofQuadLimits::HDIV_MAX_Q1D;
-
       int osc = 0;
 
       for (int c = 0; c < VDIM; ++c)  // loop over x, y components
@@ -232,10 +232,8 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const Vector &op_,
                                   Vector &diag_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -256,7 +254,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
          const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
                                          (symmetric ? 5 : 8));
 
-         double mass[DofQuadLimits::HDIV_MAX_Q1D];
+         double mass[HDIV_MAX_Q1D];
 
          for (int dz = 0; dz < D1Dz; ++dz)
          {
@@ -349,6 +347,10 @@ void PAHdivMassApply2D(const int D1D,
                        const Vector &x_,
                        Vector &y_)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
+
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -359,10 +361,6 @@ void PAHdivMassApply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HDIV_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HDIV_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -480,10 +478,8 @@ void PAHdivMassApply3D(const int D1D,
                        const Vector &x_,
                        Vector &y_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -496,7 +492,7 @@ void PAHdivMassApply3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      double mass[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D][VDIM];
+      double mass[HDIV_MAX_Q1D][HDIV_MAX_Q1D][HDIV_MAX_Q1D][VDIM];
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
@@ -522,7 +518,7 @@ void PAHdivMassApply3D(const int D1D,
 
          for (int dz = 0; dz < D1Dz; ++dz)
          {
-            double massXY[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+            double massXY[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
             for (int qy = 0; qy < Q1D; ++qy)
             {
                for (int qx = 0; qx < Q1D; ++qx)
@@ -533,7 +529,7 @@ void PAHdivMassApply3D(const int D1D,
 
             for (int dy = 0; dy < D1Dy; ++dy)
             {
-               double massX[DofQuadLimits::HDIV_MAX_Q1D];
+               double massX[HDIV_MAX_Q1D];
                for (int qx = 0; qx < Q1D; ++qx)
                {
                   massX[qx] = 0.0;
@@ -604,7 +600,7 @@ void PAHdivMassApply3D(const int D1D,
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
-         double massXY[DofQuadLimits::HDIV_MAX_D1D][DofQuadLimits::HDIV_MAX_D1D];
+         double massXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
 
          osc = 0;
 
@@ -623,7 +619,7 @@ void PAHdivMassApply3D(const int D1D,
             }
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               double massX[DofQuadLimits::HDIV_MAX_D1D];
+               double massX[HDIV_MAX_D1D];
                for (int dx = 0; dx < D1Dx; ++dx)
                {
                   massX[dx] = 0;
@@ -734,6 +730,9 @@ void PADivDivAssembleDiagonal2D(const int D1D,
                                 const Vector &op_,
                                 Vector &diag_)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
+
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Gc = Reshape(Gc_.Read(), Q1D, D1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
@@ -741,9 +740,6 @@ void PADivDivAssembleDiagonal2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_Q1D = DofQuadLimits::HDIV_MAX_Q1D;
-
       int osc = 0;
 
       for (int c = 0; c < VDIM; ++c)  // loop over x, y components
@@ -790,10 +786,8 @@ void PADivDivAssembleDiagonal3D(const int D1D,
                                 const Vector &op_,
                                 Vector &diag_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -815,7 +809,7 @@ void PADivDivAssembleDiagonal3D(const int D1D,
          {
             for (int dy = 0; dy < D1Dy; ++dy)
             {
-               double a[DofQuadLimits::HDIV_MAX_Q1D];
+               double a[HDIV_MAX_Q1D];
 
                for (int qx = 0; qx < Q1D; ++qx)
                {
@@ -861,6 +855,10 @@ void PADivDivApply2D(const int D1D,
                      const Vector &x_,
                      Vector &y_)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
+
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
    auto Gc = Reshape(Gc_.Read(), Q1D, D1D);
@@ -871,10 +869,6 @@ void PADivDivApply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HDIV_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HDIV_MAX_Q1D;
-
       double div[MAX_Q1D][MAX_Q1D];
 
       // div[qy][qx] will be computed as du_x/dx + du_y/dy
@@ -980,10 +974,8 @@ void PADivDivApply3D(const int D1D,
                      const Vector &x_,
                      Vector &y_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -996,7 +988,7 @@ void PADivDivApply3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      double div[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+      double div[HDIV_MAX_Q1D][HDIV_MAX_Q1D][HDIV_MAX_Q1D];
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
@@ -1019,7 +1011,7 @@ void PADivDivApply3D(const int D1D,
 
          for (int dz = 0; dz < D1Dz; ++dz)
          {
-            double aXY[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+            double aXY[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
             for (int qy = 0; qy < Q1D; ++qy)
             {
                for (int qx = 0; qx < Q1D; ++qx)
@@ -1030,7 +1022,7 @@ void PADivDivApply3D(const int D1D,
 
             for (int dy = 0; dy < D1Dy; ++dy)
             {
-               double aX[DofQuadLimits::HDIV_MAX_Q1D];
+               double aX[HDIV_MAX_Q1D];
                for (int qx = 0; qx < Q1D; ++qx)
                {
                   aX[qx] = 0.0;
@@ -1086,7 +1078,7 @@ void PADivDivApply3D(const int D1D,
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
-         double aXY[DofQuadLimits::HDIV_MAX_D1D][DofQuadLimits::HDIV_MAX_D1D];
+         double aXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
 
          osc = 0;
 
@@ -1105,7 +1097,7 @@ void PADivDivApply3D(const int D1D,
             }
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               double aX[DofQuadLimits::HDIV_MAX_D1D];
+               double aX[HDIV_MAX_D1D];
                for (int dx = 0; dx < D1Dx; ++dx)
                {
                   aX[dx] = 0;
@@ -1215,8 +1207,8 @@ void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
             // Compute row (rx,ry), assuming all contributions are from
             // a single element.
 
-            double row[2*DofQuadLimits::HDIV_MAX_D1D*(DofQuadLimits::HDIV_MAX_D1D-1)];
-            double div[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+            double row[2*HDIV_MAX_D1D*(HDIV_MAX_D1D-1)];
+            double div[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
 
             for (int i=0; i<2*D1D*(D1D - 1); ++i)
             {
@@ -1239,7 +1231,7 @@ void PAHdivL2AssembleDiagonal_ADAt_2D(const int D1D,
                   const int D1Dy = (c == 1) ? D1D : D1D - 1;
                   const int D1Dx = (c == 0) ? D1D : D1D - 1;
 
-                  double aX[DofQuadLimits::HDIV_MAX_D1D];
+                  double aX[HDIV_MAX_D1D];
                   for (int dx = 0; dx < D1Dx; ++dx)
                   {
                      aX[dx] = 0;
@@ -1289,10 +1281,8 @@ void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
                                       const Vector &D_,
                                       Vector &diag_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto L2Bo = Reshape(L2Bo_.Read(), Q1D, L2D1D);
@@ -1313,9 +1303,8 @@ void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
                // Compute row (rx,ry,rz), assuming all contributions are from
                // a single element.
 
-               double row[3*DofQuadLimits::HDIV_MAX_D1D*(DofQuadLimits::HDIV_MAX_D1D-1)*
-                          (DofQuadLimits::HDIV_MAX_D1D-1)];
-               double div[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+               double row[3*HDIV_MAX_D1D*(HDIV_MAX_D1D-1)*(HDIV_MAX_D1D-1)];
+               double div[HDIV_MAX_Q1D][HDIV_MAX_Q1D][HDIV_MAX_Q1D];
 
                for (int i=0; i<3*D1D*(D1D - 1)*(D1D - 1); ++i)
                {
@@ -1336,7 +1325,7 @@ void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
 
                for (int qz = 0; qz < Q1D; ++qz)
                {
-                  double aXY[DofQuadLimits::HDIV_MAX_D1D][DofQuadLimits::HDIV_MAX_D1D];
+                  double aXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
 
                   int osc = 0;
                   for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
@@ -1354,7 +1343,7 @@ void PAHdivL2AssembleDiagonal_ADAt_3D(const int D1D,
                      }
                      for (int qy = 0; qy < Q1D; ++qy)
                      {
-                        double aX[DofQuadLimits::HDIV_MAX_D1D];
+                        double aX[HDIV_MAX_D1D];
                         for (int dx = 0; dx < D1Dx; ++dx)
                         {
                            aX[dx] = 0;
@@ -1419,6 +1408,10 @@ void PAHdivL2Apply2D(const int D1D,
                      const Vector &x_,
                      Vector &y_)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
+
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Gc = Reshape(Gc_.Read(), Q1D, D1D);
    auto L2Bot = Reshape(L2Bot_.Read(), L2D1D, Q1D);
@@ -1428,10 +1421,6 @@ void PAHdivL2Apply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HDIV_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HDIV_MAX_Q1D;
-
       double div[MAX_Q1D][MAX_Q1D];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -1525,6 +1514,10 @@ void PAHdivL2ApplyTranspose2D(const int D1D,
                               const Vector &x_,
                               Vector &y_)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HDIV_MAX_D1D;
+   constexpr static int MAX_Q1D = HDIV_MAX_Q1D;
+
    auto L2Bo = Reshape(L2Bo_.Read(), Q1D, L2D1D);
    auto Gct = Reshape(Gct_.Read(), D1D, Q1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
@@ -1534,10 +1527,6 @@ void PAHdivL2ApplyTranspose2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HDIV_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HDIV_MAX_Q1D;
-
       double div[MAX_Q1D][MAX_Q1D];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -1633,10 +1622,8 @@ void PAHdivL2Apply3D(const int D1D,
                      const Vector &x_,
                      Vector &y_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
@@ -1648,7 +1635,7 @@ void PAHdivL2Apply3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      double div[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+      double div[HDIV_MAX_Q1D][HDIV_MAX_Q1D][HDIV_MAX_Q1D];
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
@@ -1671,7 +1658,7 @@ void PAHdivL2Apply3D(const int D1D,
 
          for (int dz = 0; dz < D1Dz; ++dz)
          {
-            double aXY[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+            double aXY[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
             for (int qy = 0; qy < Q1D; ++qy)
             {
                for (int qx = 0; qx < Q1D; ++qx)
@@ -1682,7 +1669,7 @@ void PAHdivL2Apply3D(const int D1D,
 
             for (int dy = 0; dy < D1Dy; ++dy)
             {
-               double aX[DofQuadLimits::HDIV_MAX_Q1D];
+               double aX[HDIV_MAX_Q1D];
                for (int qx = 0; qx < Q1D; ++qx)
                {
                   aX[qx] = 0.0;
@@ -1737,7 +1724,7 @@ void PAHdivL2Apply3D(const int D1D,
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
-         double aXY[DofQuadLimits::HDIV_MAX_D1D][DofQuadLimits::HDIV_MAX_D1D];
+         double aXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
 
          for (int dy = 0; dy < L2D1D; ++dy)
          {
@@ -1748,7 +1735,7 @@ void PAHdivL2Apply3D(const int D1D,
          }
          for (int qy = 0; qy < Q1D; ++qy)
          {
-            double aX[DofQuadLimits::HDIV_MAX_D1D];
+            double aX[HDIV_MAX_D1D];
             for (int dx = 0; dx < L2D1D; ++dx)
             {
                aX[dx] = 0;
@@ -1796,10 +1783,8 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
                               const Vector &x_,
                               Vector &y_)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HDIV_MAX_D1D,
-               "Error: D1D > HDIV_MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HDIV_MAX_Q1D,
-               "Error: Q1D > HDIV_MAX_Q1D");
+   MFEM_VERIFY(D1D <= HDIV_MAX_D1D, "Error: D1D > HDIV_MAX_D1D");
+   MFEM_VERIFY(Q1D <= HDIV_MAX_Q1D, "Error: Q1D > HDIV_MAX_Q1D");
    constexpr static int VDIM = 3;
 
    auto L2Bo = Reshape(L2Bo_.Read(), Q1D, L2D1D);
@@ -1811,7 +1796,7 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      double div[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+      double div[HDIV_MAX_Q1D][HDIV_MAX_Q1D][HDIV_MAX_Q1D];
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
@@ -1826,7 +1811,7 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
 
       for (int dz = 0; dz < L2D1D; ++dz)
       {
-         double aXY[DofQuadLimits::HDIV_MAX_Q1D][DofQuadLimits::HDIV_MAX_Q1D];
+         double aXY[HDIV_MAX_Q1D][HDIV_MAX_Q1D];
          for (int qy = 0; qy < Q1D; ++qy)
          {
             for (int qx = 0; qx < Q1D; ++qx)
@@ -1837,7 +1822,7 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
 
          for (int dy = 0; dy < L2D1D; ++dy)
          {
-            double aX[DofQuadLimits::HDIV_MAX_Q1D];
+            double aX[HDIV_MAX_Q1D];
             for (int qx = 0; qx < Q1D; ++qx)
             {
                aX[qx] = 0.0;
@@ -1889,7 +1874,7 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
 
       for (int qz = 0; qz < Q1D; ++qz)
       {
-         double aXY[DofQuadLimits::HDIV_MAX_D1D][DofQuadLimits::HDIV_MAX_D1D];
+         double aXY[HDIV_MAX_D1D][HDIV_MAX_D1D];
 
          int osc = 0;
          for (int c = 0; c < VDIM; ++c)  // loop over x, y, z components
@@ -1907,7 +1892,7 @@ void PAHdivL2ApplyTranspose3D(const int D1D,
             }
             for (int qy = 0; qy < Q1D; ++qy)
             {
-               double aX[DofQuadLimits::HDIV_MAX_D1D];
+               double aX[HDIV_MAX_D1D];
                for (int dx = 0; dx < D1Dx; ++dx)
                {
                   aX[dx] = 0;

fem/integ/bilininteg_hdiv_kernels.hpp

@@ -140,8 +140,8 @@ inline void SmemPAHdivMassApply2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
       constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
       MFEM_SHARED double smo[MQ1*(MD1-1)];
@@ -310,8 +310,8 @@ inline void SmemPAHdivMassApply3D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::HDIV_MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::HDIV_MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
       constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
       MFEM_SHARED double smo[MQ1*(MD1-1)];

fem/integ/bilininteg_interp_pa.cpp

@@ -34,12 +34,11 @@ static void PAHcurlApplyGradient2D(const int c_dofs1D,
    auto x = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
    auto y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
       double w[MAX_D1D][MAX_D1D];
 
       // horizontal part
@@ -111,12 +110,11 @@ static void PAHcurlApplyGradient2DBId(const int c_dofs1D,
    auto x = Reshape(x_.Read(), c_dofs1D, c_dofs1D, NE);
    auto y = Reshape(y_.ReadWrite(), 2 * c_dofs1D * o_dofs1D, NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
       double w[MAX_D1D][MAX_D1D];
 
       // horizontal part
@@ -180,12 +178,11 @@ static void PAHcurlApplyGradientTranspose2D(
    auto x = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
    auto y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
       double w[MAX_D1D][MAX_D1D];
 
       // horizontal part (open x, closed y)
@@ -256,12 +253,11 @@ static void PAHcurlApplyGradientTranspose2DBId(
    auto x = Reshape(x_.Read(), 2 * c_dofs1D * o_dofs1D, NE);
    auto y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
       double w[MAX_D1D][MAX_D1D];
 
       // horizontal part (open x, closed y)
@@ -328,12 +324,11 @@ static void PAHcurlApplyGradient3D(const int c_dofs1D,
    auto x = Reshape(x_.Read(), c_dofs1D, c_dofs1D, c_dofs1D, NE);
    auto y = Reshape(y_.ReadWrite(), (3 * c_dofs1D * c_dofs1D * o_dofs1D), NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
       double w1[MAX_D1D][MAX_D1D][MAX_D1D];
       double w2[MAX_D1D][MAX_D1D][MAX_D1D];
 
@@ -516,13 +511,11 @@ static void PAHcurlApplyGradient3DBId(const int c_dofs1D,
    auto x = Reshape(x_.Read(), c_dofs1D, c_dofs1D, c_dofs1D, NE);
    auto y = Reshape(y_.ReadWrite(), (3 * c_dofs1D * c_dofs1D * o_dofs1D), NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-
       double w1[MAX_D1D][MAX_D1D][MAX_D1D];
       double w2[MAX_D1D][MAX_D1D][MAX_D1D];
 
@@ -685,12 +678,11 @@ static void PAHcurlApplyGradientTranspose3D(
    auto x = Reshape(x_.Read(), (3 * c_dofs1D * c_dofs1D * o_dofs1D), NE);
    auto y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, c_dofs1D, NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
       double w1[MAX_D1D][MAX_D1D][MAX_D1D];
       double w2[MAX_D1D][MAX_D1D][MAX_D1D];
       // ---
@@ -871,13 +863,11 @@ static void PAHcurlApplyGradientTranspose3DBId(
    auto x = Reshape(x_.Read(), (3 * c_dofs1D * c_dofs1D * o_dofs1D), NE);
    auto y = Reshape(y_.ReadWrite(), c_dofs1D, c_dofs1D, c_dofs1D, NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-
       double w1[MAX_D1D][MAX_D1D][MAX_D1D];
       double w2[MAX_D1D][MAX_D1D][MAX_D1D];
       // ---
@@ -1162,13 +1152,12 @@ static void PAHcurlVecH1IdentityApply2D(const int c_dofs1D,
 
    auto vk = Reshape(pa_data.Read(), 2, (2 * c_dofs1D * o_dofs1D), NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-
       double w[2][MAX_D1D][MAX_D1D];
 
       // dofs that point parallel to x-axis (open in x, closed in y)
@@ -1262,13 +1251,13 @@ static void PAHcurlVecH1IdentityApplyTranspose2D(const int c_dofs1D,
 
    auto vk = Reshape(pa_data.Read(), 2, (2 * c_dofs1D * o_dofs1D), NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   //constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-
       double w[2][MAX_D1D][MAX_D1D];
 
       // dofs that point parallel to x-axis (open in x, closed in y)
@@ -1371,13 +1360,12 @@ static void PAHcurlVecH1IdentityApply3D(const int c_dofs1D,
 
    auto vk = Reshape(pa_data.Read(), 3, (3 * c_dofs1D * c_dofs1D * o_dofs1D),
                      NE);
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-
       double w1[3][MAX_D1D][MAX_D1D][MAX_D1D];
       double w2[3][MAX_D1D][MAX_D1D][MAX_D1D];
 
@@ -1586,13 +1574,12 @@ static void PAHcurlVecH1IdentityApplyTranspose3D(const int c_dofs1D,
    auto vk = Reshape(pa_data.Read(), 3, (3 * c_dofs1D * c_dofs1D * o_dofs1D),
                      NE);
 
-   MFEM_VERIFY(c_dofs1D <= DeviceDofQuadLimits::Get().MAX_D1D &&
-               o_dofs1D <= c_dofs1D, "");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+
+   MFEM_VERIFY(c_dofs1D <= MAX_D1D && o_dofs1D <= c_dofs1D, "");
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-
       double w1[3][MAX_D1D][MAX_D1D][MAX_D1D];
       double w2[3][MAX_D1D][MAX_D1D][MAX_D1D];
 

fem/integ/bilininteg_mass_ea.cpp

@@ -27,8 +27,8 @@ static void EAMassAssemble1D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, NE);
    auto M = Reshape(eadata.ReadWrite(), D1D, D1D, NE);
@@ -36,7 +36,7 @@ static void EAMassAssemble1D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_Bi[MQ1];
       double r_Bj[MQ1];
       for (int q = 0; q < Q1D; q++)
@@ -77,8 +77,8 @@ static void EAMassAssemble2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, NE);
    auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, NE);
@@ -86,8 +86,8 @@ static void EAMassAssemble2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double r_B[MQ1][MD1];
       for (int d = 0; d < D1D; d++)
       {
@@ -149,8 +149,8 @@ static void EAMassAssemble3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(basis.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, NE);
    auto M = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
@@ -158,8 +158,8 @@ static void EAMassAssemble3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       constexpr int DQ = T_D1D * T_Q1D;
 
       // For quadratic and lower it's better to use registers but for higher-order you start to

fem/integ/bilininteg_mass_kernels.cpp

@@ -25,6 +25,8 @@ static void PAMassAssembleDiagonal1D(const int NE,
                                      const int D1D,
                                      const int Q1D)
 {
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto D = Reshape(d.Read(), Q1D, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, NE);
@@ -32,6 +34,7 @@ static void PAMassAssembleDiagonal1D(const int NE,
    {
       for (int dx = 0; dx < D1D; ++dx)
       {
+         Y(dx, e) = 0.0;
          for (int qx = 0; qx < Q1D; ++qx)
          {
             Y(dx, e) += B(qx, dx) * B(qx, dx) * D(qx, e);
@@ -195,7 +198,8 @@ void PAMassApply1D_Element(const int e,
    auto X = ConstDeviceMatrix(x_, D1D, NE);
    auto Y = DeviceMatrix(y_, D1D, NE);
 
-   double XQ[DofQuadLimits::MAX_Q1D];
+   constexpr int max_Q1D = MAX_Q1D;
+   double XQ[max_Q1D];
    for (int qx = 0; qx < Q1D; ++qx)
    {
       XQ[qx] = 0.0;
@@ -228,8 +232,8 @@ static void PAMassApply1D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   MFEM_VERIFY(d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(d1d <= MAX_D1D, "");
+   MFEM_VERIFY(q1d <= MAX_Q1D, "");
 
    const auto B = b_.Read();
    const auto Bt = bt_.Read();

fem/integ/bilininteg_mass_kernels.hpp

@@ -42,8 +42,8 @@ inline void PAMassAssembleDiagonal2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto D = Reshape(d.Read(), Q1D, Q1D, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, NE);
@@ -51,8 +51,8 @@ inline void PAMassAssembleDiagonal2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double QD[MQ1][MD1];
       for (int qx = 0; qx < Q1D; ++qx)
       {
@@ -90,10 +90,10 @@ inline void SmemPAMassAssembleDiagonal2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
@@ -103,8 +103,8 @@ inline void SmemPAMassAssembleDiagonal2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
       MFEM_SHARED double B[MQ1][MD1];
       MFEM_SHARED double QDZ[NBZ][MQ1][MD1];
       double (*QD)[MD1] = (double (*)[MD1])(QDZ + tidz);
@@ -156,8 +156,8 @@ inline void PAMassAssembleDiagonal3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto D = Reshape(d.Read(), Q1D, Q1D, Q1D, NE);
    auto Y = Reshape(y.ReadWrite(), D1D, D1D, D1D, NE);
@@ -165,8 +165,8 @@ inline void PAMassAssembleDiagonal3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double QQD[MQ1][MQ1][MD1];
       double QDD[MQ1][MD1][MD1];
       for (int qx = 0; qx < Q1D; ++qx)
@@ -226,10 +226,10 @@ inline void SmemPAMassAssembleDiagonal3D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto b = Reshape(b_.Read(), Q1D, D1D);
    auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
    auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
@@ -238,8 +238,8 @@ inline void SmemPAMassAssembleDiagonal3D(const int NE,
       const int tidz = MFEM_THREAD_ID(z);
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       MFEM_SHARED double B[MQ1][MD1];
       MFEM_SHARED double QQD[MQ1][MQ1][MD1];
       MFEM_SHARED double QDD[MQ1][MD1][MD1];
@@ -365,8 +365,8 @@ void PAMassApply2D_Element(const int e,
       }
    }
 
-   constexpr int max_D1D = DofQuadLimits::MAX_D1D;
-   constexpr int max_Q1D = DofQuadLimits::MAX_Q1D;
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
    double sol_xy[max_Q1D][max_Q1D];
    for (int qy = 0; qy < Q1D; ++qy)
    {
@@ -447,8 +447,8 @@ void SmemPAMassApply2D_Element(const int e,
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
 
-   constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
    constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
    auto b = ConstDeviceMatrix(b_, Q1D, D1D);
@@ -592,8 +592,8 @@ void PAMassApply3D_Element(const int e,
       }
    }
 
-   constexpr int max_D1D = DofQuadLimits::MAX_D1D;
-   constexpr int max_Q1D = DofQuadLimits::MAX_Q1D;
+   constexpr int max_D1D = MAX_D1D;
+   constexpr int max_Q1D = MAX_Q1D;
    double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
    for (int qz = 0; qz < Q1D; ++qz)
    {
@@ -722,8 +722,8 @@ void SmemPAMassApply3D_Element(const int e,
 {
    constexpr int D1D = T_D1D ? T_D1D : d1d;
    constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
    constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
 
    auto b = ConstDeviceMatrix(b_, Q1D, D1D);
@@ -948,8 +948,8 @@ inline void PAMassApply2D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
 
    const auto B = b_.Read();
    const auto Bt = bt_.Read();
@@ -978,10 +978,10 @@ inline void SmemPAMassApply2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    const auto b = b_.Read();
    const auto D = d_.Read();
    const auto x = x_.Read();
@@ -1004,8 +1004,8 @@ inline void PAMassApply3D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
+   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
 
    const auto B = b_.Read();
    const auto Bt = bt_.Read();
@@ -1033,10 +1033,10 @@ inline void SmemPAMassApply3D(const int NE,
    MFEM_CONTRACT_VAR(bt_);
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int M1Q = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= M1D, "");
+   MFEM_VERIFY(Q1D <= M1Q, "");
    auto b = b_.Read();
    auto d = d_.Read();
    auto x = x_.Read();

fem/integ/bilininteg_mass_pa.cpp

@@ -128,7 +128,7 @@ void MassIntegrator::AssemblePABoundary(const FiniteElementSpace &fes)
 
    int map_type = el.GetMapType();
    dim = el.GetDim(); // Dimension of the boundary element, *not* the mesh
-   ne = fes.GetMesh()->GetNFbyType(FaceType::Boundary);
+   ne = fes.GetMesh()->GetNBE();
    nq = ir->GetNPoints();
    face_geom = mesh->GetFaceGeometricFactors(*ir, GeometricFactors::DETERMINANTS,
                                              FaceType::Boundary, mt);

fem/integ/bilininteg_mixedvecgrad_pa.cpp

@@ -31,6 +31,10 @@ static void PAHcurlH1Apply2D(const int D1D,
                              const Vector &x,
                              Vector &y)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Gc = Reshape(gc.Read(), Q1D, D1D);
    auto Bot = Reshape(bot.Read(), D1D-1, Q1D);
@@ -41,10 +45,6 @@ static void PAHcurlH1Apply2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -155,6 +155,10 @@ static void PAHcurlH1ApplyTranspose2D(const int D1D,
                                       const Vector &x,
                                       Vector &y)
 {
+   constexpr static int VDIM = 2;
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
    auto Bc = Reshape(bc.Read(), Q1D, D1D);
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Bt = Reshape(bct.Read(), D1D, Q1D);
@@ -165,10 +169,6 @@ static void PAHcurlH1ApplyTranspose2D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int VDIM = 2;
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qy = 0; qy < Q1D; ++qy)
@@ -280,10 +280,11 @@ static void PAHcurlH1Apply3D(const int D1D,
                              const Vector &x,
                              Vector &y)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: Q1D > MAX_Q1D");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
 
    constexpr static int VDIM = 3;
 
@@ -297,9 +298,6 @@ static void PAHcurlH1Apply3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qz = 0; qz < Q1D; ++qz)
@@ -472,10 +470,11 @@ static void PAHcurlH1ApplyTranspose3D(const int D1D,
                                       const Vector &x,
                                       Vector &y)
 {
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().HCURL_MAX_D1D,
-               "Error: D1D > MAX_D1D");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().HCURL_MAX_Q1D,
-               "Error: Q1D > MAX_Q1D");
+   constexpr static int MAX_D1D = HCURL_MAX_D1D;
+   constexpr static int MAX_Q1D = HCURL_MAX_Q1D;
+
+   MFEM_VERIFY(D1D <= MAX_D1D, "Error: D1D > MAX_D1D");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "Error: Q1D > MAX_Q1D");
 
    constexpr static int VDIM = 3;
 
@@ -489,9 +488,6 @@ static void PAHcurlH1ApplyTranspose3D(const int D1D,
 
    mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
    {
-      constexpr static int MAX_D1D = DofQuadLimits::HCURL_MAX_D1D;
-      constexpr static int MAX_Q1D = DofQuadLimits::HCURL_MAX_Q1D;
-
       double mass[MAX_Q1D][MAX_Q1D][MAX_Q1D][VDIM];
 
       for (int qz = 0; qz < Q1D; ++qz)

fem/integ/bilininteg_vecdiffusion_pa.cpp

@@ -233,8 +233,8 @@ static void PAVectorDiffusionDiagonal2D(const int NE,
 {
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    // note the different shape for D, this is a (symmetric) matrix so we only
@@ -245,8 +245,8 @@ static void PAVectorDiffusionDiagonal2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       // gradphi \cdot Q \gradphi has four terms
       double QD0[MQ1][MD1];
       double QD1[MQ1][MD1];
@@ -301,10 +301,10 @@ static void PAVectorDiffusionDiagonal3D(const int NE,
    constexpr int DIM = 3;
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int max_q1d = T_Q1D ? T_Q1D : DeviceDofQuadLimits::Get().MAX_Q1D;
-   const int max_d1d = T_D1D ? T_D1D : DeviceDofQuadLimits::Get().MAX_D1D;
-   MFEM_VERIFY(D1D <= max_d1d, "");
-   MFEM_VERIFY(Q1D <= max_q1d, "");
+   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+   MFEM_VERIFY(D1D <= MD1, "");
+   MFEM_VERIFY(Q1D <= MQ1, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Q = Reshape(d.Read(), Q1D*Q1D*Q1D, 6, NE);
@@ -313,8 +313,8 @@ static void PAVectorDiffusionDiagonal3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MD1 = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
       double QQD[MQ1][MQ1][MD1];
       double QDD[MQ1][MD1][MD1];
       for (int i = 0; i < DIM; ++i)
@@ -442,8 +442,8 @@ void PAVectorDiffusionApply2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -456,8 +456,8 @@ void PAVectorDiffusionApply2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double grad[max_Q1D][max_Q1D][2];
       for (int c = 0; c < VDIM; c++)
@@ -563,8 +563,8 @@ void PAVectorDiffusionApply3D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto Bt = Reshape(bt.Read(), D1D, Q1D);
@@ -576,8 +576,8 @@ void PAVectorDiffusionApply3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       for (int c = 0; c < VDIM; ++ c)
       {
          double grad[max_Q1D][max_Q1D][max_Q1D][3];

fem/integ/bilininteg_vecdiv_pa.cpp

@@ -170,9 +170,9 @@ static void PADivergenceApply2D(const int NE,
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, TR_D1D);
    auto G = Reshape(g.Read(), Q1D, TR_D1D);
    auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
@@ -186,8 +186,8 @@ static void PADivergenceApply2D(const int NE,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = 2;
       // the following variables are evaluated at compile time
-      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double grad[max_Q1D][max_Q1D][VDIM];
       double div[max_Q1D][max_Q1D];
@@ -308,9 +308,9 @@ static void PADivergenceApplyTranspose2D(const int NE,
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto Bt = Reshape(bt.Read(), TR_D1D, Q1D);
    auto Gt = Reshape(gt.Read(), TR_D1D, Q1D);
    auto B  = Reshape(b.Read(), Q1D, TE_D1D);
@@ -324,8 +324,8 @@ static void PADivergenceApplyTranspose2D(const int NE,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = 2;
       // the following variables are evaluated at compile time
-      constexpr int max_TR_D1D = T_TR_D1D ? T_TR_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_TR_D1D = T_TR_D1D ? T_TR_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double quadTest[max_Q1D][max_Q1D];
       double grad[max_Q1D][max_Q1D][VDIM];
@@ -424,9 +424,9 @@ static void PADivergenceApply3D(const int NE,
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(b.Read(), Q1D, TR_D1D);
    auto G = Reshape(g.Read(), Q1D, TR_D1D);
    auto Bt = Reshape(bt.Read(), TE_D1D, Q1D);
@@ -440,8 +440,8 @@ static void PADivergenceApply3D(const int NE,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = 3;
       // the following variables are evaluated at compile time
-      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_TE_D1D = T_TE_D1D ? T_TE_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double grad[max_Q1D][max_Q1D][max_Q1D][VDIM];
       double div[max_Q1D][max_Q1D][max_Q1D];
@@ -607,9 +607,9 @@ static void PADivergenceApplyTranspose3D(const int NE,
    const int TR_D1D = T_TR_D1D ? T_TR_D1D : tr_d1d;
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto Bt = Reshape(bt.Read(), TR_D1D, Q1D);
    auto Gt = Reshape(gt.Read(), TR_D1D, Q1D);
    auto B  = Reshape(b.Read(), Q1D, TE_D1D);
@@ -623,8 +623,8 @@ static void PADivergenceApplyTranspose3D(const int NE,
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       const int VDIM = 3;
       // the following variables are evaluated at compile time
-      constexpr int max_TR_D1D = T_TR_D1D ? T_TR_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_TR_D1D = T_TR_D1D ? T_TR_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double quadTest[max_Q1D][max_Q1D][max_Q1D];
       double grad[max_Q1D][max_Q1D][max_Q1D][VDIM];
@@ -786,9 +786,9 @@ static void SmemPADivergenceApply3D(const int NE,
    const int TE_D1D = T_TE_D1D ? T_TE_D1D : te_d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
 
-   MFEM_VERIFY(TR_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(TE_D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(TR_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(TE_D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
 
    auto b = Reshape(b_.Read(), Q1D, TR_D1D);
    auto g = Reshape(g_.Read(), Q1D, TR_D1D);
@@ -804,9 +804,9 @@ static void SmemPADivergenceApply3D(const int NE,
       const int D1DR = T_TR_D1D ? T_TR_D1D : tr_d1d;
       const int D1DE = T_TE_D1D ? T_TE_D1D : te_d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int MQ1 = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int MD1R = T_TR_D1D ? T_TR_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MD1E = T_TE_D1D ? T_TE_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1R = T_TR_D1D ? T_TR_D1D : MAX_D1D;
+      constexpr int MD1E = T_TE_D1D ? T_TE_D1D : MAX_D1D;
       constexpr int MD1 = MD1E > MD1R ? MD1E : MD1R;
       constexpr int MDQ = MQ1 > MD1 ? MQ1 : MD1;
       MFEM_SHARED double sBG[2][MQ1*MD1];

fem/integ/bilininteg_vecmass_pa.cpp

@@ -118,8 +118,8 @@ static void PAVectorMassAssembleDiagonal2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int VDIM = 2;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(B_.Read(), Q1D, D1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
    auto y = Reshape(diag_.ReadWrite(), D1D, D1D, VDIM, NE);
@@ -127,8 +127,8 @@ static void PAVectorMassAssembleDiagonal2D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double temp[max_Q1D][max_D1D];
       for (int qx = 0; qx < Q1D; ++qx)
@@ -170,8 +170,8 @@ static void PAVectorMassAssembleDiagonal3D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(B_.Read(), Q1D, D1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE);
    auto y = Reshape(diag_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
@@ -180,8 +180,8 @@ static void PAVectorMassAssembleDiagonal3D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
 
       double temp[max_Q1D][max_Q1D][max_D1D];
       for (int qx = 0; qx < Q1D; ++qx)
@@ -281,8 +281,8 @@ static void PAVectorMassApply2D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int VDIM = 2;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(B_.Read(), Q1D, D1D);
    auto Bt = Reshape(Bt_.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, NE);
@@ -293,8 +293,8 @@ static void PAVectorMassApply2D(const int NE,
       const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
       const int Q1D = T_Q1D ? T_Q1D : q1d;
       // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double sol_xy[max_Q1D][max_Q1D];
       for (int c = 0; c < VDIM; ++c)
       {
@@ -377,8 +377,8 @@ static void PAVectorMassApply3D(const int NE,
    const int D1D = T_D1D ? T_D1D : d1d;
    const int Q1D = T_Q1D ? T_Q1D : q1d;
    constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
    auto B = Reshape(B_.Read(), Q1D, D1D);
    auto Bt = Reshape(Bt_.Read(), D1D, Q1D);
    auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, NE);
@@ -388,8 +388,8 @@ static void PAVectorMassApply3D(const int NE,
    {
       const int D1D = T_D1D ? T_D1D : d1d;
       const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
       double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
       for (int c = 0; c < VDIM; ++ c)
       {

fem/integ/lininteg_boundary.cpp

@@ -38,7 +38,7 @@ static void BLFEvalAssemble2D(const int vdim, const int nbe, const int d,
    {
       if (M(e) == 0) { return; } // ignore
 
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
       double QQ[Q];
 
       for (int c = 0; c < vdim; ++c)
@@ -92,8 +92,8 @@ static void BLFEvalAssemble3D(const int vdim, const int nbe, const int d,
    {
       if (M(e) == 0) { return; } // ignore
 
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : MAX_D1D;
 
       MFEM_SHARED double sBt[Q*D];
       MFEM_SHARED double sQQ[Q*Q];

fem/integ/lininteg_boundary_flux.cpp

@@ -33,7 +33,7 @@ void BFLFEvalAssemble2D(const int nbe, const int d, const int q,
    {
       if (M(e) == 0) { return; } // ignore (in a lambda return acts as continue)
 
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
       double QQ[Q];
 
       for (int qx = 0; qx < q; ++qx)
@@ -67,8 +67,8 @@ void BFLFEvalAssemble3D(const int nbe, const int d, const int q,
    {
       if (M(e) == 0) { return; } // ignore
 
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int Q = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : MAX_D1D;
 
       MFEM_SHARED double sBt[Q*D];
       MFEM_SHARED double sQQ[Q*Q];