Merge remote-tracking branch 'upstream/feature/partial_ess_bcs' into exaconstit-csr

Removed the more or less duplicated FESpace::GetEssentialVDofs and moved the necessary book-keeping portions of things to GetEssentialTrueDofs
Additionally, I added an overwrite optional variable (defaulted to true) to GetEssentialVDofs in-order to allow one to overwrite the values of ess_vdofs if they'd like.
I still need to add some unit tests for the functionality though.

.appveyor.yml

@@ -23,8 +23,9 @@ install:
 - set MSMPI_LIB64=C:\Program Files (x86)\Microsoft SDKs\MPI\Lib\x64
 - set MSMPI_INC=C:\Program Files (x86)\Microsoft SDKs\MPI\Include
 
-# Install METIS, use MFEM's mirror because the original source server is often
-# down and we don't support yet the new repo https://github.com/KarypisLab/METIS
+# Install METIS, use a mirror because the original source server is not always
+# up. Original url:
+#    http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz
 - ps: Start-FileDownload 'https://mfem.github.io/tpls/metis-5.1.0.tar.gz'
 - 7z x metis-5.1.0.tar.gz -so | 7z x -si -ttar > nul
 - cd metis-5.1.0

.github/workflows/build-container.yaml

@@ -1,4 +1,4 @@
-name: "Docker"
+name: Build Deploy Container
 
 on:
 
@@ -20,7 +20,6 @@ on:
 
 jobs:
   build:
-    if: github.repository == 'mfem/mfem' # Don't run in forks
     permissions:
       packages: write
     strategy:
@@ -28,8 +27,7 @@ jobs:
       matrix:
 
         # Dockerfiles to build, a matrix supports future expanded builds
-        container: [["config/docker/Dockerfile.base", "ghcr.io/mfem/mfem-ubuntu-base"],
-                    ["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu"]]
+        container: [["config/docker/Dockerfile", "ghcr.io/mfem/mfem-ubuntu-base"]]
 
     runs-on: ubuntu-latest
     name: Build
@@ -62,7 +60,7 @@ jobs:
 
       - name: GHCR Login
         if: (github.event_name != 'pull_request')
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/builds-and-tests.yml

@@ -10,7 +10,7 @@
 # CONTRIBUTING.md for details.
 
 # In this CI section, we build different variants of mfem and run test on them.
-name: "Tests"
+name: builds-and-tests
 
 # Github actions can use the default "GITHUB_TOKEN". By default, this token
 # is set to have permissive access. However, this is not a good practice
@@ -47,17 +47,17 @@ jobs:
   builds-and-tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-20.04, macos-10.15, windows-2022]
         target: [dbg, opt]
         mpi: [seq, par]
         build-system: [make, cmake]
         hypre-target: [int32]
         exclude:
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
             build-system: cmake
-          - os: macos-latest
+          - os: macos-10.15
             build-system: cmake
-          - os: windows-latest
+          - os: windows-2022
             build-system: make
         # 'include' allows us to:
         # - Add a variable to all jobs without creating a new matrix dimension.
@@ -72,15 +72,15 @@ jobs:
             codecov: NO
           - target: opt
             codecov: YES
-          - os: windows-latest
+          - os: windows-2022
             codecov: NO
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
             target: opt
             codecov: NO
             mpi: par
             build-system: cmake
             hypre-target: int32
-          - os: ubuntu-latest
+          - os: ubuntu-20.04
             target: opt
             codecov: NO
             mpi: par
@@ -94,7 +94,7 @@ jobs:
     # This external action allows to interrupt a workflow already running on
     # the same branch to save resource
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
       with:
         access_token: ${{ github.token }}
 
@@ -102,7 +102,7 @@ jobs:
     # /home/runner/work/mfem/mfem/mfem
     # Note: Done now to access "install-hypre" and "install-metis" actions.
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         path: ${{ env.MFEM_TOP_DIR }}
         # Fetch the complete history for codecov to access commits ID
@@ -112,49 +112,49 @@ jobs:
     # TODO: It would be nice to have only one step, e.g. with a dedicated
     # action, but I (@adrienbernede) don't see how at the moment.
     - name: get MPI (Linux)
-      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
     - name: get lcov (Linux)
-      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-latest'
+      if: matrix.codecov == 'YES' && matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get install lcov
 
-    # Keep the following section in case we need it again in the future,
-    # see: https://github.com/mfem/mfem/pull/3385#discussion_r1058013032
-    # - name: Set up Homebrew
-    #   if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-latest'
-    #   uses: Homebrew/actions/setup-homebrew@master
+    - name: Set up Homebrew
+      if: ( matrix.mpi == 'par' || matrix.codecov == 'YES' ) && matrix.os == 'macos-10.15'
+      uses: Homebrew/actions/setup-homebrew@c4aafe8c4620bf08883dd4679c374f11e73329d3
 
     - name: get MPI (MacOS)
-      if: matrix.mpi == 'par' && matrix.os == 'macos-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'macos-10.15'
       run: |
         export HOMEBREW_NO_INSTALL_CLEANUP=1
         brew install openmpi
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
-    - name: get lcov (MacOS)
-      if: matrix.codecov == 'YES' && matrix.os == 'macos-latest'
+    - name: get MPI (MacOS)
+      if: matrix.codecov == 'YES' && matrix.os == 'macos-10.15'
       run: |
         export HOMEBREW_NO_INSTALL_CLEANUP=1
         brew install lcov
 
     - name: get MPI (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
-      uses: mpi4py/setup-mpi@v1.1.2
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
+      uses: mpi4py/setup-mpi@v1.0.3
 
     # Get Hypre through cache, or build it.
     # Install will only run on cache miss.
     - name: cache hypre
       id: hypre-cache
       if: matrix.mpi == 'par'
-      uses: actions/cache@v3
+      uses: actions/cache@v2
       with:
         path: ${{ env.HYPRE_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-v2.2
 
     - name: get hypre
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-2022'
       uses: mfem/github-actions/build-hypre@v2.2
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
@@ -163,7 +163,7 @@ jobs:
         build-system: make
 
     - name: get hypre (Windows)
-      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-2022'
       uses: mfem/github-actions/build-hypre@v2.2
       with:
         archive: ${{ env.HYPRE_ARCHIVE }}
@@ -175,14 +175,14 @@ jobs:
     # Install will only run on cache miss.
     - name: cache metis
       id: metis-cache
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
-      uses: actions/cache@v3
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022'
+      uses: actions/cache@v2
       with:
         path: ${{ env.METIS_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2
 
     - name: install metis
-      if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
+      if: matrix.mpi == 'par' && matrix.os != 'windows-2022' && steps.metis-cache.outputs.cache-hit != 'true'
       uses: mfem/github-actions/build-metis@v2.2
       with:
         archive: ${{ env.METIS_ARCHIVE }}
@@ -190,23 +190,22 @@ jobs:
 
     - name: cache vcpkg (Windows)
       id: vcpkg-cache
-      if: matrix.os == 'windows-latest'
       uses: actions/cache@v3
       with:
         path: vcpkg_cache
         key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
 
-    - name: prepare vcpkg binary cache location (Windows)
-      if: matrix.os == 'windows-latest' && steps.vcpkg-cache.outputs.cache-hit != 'true'
+    - name: prepare binary cache location
+      if: matrix.os == 'windows-2022' && steps.vcpkg-cache.outputs.cache-hit != 'true'
       run: |
         mkdir -p vcpkg_cache
 
     - name: install metis (Windows)
-      if: matrix.mpi == 'par' && matrix.os == 'windows-latest'
+      if: matrix.mpi == 'par' && matrix.os == 'windows-2022'
       env:
         VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
       run: |
-        vcpkg install metis-mfem --triplet=x64-windows-static --overlay-ports=${{ env.MFEM_TOP_DIR }}/config/vcpkg/ports
+        vcpkg install metis --triplet=x64-windows-static
 
     # MFEM build and test
     - name: build
@@ -231,7 +230,11 @@ jobs:
       run: |
         cd ${{ env.MFEM_TOP_DIR }}  && make check
 
-    # Note: 'tests' include the unit tests
+    - name: unit tests
+      if: matrix.build-system == 'make' && matrix.target == 'opt'
+      run: |
+        cd ${{ env.MFEM_TOP_DIR }} && make unittest
+
     - name: tests
       if: matrix.build-system == 'make' && matrix.target == 'opt'
       run: |
@@ -244,8 +247,8 @@ jobs:
         cd ${{ env.MFEM_TOP_DIR }} && cmake --build build --target check --config ${CTEST_CONFIG}
       shell: bash
 
-    - name: cmake unit tests (Ubuntu)
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-latest'
+    - name: cmake unit tests (Ubuntu 20.04)
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os == 'ubuntu-20.04'
       run: |
         CTEST_CONFIG="Release"
         [[ ${{ matrix.target }} == 'dbg' ]] && CTEST_CONFIG="Debug"
@@ -253,7 +256,7 @@ jobs:
       shell: bash
 
     - name: cmake tests
-      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-latest'
+      if: matrix.build-system == 'cmake' && matrix.target == 'opt' && matrix.os != 'ubuntu-20.04'
       run: |
         CTEST_CONFIG="Release"
         cd ${{ env.MFEM_TOP_DIR }}/build && ctest --output-on-failure -C ${CTEST_CONFIG}
@@ -262,7 +265,7 @@ jobs:
     # Code coverage (process and upload reports)
     - name: codecov
       if: matrix.codecov == 'YES'
-      uses: mfem/github-actions/upload-coverage@v2.2
+      uses: mfem/github-actions/upload-coverage@v2.0
       with:
         name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
         project_dir: ${{ env.MFEM_TOP_DIR }}

.github/workflows/codeql-analysis.yml

@@ -1,70 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "Static Analysis"
-
-on:
-  push:
-    branches: [ "master", "next"]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "master" ]
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines.
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

.github/workflows/mfem-analysis.yml

@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-name: "Build Analysis"
+name: build-analysis
 
 permissions:
   actions: write
@@ -31,26 +31,27 @@ env:
 
 jobs:
   gitignore:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
 
     steps:
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
       with:
         access_token: ${{ github.token }}
 
     - name: checkout MFEM
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         path: mfem
 
     - name: Get MPI (Linux)
       run: |
         sudo apt-get install mpich libmpich-dev
+        export MAKE_CXX_FLAG="MPICXX=mpic++"
 
     - name: Cache Hypre Install
       id: hypre-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v2
       with:
         path: ${{ env.HYPRE_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.HYPRE_TOP_DIR }}-v2.2
@@ -65,7 +66,7 @@ jobs:
 
     - name: Cache Metis Install
       id: metis-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v2
       with:
         path: ${{ env.METIS_TOP_DIR }}
         key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.2

.github/workflows/repo-check.yml

@@ -9,7 +9,7 @@
 # terms of the BSD-3 license. We welcome feedback and contributions, see file
 # CONTRIBUTING.md for details.
 
-name: "Checks"
+name: repo-check
 
 permissions:
   actions: write
@@ -28,18 +28,18 @@ on:
 
 jobs:
   file-headers-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     if: |
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.11.0
+      uses: styfle/cancel-workflow-action@0.9.0
       with:
         access_token: ${{ github.token }}
 
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
     - name: copyright check
       id: copyright
@@ -78,40 +78,35 @@ jobs:
         exit 1
 
   code-style:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     if: |
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
     - name: get astyle
       run: |
-        sudo apt-get install astyle
+        sudo apt-get install astyle=3.1-1ubuntu2
 
     - name: style check
       run: |
         ./config/githooks/pre-push --style
 
   documentation:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     if: |
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
     steps:
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
 
     - name: get doxygen and graphviz
       run: |
         sudo apt-get install doxygen graphviz
 
-    - name: update doxygen config file
-      run: |
-        cd doc
-        doxygen -u CodeDocumentation.conf.in
-
     - name: build documentation
       run: |
         cd tests/scripts
@@ -123,10 +118,10 @@ jobs:
       github.ref != 'refs/heads/master' &&
       (github.event_name == 'push' ||
        github.event.pull_request.head.repo.full_name != github.repository)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
     - name: checkout mfem
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         fetch-depth: 0
 

.gitignore

@@ -275,7 +275,6 @@ miniapps/tools/load-dc
 miniapps/tools/convert-dc
 miniapps/tools/lor-transfer
 miniapps/tools/get-values
-miniapps/tools/check-tmop-metric
 
 miniapps/toys/automata
 miniapps/toys/life
@@ -308,9 +307,6 @@ miniapps/solvers/sol.*
 miniapps/parelag/MultilevelHcurlHdivSolver
 miniapps/parelag/*.mesh
 
-miniapps/multidomain/multidomain
-miniapps/hooke/hooke
-
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

CHANGELOG

@@ -8,93 +8,10 @@
                                https://mfem.org
 
 
-
-
-Version 4.5.1 (development)
+Version 4.4.1 (development)
 ===========================
-
-Meshing improvements
---------------------
-- Added support for pyramids in non-conforming meshes. Currently only isotropic
-  refinement is supported in this case.
-
-Discretization improvements
----------------------------
-- TBD
-
-Linear and nonlinear solvers
-----------------------------
-- Added a fast normalization-based distance solver, see the Distance miniapp
-  in the miniapps/shifted/ directory.
-
-New and updated examples and miniapps
--------------------------------------
-- TBD
-
-Integrations, testing and documentation
----------------------------------------
-- Removed the support for the Mesquite toolkit. We recommend using MFEM's TMOP
-  functionality instead for mesh optimization. See the mesh-optimizer miniapp.
-
-Miscellaneous
--------------
-- VisItDataCollection now correctly handles collection names containing
-  underscores.
-
-- VisItDataCollection::SetPadDigits() no longer alters the number of digits
-  used to represent the MPI rank because VisIt seems to require 6 digits.
-  This parameter can still be explicitly overridden with
-  VisItDataCollection::SetPadDigitsRank().
-
-API changes
------------
-- The implicit cast methods of class Vector to 'double *' and 'const double *'
-  have been deprecated and generate deprecation warnings if used. They will be
-  removed in a future release.
-
-
-Version 4.5, released on October 22, 2022
-=========================================
-
-Meshing improvements
---------------------
-- Added new SubMesh and ParSubMesh classes that can be used to extract a subset
-  of a given Mesh. These classes have the same functionality as Mesh and ParMesh
-  and work with all existing MFEM interfaces like finite element spaces etc.
-
-- Added a method, ParMesh::GetSerialMesh(), that reconstructs a partitioned
-  parallel mesh on a given single rank. Also, added ParMesh::PrintAsSerial(),
-  which saves the reconstructed serial mesh to a C++ stream on rank 0.
-
-- Added more 3D TMOP metrics, as well as specialized metrics for mesh
-  untangling and worst-case quality improvement.
-
-- Added a new method, Mesh::NodesUpdated, which should be called after the mesh
-  node coordinates have changed, e.g. after the mesh has moved. This is
-  necessary, for example, with device assembly of linear and bilinear forms.
-
-- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
-
-Discretization improvements
----------------------------
-- Added full assembly and device support for several LinearForm integrators:
-  * DomainLF: (f, v)
-  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
-  * DomainLFGrad: (f, grad(v))
-  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
-  The device assembly of linear forms has to be explicitly enabled by calling
-  LinearForm::UseFastAssembly(true), otherwise the legacy linear form assembly
-  is used by default.
-
-- Added support for assembling low-order-refined matrices using a GPU-enabled
-  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
-  acceleration with arbitrary user-supplied coefficients.
-
-- Added a new class FaceQuadratureSpace that allows for the construction of
-  QuadratureFunctions on the interior or boundary faces of a mesh.
-
-- Added a class CoefficientVector for efficient access of variable coefficient
-  values at quadrature points (in particular for GPU/device kernels).
+- Added example for body-fitted volumetric and shape integration using the 
+  Algoim library.
 
 - Added WhiteGaussianNoiseDomainLFIntegrator: a LinearFormIntegrator class for
   spatial Gaussian white noise.
@@ -102,97 +19,30 @@ Discretization improvements
 - Added a new Zienkiewicz-Zhu patch recovery-based a posteriori error estimator.
   See fem/estimators.hpp.
 
-- Various fixes and improvements in LinearFormExtension.
-
-Linear and nonlinear solvers
-----------------------------
-- Added a new class DGMassInverse that performs a local element-wise CG
-  iteration to solve systems involving the discontinuous Galerkin mass matrix,
-  including support for device/GPU acceleration.
-
-- Added more flexibility to the constrained solver classes:
-  * PenaltyConstrainedSolver now allows for a vector of penalty parameters
-    (necessary for penalty contact)
-  * PenaltyConstrainedSolver and EliminationSolver can use GMRES or PCG
-  * All constraint solver classes can take a user-defined preconditioner
-
- - Added functions to toggle additional options for the SuperLU_Dist and Hypre
-   preconditioners (ParaSails, Euclid, ILU).
-
-- Added boundary elimination with device support for `SparseMatrix` and
-  `HypreParMatrix`.
-
-New and updated examples and miniapps
--------------------------------------
-- Added a new elasticity miniapp, Hooke, that showcases a low-level approach of
-  using MFEM to solve a nonlinear elasticity problem based on the fundamental
-  finite element operator decomposition. The miniapp also integrates with
-  automatic differentiation tools like a native dual number implementation or a
-  third party library such as Enzyme. See miniapps/elasticity for more details.
-
-- Added example for body-fitted volumetric and shape integration using the
-  Algoim library in miniapps/shifted.
-
-- Add a new example code, Example 33/33p, to demonstrate the solution of
-  spectral fractional PDEs with MFEM.
-
-Integrations, testing and documentation
----------------------------------------
-- Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
-  More sophisticated developer containers are available in the new repo
-  https://github.com/mfem/containers.
-
-- Added support for the LLVM-based automatic differentiation tool Enzyme, see
-  https://github.com/EnzymeAD/Enzyme. Build system flags and a convenience
-  header are provided. The functionality and interaction are demonstrated in a
-  new miniapp in miniapps/elasticity.
-
-- Added support for partial assembly and fully matrix-free operators on mixed
-  meshes (different element types and p-adaptivity) through libCEED, including
-  device acceleration, e.g. with NVIDIA and AMD GPUs. The p-adaptivity is
-  currently limited to 2D serial meshes. All mixed element topologies are
-  supported in both serial and parallel.
-
 - Added support for ParMoonolith, https://bitbucket.org/zulianp/par_moonolith,
   which provides parallel non-conforming, non-matching, variational, volumetric
   mesh information transfer. With ParMortarAssember, fields can be exchanged
   between arbitrarily distributed and unrelated finite element meshes in a
   variationally consistent way.
 
-- Fully encapsulated SUNDIALS `N_Vector` object within the `SundialsNVector`
-  class by removing deprecated (e.g. `HypreParVector::ToNVector`) and
-  non-deprecated (e.g. `Vector::ToNVector`) functions in other classes.
+- Added full assembly and device support for several LinearForm integrators:
+  * DomainLF: (f, v)
+  * VectorDomainLF: ((f1,...,fn), (v1,...,vn))
+  * DomainLFGrad: (f, grad(v))
+  * VectorDomainLFGrad: ((f1x,f1y,f1z,...,fnx,fny,fnz), grad(v1,...,vn))
 
-- New benchmark for the different assembly levels inspired by the CEED
-  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+- Add a new example code, Example 33/33p, to demonstrate the solution of
+  spectral fractional PDEs with MFEM.
+
+- Added a Dockerfile for a simple MFEM container, see config/docker/README.md.
+
+- Added support for assembling low-order-refined matrices using a GPU-enabled
+  "batched" algorithm. The lor_solvers and plor_solvers now fully support GPU
+  acceleration.
 
 - Added Windows 2022 CI testing with GitHub actions.
 
-Miscellaneous
--------------
-- The method SparseMatrix::EnsureMultTranspose() is now automatically called
-  by the methods AddMultTranspose(), MultTranspose(), and AbsMultTranspose().
-  Added a method with the same name to class HypreParMatrix which is also called
-  automatically by the HypreParMatrix::MultTranspose() methods.
-
-- Updated various MemoryUsage methods to return 'std::size_t' instead of 'long'
-  since the latter is 32-bit in Win64 builds.
-
-- When using `AssemblyLevel::FULL`, `FABilinearFormExtension::FormSystemMatrix`
-  outputs an `OperatorHandle` containing a `SparseMatrix` in serial, and an
-  `HypreParMatrix` in parallel (instead of a `ConstrainedOperator`).
-
-- In various places in the library, replace the use of 'long' with 'long long'
-  to better support Win64 builds where 'long' is 32-bit and 'long long' is
-  64-bit. On Linux and MacOS, both types are typically 64-bit.
-
-- The behavior of GridFunction::GetTrueVector() has been changed to not return
-  an empty true vector.
-
-- Added support for ordering search points byVDIM in FindPointsGSLIB.
-
-- Various other simplifications, extensions, and bugfixes in the code.
-
+- Added support for mixed meshes and pyramids in GSLIB-FindPoints.
 
 Version 4.4, released on March 21, 2022
 =======================================
@@ -314,6 +164,9 @@ Integrations, testing and documentation
 - Switched from Artistic Style (astyle) version 2.05.1 to version 3.1 for code
   formatting. See the "make style" target.
 
+- New benchmark for the different assembly levels inspired by the CEED
+  Bake-Off Problems, see tests/benchmarks/bench_assembly_levels.cpp.
+
 Miscellaneous
 -------------
 - Added a simple singleton class, Mpi, as a replacement for MPI_Session. New

CMakeLists.txt

@@ -10,9 +10,7 @@
 # CONTRIBUTING.md for details.
 
 # The variable CMAKE_CXX_STANDARD and related were introduced in CMake v3.1
-# Version 3.8 fixes the handling of CMAKE_CXX_STANDARD for try_compile.
-# Version 3.8 or newer is required for direct CUDA support.
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.1)
 set(USER_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/config/user.cmake" CACHE PATH
   "Path to optional user configuration file.")
 
@@ -53,7 +51,7 @@ project(mfem NONE)
 # Current version of MFEM, see also `makefile`.
 #   mfem_VERSION = (string)
 #   MFEM_VERSION = (int)   [automatically derived from mfem_VERSION]
-set(${PROJECT_NAME}_VERSION 4.5.1)
+set(${PROJECT_NAME}_VERSION 4.4.1)
 
 # Prohibit in-source build
 if (${PROJECT_SOURCE_DIR} STREQUAL ${PROJECT_BINARY_DIR})
@@ -68,7 +66,8 @@ set(XSDK_ENABLE_C OFF)
 set(XSDK_ENABLE_Fortran OFF)
 
 # Check if we need to enable C or Fortran.
-if (MFEM_USE_CONDUIT OR
+if (CMAKE_VERSION VERSION_LESS 3.2 OR
+    MFEM_USE_CONDUIT OR
     MFEM_USE_SIDRE OR
     MFEM_USE_PETSC)
    # This seems to be needed by:
@@ -82,9 +81,7 @@ if (MFEM_USE_STRUMPACK)
   # Just needed to find the MPI_Fortran libraries to link with
   set(XSDK_ENABLE_Fortran ON)
 endif()
-# SUNDIALS, STRUMPACK, and Ginkgo require C++14:
-if ((MFEM_USE_SUNDIALS OR MFEM_USE_STRUMPACK OR MFEM_USE_GINKGO) AND
-    ("${CMAKE_CXX_STANDARD}" LESS "14"))
+if (MFEM_USE_GINKGO AND ("${CMAKE_CXX_STANDARD}" LESS "14"))
   set(CMAKE_CXX_STANDARD 14)
 endif()
 
@@ -102,6 +99,8 @@ if (MFEM_USE_CUDA)
    if (MFEM_USE_HIP)
       message(FATAL_ERROR " *** MFEM_USE_HIP cannot be combined with MFEM_USE_CUDA.")
    endif()
+   # MFEM_USE_CUDA requires CMake 3.8 or newer (for direct CUDA support)
+   cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
    # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler.
    if (NOT CMAKE_CUDA_HOST_COMPILER)
       set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
@@ -137,8 +136,6 @@ if (MFEM_USE_CUDA)
        "CUDA flags set for MFEM" FORCE)
    set(CUSPARSE_FOUND TRUE)
    set(CUSPARSE_LIBRARIES "cusparse")
-   set(CUBLAS_FOUND TRUE)
-   set(CUBLAS_LIBRARIES "cublas")
 endif()
 
 if (XSDK_ENABLE_C)
@@ -196,35 +193,15 @@ if (MFEM_USE_HIP)
   find_package(HIPSPARSE REQUIRED)
 endif()
 
-# OpenMP
-if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
-  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
-    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
-  endif()
-  find_package(OpenMP REQUIRED)
-  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
-  if(APPLE)
-    # On macOS, the compiler needs additional help to find the <omp.h> header.
-    # See issue #2642 for more information.
-    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
-  endif(APPLE)
-  if (OPENMP_FOUND)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    if (MFEM_USE_CUDA)
-      set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
-    endif()
-  endif()
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
   set(MPI_CXX_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH})
   if (MFEM_MPIEXEC)
-    string(REPLACE " " ";" MPIEXEC ${MFEM_MPIEXEC})
+    set(MPIEXEC ${MFEM_MPIEXEC})
   endif()
   if (MFEM_MPIEXEC_NP)
-    string(REPLACE " " ";" MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
+    set(MPIEXEC_NUMPROC_FLAG ${MFEM_MPIEXEC_NP})
   endif()
   # Parallel MFEM depends on hypre
   find_package(HYPRE REQUIRED)
@@ -281,6 +258,20 @@ if (MFEM_USE_LAPACK)
   find_package(LAPACK REQUIRED)
 endif()
 
+# OpenMP
+if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
+  if (NOT MFEM_THREAD_SAFE AND MFEM_USE_LEGACY_OPENMP)
+    message(FATAL_ERROR " *** MFEM_USE_LEGACY_OPENMP requires MFEM_THREAD_SAFE=ON.")
+  endif()
+  find_package(OpenMP REQUIRED)
+  set(OPENMP_LIBRARIES ${OpenMP_CXX_LIBRARIES})
+  if(APPLE)
+    # On macOS, the compiler needs additional help to find the <omp.h> header.
+    # See issue #2642 for more information.
+    set(OPENMP_INCLUDE_DIRS ${OpenMP_CXX_INCLUDE_DIRS})
+  endif(APPLE)
+endif()
+
 # SuiteSparse (before SUNDIALS which may depend on KLU)
 if (MFEM_USE_SUITESPARSE)
   find_package(SuiteSparse REQUIRED
@@ -299,6 +290,11 @@ if (MFEM_USE_SUNDIALS)
   find_package(SUNDIALS REQUIRED ${SUNDIALS_COMPONENTS})
 endif()
 
+# Mesquite
+if (MFEM_USE_MESQUITE)
+  find_package(Mesquite REQUIRED)
+endif()
+
 # SuperLU_DIST can only be enabled in parallel
 if (MFEM_USE_SUPERLU)
   if (MFEM_USE_MPI)
@@ -456,11 +452,6 @@ if (MFEM_USE_PARELAG)
    find_package(PARELAG REQUIRED)
 endif()
 
-# Enzyme
-if (MFEM_USE_ENZYME)
-   find_package(ENZYME REQUIRED)
-endif()
-
 # MFEM_TIMER_TYPE
 if (NOT DEFINED MFEM_TIMER_TYPE)
   if (APPLE)
@@ -479,29 +470,16 @@ if (NOT DEFINED MFEM_TIMER_TYPE)
   endif()
 endif()
 
-# Without this, CMake 3.21.1 (and 3.20.2) run into CMake Errors like the following:
-# CMake Error at config/cmake/modules/MfemCmakeUtilities.cmake:60 (add_library):
-#   Target "mfem" links to target "Threads::Threads" but the target was not
-#   found.  Perhaps a find_package() call is missing for an IMPORTED target, or
-#   an ALIAS target is missing?
-# Call Stack (most recent call first):
-#   CMakeLists.txt:474 (mfem_add_library)
-#
-# NOTE: We need to figure out which TPL library adds the dependency on
-#       "Threads::Threads" and call the next line only when that TPL library is
-#       enabled.  -V. Dobrev
-find_package(Threads REQUIRED)
-
 # List all possible libraries in order of dependencies.
 # [METIS < SuiteSparse]:
 #    With newer versions of SuiteSparse which include METIS header using 64-bit
 #    integers, the METIS header (with 32-bit indices, as used by mfem) needs to
 #    be before SuiteSparse.
-set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
-    SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
+set(MFEM_TPLS OPENMP HYPRE BLAS LAPACK SuperLUDist METIS SuiteSparse SUNDIALS
+    PETSC SLEPC MESQUITE MUMPS STRUMPACK AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
-    ADIOS2 CUBLAS CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
-    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM ENZYME)
+    ADIOS2 CUSPARSE MKL_CPARDISO AMGX CALIPER CODIPACK BENCHMARK PARELAG
+    MPI_CXX HIP HIPSPARSE MOONOLITH BLITZ ALGOIM)
 
 # Add all *_FOUND libraries in the variable TPL_LIBRARIES.
 set(TPL_LIBRARIES "")
@@ -513,12 +491,18 @@ foreach(TPL IN LISTS MFEM_TPLS)
     list(APPEND TPL_INCLUDE_DIRS ${${TPL}_INCLUDE_DIRS})
   endif()
 endforeach(TPL)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
-list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_INCLUDE_DIRS)
 # message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 
+if (OPENMP_FOUND)
+  message(STATUS "MFEM: using package OpenMP")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if (MFEM_USE_CUDA)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS}")
+  endif()
+endif()
+
 message(STATUS "MFEM build type: CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 message(STATUS "MFEM version: v${MFEM_VERSION_STRING}")
 message(STATUS "MFEM git string: ${MFEM_GIT_STRING}")
@@ -572,9 +556,9 @@ endif()
 message(STATUS "TPL_INCLUDE_DIRS = ${TPL_INCLUDE_DIRS}")
 target_include_directories(mfem
   PUBLIC
+    ${TPL_INCLUDE_DIRS}
     $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
-    ${TPL_INCLUDE_DIRS})
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
 set_target_properties(mfem PROPERTIES VERSION "${mfem_VERSION}")
 set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
 

CONTRIBUTING.md

@@ -102,9 +102,7 @@ The MFEM source code has the following structure:
   .
   ├── config
   │   ├── cmake
-  │   ├── docker
-  │   ├── githooks
-  │   └── vcpkg
+  │   └── githooks
   ├── data
   ├── doc
   ├── examples
@@ -113,7 +111,6 @@ The MFEM source code has the following structure:
   │   ├── ginkgo
   │   ├── hiop
   │   ├── jupyter
-  │   ├── moonolith
   │   ├── petsc
   │   ├── pumi
   │   ├── sundials
@@ -121,25 +118,21 @@ The MFEM source code has the following structure:
   ├── fem
   │   ├── ceed
   │   ├── fe
-  │   ├── lor
-  │   ├── moonolith
   │   ├── qinterp
+  │   ├── moonolith
   │   └── tmop
   ├── general
   ├── linalg
   │   └── simd
   ├── mesh
-  │   └── submesh
   ├── miniapps
   │   ├── adjoint
   │   ├── autodiff
   │   ├── common
   │   ├── electromagnetics
   │   ├── gslib
-  │   ├── hooke
   │   ├── meshing
   │   ├── mtop
-  │   ├── multidomain
   │   ├── navier
   │   ├── nurbs
   │   ├── parelag

INSTALL

@@ -7,10 +7,6 @@
 
                                https://mfem.org
 
-This file provides a detailed description of how to build and install the MFEM
-library. For a simple build, see the step-by-step instructions on the website
-at https://mfem.org/building.
-
 The MFEM library has a serial and an MPI-based parallel version, which largely
 share the same code base. The only prerequisite for building the serial version
 of MFEM is a (modern) C++ compiler, such as g++. The parallel version of MFEM
@@ -20,11 +16,7 @@ requires an MPI C++ compiler, as well as the following external libraries:
   https://github.com/hypre-space/hypre
 
 - METIS (a family of multilevel partitioning algorithms)
-  https://github.com/mfem/tpls
-
-  Note: We recommend our mirror of metis-4.0.3/5.1.0 above because the METIS
-  webpage, http://glaros.dtc.umn.edu/gkhome/metis/metis/overview, is often down
-  and we don't support yet the new repo https://github.com/KarypisLab/METIS.
+  http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
 
 The hypre dependency can be downloaded as a tarball from GitHub or from the
 project webpage https://www.llnl.gov/casc/hypre. For example, the 2.24.0 release
@@ -337,6 +329,10 @@ MFEM_USE_SUNDIALS = YES/NO
    library. When enabled, this option uses the SUNDIALS_* library options,
    see below.
 
+MFEM_USE_MESQUITE = YES/NO
+   Enable MFEM functionality based on the Mesquite library. When enabled, this
+   option uses the MESQUITE_* library options, see below.
+
 MFEM_USE_SUITESPARSE = YES/NO
    Enable MFEM functionality based on the SuiteSparse library. Currently, this
    option adds the classes UMFPackSolver and KLUSolver (both sparse serial
@@ -476,10 +472,10 @@ MFEM_USE_CODIPACK = YES/NO
   www.scicomp.uni-kl.de/codi/
 
 MFEM_USE_ALGOIM = YES/NO
-  Enable the usage of Algoim - a collection of high-order accurate numerical
-  methods and C++ algorithms for working with implicitly-defined geometry and
-  level set methods. The Algoim library requires the Blitz++ library. The MFEM
-  provides interface to Algoim v1. Thus, to check out the specific state use:
+  Enable the usage of Algoim - a collection of high-order accurate numerical 
+  methods and C++ algorithms for working with implicitly-defined geometry and 
+  level set methods. The Algoim library requires the Blitz++ library. The MFEM 
+  provides interface to Algoim v1. Thus, to check out the specific state use: 
   git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a
   https://algoim.github.io
 
@@ -554,7 +550,7 @@ MFEM_USE_FMS = YES/NO
    Enables support for the FMS library which consists of the DataCollection
    sub-class mfem::FMSDataCollection for I/O in FMS formats, see the header file
    fem/fmsdatacollection.hpp. In addition, this option enables in-memory
-   conversion routines between FMS's FmsDataCollection structure and MFEM's
+   convetion routines between FMS's FmsDataCollection structure and MFEM's
    DataCollection class, see the header file fem/fmsconvert.hpp.
 
 MFEM_USE_PARELAG = YES/NO
@@ -562,14 +558,6 @@ MFEM_USE_PARELAG = YES/NO
    use ParELAG. In fact, ParELAG is dependent on MFEM. Therefore, this option
    currently only concerns the miniapps.
 
-MFEM_USE_ENZYME = YES/NO
-   Enables automatic differentiation support through the LLVM plugin Enzyme.
-   This requires the compiler to be set to clang (>=14.0.0). We also advise to
-   use the link time optimization (LTO) plugin, to enable functions that you
-   define over multiple files (compilation units) and want to be differentiated
-   automatically, to work. This requires to also use LLVM/LLD for linking.
-   Recommended options are in config/defaults.mk.
-
 MFEM_BUILD_TAG = (any value)
    An optional tag to characterize the build. Exported to config/config.mk.
    Can be used to identify the MFEM build from other makefiles.
@@ -601,7 +589,7 @@ The specific libraries and their options are:
 
 - METIS, used when MFEM_USE_METIS = YES. If using METIS 5, set
   MFEM_USE_METIS_5 = YES (default is to use METIS 4).
-  URL: https://github.com/mfem/tpls (MFEM mirror, see above)
+  URL: http://glaros.dtc.umn.edu/gkhome/metis/metis/overview
   Options: METIS_OPT, METIS_LIB.
   Versions: METIS 4.0.3 or 5.1.0.
 
@@ -632,6 +620,11 @@ The specific libraries and their options are:
   Options: SUNDIALS_OPT, SUNDIALS_LIB.
   Versions: SUNDIALS >= 5.0.0, SUNDIALS >= 5.4.0 for CUDA support.
 
+- Mesquite (optional), used when MFEM_USE_MESQUITE = YES.
+  URL: http://trilinos.org/oldsite/packages/mesquite
+  Options: MESQUITE_OPT, MESQUITE_LIB.
+  The Mesquite support is deprecated and will be removed in the future.
+
 - SuiteSparse (optional), used when MFEM_USE_SUITESPARSE = YES.
   URL: http://faculty.cse.tamu.edu/davis/suitesparse.html
   Options: SUITESPARSE_OPT, SUITESPARSE_LIB.
@@ -753,12 +746,12 @@ The specific libraries and their options are:
   Options: GSLIB_OPT, GSLIB_LIB.
   Versions: GSLIB >= 1.0.7.
 
-- ALGOIM (optional), used when MFEM_USE_ALGOIM=YES. The library provides only
+- ALGOIM (optional), used when MFE_USE_ALGOIM=YES. The library provides only
   headers so it just needs to be downloaded at the same level as MFEM. Download
   the specific version we use as:
   "git clone https://github.com/algoim/algoim.git;
    git checkout 9c9ca0ef094d8ab0390ed36367a1151b459bbe0a"
-  ALGOIM depends on BLITZ and the library must be built prior to the MFEM build.
+  ALGOIM depends on BLITZ and rhe library must be built prior to the MFEM build.
   Download v1.0.2, untar it at the same level as MFEM and create a symbolic link:
   "ln -s blitz-1.0.2 blitz".
   Build Blitz using CMake as:
@@ -767,6 +760,8 @@ The specific libraries and their options are:
   Options: BLITZ_OPT, BLITZ_LIB
   Versions: BLITZ = 1.0.2
 
+
+
 - MKL CPardiso (optional), used when MFEM_USE_MKL_CPARDISO = YES.
   URL: https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html
   Options: MKL_CPARDISO_OPT, MKL_CPARDISO_LIB.
@@ -843,12 +838,6 @@ The specific libraries and their options are:
   URL: https://github.com/LLNL/parelag
   Options: PARELAG_DIR, PARELAG_OPT, PARELAG_LIB.
 
-- Enzyme, used when MFEM_USE_ENZYME = YES. Requires LLVM/Clang >= 14.0.0.
-  URL: https://github.com/EnzymeAD/Enzyme
-  Options: ENZYME_DIR, ENZYME_OPT, ENZYME_LIB.
-  Versions: Enzyme >= v0.0.33.
-
-
 Building with CMake
 ===================
 The MFEM build system consists of two steps: configuration and compilation.
@@ -960,6 +949,7 @@ MFEM_USE_LEGACY_OPENMP
 MFEM_USE_OPENMP
 MFEM_USE_MEMALLOC
 MFEM_TIMER_TYPE - Set automatically, can be overwritten.
+MFEM_USE_MESQUITE
 MFEM_USE_SUITESPARSE
 MFEM_USE_SUPERLU
 MFEM_USE_MUMPS
@@ -986,7 +976,6 @@ MFEM_USE_CALIPER
 MFEM_USE_FMS
 MFEM_USE_BENCHMARK
 MFEM_USE_PARELAG
-MFEM_USE_ENZYME
 
 The following options are CMake specific:
 
@@ -1024,6 +1013,7 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - HYPRE
  - METIS - The option MFEM_USE_METIS_5 is auto-detected.
  - ParMETIS
+ - MESQUITE
  - SuiteSparse
  - SuperLUDist, STRUMPACK
  - Ginkgo
@@ -1045,7 +1035,6 @@ The CMake build system adds auto-detection for the following packages/libraries:
  - FMS
  - BENCHMARK
  - ParELAG
- - Enzyme
 
 The following built-in CMake packages are also used:
 

config/XSDKDefaults.cmake

@@ -212,6 +212,10 @@ IF (DEFINED TPL_ENABLE_SUNDIALS)
   SET(MFEM_USE_SUNDIALS ${TPL_ENABLE_SUNDIALS} CACHE BOOL "Enable SUNDIALS usage" FORCE)
 ENDIF()
 
+IF (DEFINED TPL_ENABLE_MESQUITE)
+  SET(MFEM_USE_MESQUITE ${TPL_ENABLE_MESQUITE} CACHE BOOL "Enable MESQUITE usage" FORCE)
+ENDIF()
+
 IF (DEFINED TPL_ENABLE_SUITESPARSE)
   SET(MFEM_USE_SUITESPARSE ${TPL_ENABLE_SUITESPARSE} CACHE BOOL "Enable SuiteSparse usage" FORCE)
 ENDIF()

config/cmake/MFEMConfig.cmake.in

@@ -29,6 +29,7 @@ set(MFEM_USE_LEGACY_OPENMP @MFEM_USE_LEGACY_OPENMP@)
 set(MFEM_USE_MEMALLOC @MFEM_USE_MEMALLOC@)
 set(MFEM_TIMER_TYPE @MFEM_TIMER_TYPE@)
 set(MFEM_USE_SUNDIALS @MFEM_USE_SUNDIALS@)
+set(MFEM_USE_MESQUITE @MFEM_USE_MESQUITE@)
 set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
 set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
 set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
@@ -60,7 +61,6 @@ set(MFEM_USE_CALIPER @MFEM_USE_CALIPER@)
 set(MFEM_USE_ALGOIM @MFEM_USE_ALGOIM@)
 set(MFEM_USE_BENCHMARK @MFEM_USE_BENCHMARK@)
 set(MFEM_USE_PARELAG @MFEM_USE_PARELAG@)
-set(MFEM_USE_ENZYME @MFEM_USE_ENZYME@)
 
 set(MFEM_CXX_COMPILER "@CMAKE_CXX_COMPILER@")
 set(MFEM_CXX_FLAGS "@CMAKE_CXX_FLAGS@")

config/cmake/config.hpp.in

@@ -77,6 +77,9 @@
 // Internal MFEM option: enable group/batch allocation for some small objects.
 #cmakedefine MFEM_USE_MEMALLOC
 
+// Enable MFEM functionality based on the Mesquite library.
+#cmakedefine MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 #cmakedefine MFEM_USE_SUITESPARSE
 
@@ -187,7 +190,4 @@
 // Enable MFEM functionality based on the Google Benchmark library.
 #cmakedefine MFEM_USE_BENCHMARK
 
-// Enable Enzyme for AD
-#cmakedefine MFEM_USE_ENZYME
-
 #endif // MFEM_CONFIG_HEADER

config/cmake/modules/FindCaliper.cmake

@@ -16,22 +16,7 @@
 
 include(MfemCmakeUtilities)
 mfem_find_package(Caliper CALIPER CALIPER_DIR
-      "include" "caliper/cali.h"
-      "lib" "caliper"
-      "Paths to headers required by Caliper."
-      "Libraries required by Caliper.")
-
-# Append adiak path/lib if the user provided ADIAK_DIR
-if(ADIAK_DIR AND EXISTS ${ADIAK_DIR})
-    find_package(adiak NO_DEFAULT_PATH REQUIRED PATHS ${ADIAK_DIR}/lib/cmake/adiak ${ADIAK_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${adiak_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${adiak_LIBRARIES})
-endif()
-
-# Append gotcha path/lib if the user provided GOTCHA_DIR
-if(GOTCHA_DIR AND EXISTS ${GOTCHA_DIR})
-    find_package(gotcha NO_DEFAULT_PATH REQUIRED PATHS ${GOTCHA_DIR}/lib/cmake/gotcha ${GOTCHA_DIR})
-    list(APPEND CALIPER_INCLUDE_DIRS ${gotcha_INCLUDE_DIRS})
-    list(APPEND CALIPER_LIBRARIES ${gotcha_LIBRARIES})
-endif()
-
+       	"include" "caliper/cali.h"
+       	"lib" "caliper"
+  	"Paths to headers required by Caliper."
+       	"Libraries required by Caliper.")

config/cmake/modules/FindENZYME.cmake

@@ -1,27 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-message(STATUS "Looking for ENZYME ...")
-message(STATUS "   in ENZYME_DIR = ${ENZYME_DIR}")
-
-# Make sure the directory and version combination works. Do nothing otherwise.
-if(EXISTS "${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
-  message(STATUS "Found ENZYME: ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so")
-
-  # Set ENZYME_FOUND
-  set(ENZYME_FOUND TRUE CACHE BOOL "ENZYME was found." FORCE)
-
-  # Set CXX flags to accommodate the Enzyme Clang plugin
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -load -Xclang ${ENZYME_DIR}/ClangEnzyme-${ENZYME_VERSION}.so -mllvm -enzyme-loose-types=1")
-  set(MFEM_USE_ENZYME YES)
-else()
-
-endif()

config/cmake/modules/FindHDF5.cmake

@@ -1,70 +0,0 @@
-# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-# LICENSE and NOTICE for details. LLNL-CODE-806117.
-#
-# This file is part of the MFEM library. For more information and source code
-# availability visit https://mfem.org.
-#
-# MFEM is free software; you can redistribute it and/or modify it under the
-# terms of the BSD-3 license. We welcome feedback and contributions, see file
-# CONTRIBUTING.md for details.
-
-# Defines the following variables:
-#   - HDF5_FOUND - If HDF5 was found
-#   - HDF5_LIBRARIES - The HDF5 libraries
-#   - HDF5_INCLUDE_DIRS - The HDF5 include directories
-
-# NOTE: Using this FindHDF5.cmake instead of the CMake provided version may lead
-#       to issues with some TPL libraries that depend (or may depend) on HDF5.
-#       For this reason, we should consider removing this file, or at least
-#       making it use the CMake provided version by default and apply the logic
-#       below only when specifically requested by a user.  -V. Dobrev
-
-# First Check for HDF5_DIR
-if(NOT HDF5_DIR)
-  message(FATAL_ERROR
-    "Could not find HDF5. HDF5 support needs explicit HDF5_DIR")
-endif()
-
-if (NOT HDF5_FIND_QUIETLY)
-  message(STATUS "Looking for HDF5 ...")
-  message(STATUS "   in HDF5_DIR = ${HDF5_DIR}")
-endif()
-
-# Find includes
-find_path( HDF5_INCLUDE_DIRS hdf5.h
-           PATHS  ${HDF5_DIR}/include/
-           NO_DEFAULT_PATH
-           NO_CMAKE_ENVIRONMENT_PATH
-           NO_CMAKE_PATH
-           NO_SYSTEM_ENVIRONMENT_PATH
-           NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_LIBRARY NAMES hdf5 libhdf5 libhdf5_D libhdf5_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-find_library( __HDF5_HL_LIBRARY NAMES hdf5_hl libhdf5_hl libhdf5_hl_D libhdf5_hl_debug
-              PATHS ${HDF5_DIR}/lib
-              NO_DEFAULT_PATH
-              NO_CMAKE_ENVIRONMENT_PATH
-              NO_CMAKE_PATH
-              NO_SYSTEM_ENVIRONMENT_PATH
-              NO_CMAKE_SYSTEM_PATH)
-
-set(HDF5_LIBRARIES ${__HDF5_HL_LIBRARY} ${__HDF5_LIBRARY})
-
-include(FindPackageHandleStandardArgs)
-
-# Handle the QUIETLY and REQUIRED arguments and set HDF5_FOUND to TRUE if all
-# listed variables are TRUE
-find_package_handle_standard_args(HDF5
-                                  " *** HDF5 not found. Please set HDF5_DIR."
-                                  HDF5_LIBRARIES
-                                  HDF5_INCLUDE_DIRS
-                                  __HDF5_LIBRARY
-                                  __HDF5_HL_LIBRARY)

config/cmake/modules/FindMesquite.cmake

@@ -0,0 +1,20 @@
+# Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-806117.
+#
+# This file is part of the MFEM library. For more information and source code
+# availability visit https://mfem.org.
+#
+# MFEM is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+# Defines the following variables:
+#   - MESQUITE_FOUND
+#   - MESQUITE_LIBRARIES
+#   - MESQUITE_INCLUDE_DIRS
+
+include(MfemCmakeUtilities)
+mfem_find_package(Mesquite MESQUITE MESQUITE_DIR
+  "include" "Mesquite_all_headers.hpp" "lib" "mesquite"
+  "Paths to headers required by Mesquite." "Libraries required by Mesquite.")

config/cmake/modules/FindNetCDF.cmake

@@ -17,24 +17,18 @@
 include(MfemCmakeUtilities)
 
 # FindHDF5.cmake uses HDF5_ROOT, so we "translate" from the MFEM convention
-# (MFEM's FindHDF5.cmake does not need HDF5_ROOT)
-# set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
-
+set(HDF5_ROOT ${HDF5_DIR} CACHE PATH "")
 # We need to guard against the case where HDF5 was already found but without
 # the HL extensions (in which case mfem_find_package will treat the package
 # as already having been found), so we reset the variable to force FindHDF5.cmake
 # to be called for a second time
 set(HDF5_FOUND OFF)
 enable_language(C) # FindHDF5.cmake uses the C compiler
-
 mfem_find_package(NetCDF NETCDF NETCDF_DIR "include" netcdf.h "lib" netcdf
   "Paths to headers required by NetCDF." "Libraries required by NetCDF.")
-
 # The HL extension libraries are in a separate variable and must precede
 # the "regular" hdf5 library, as hdf5_hl depends on hdf5
 # The netcdf library will always be the first element of NETCDF_LIBRARIES
 # and we need to insert after that library but before the hdf5 library, so
 # position 1 is used
-# (MFEM's FindHDF5.cmake does not set HDF5_C_LIBRARY_hdf5_hl and the HL library
-# is already added to NETCDF_LIBRARIES)
-# list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})
+list(INSERT NETCDF_LIBRARIES 1 ${HDF5_C_LIBRARY_hdf5_hl})

config/cmake/modules/FindUMPIRE.cmake

@@ -14,6 +14,6 @@
 #   - UMPIRE_LIBRARIES
 #   - UMPIRE_INCLUDE_DIRS
 
-find_package(umpire REQUIRED CONFIG)
-set(UMPIRE_FOUND ${umpire_FOUND})
-set(UMPIRE_LIBRARIES "umpire")
+include(MfemCmakeUtilities)
+mfem_find_package(UMPIRE UMPIRE UMPIRE_DIR "include" "umpire/Umpire.hpp" "lib" "umpire"
+  "Paths to headers required by UMPIRE." "Libraries required by UMPIRE.")

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -43,18 +43,22 @@ function(convert_filenames_to_full_paths NAMES)
   set(${NAMES} ${tmp_names} PARENT_SCOPE)
 endfunction()
 
-# Wrapper for add_executable
+# Wrapper for add_executable that calls the HIP wrapper if applicable
 macro(mfem_add_executable NAME)
-  add_executable(${NAME} ${ARGN})
-  if (MFEM_USE_CUDA)
-    set_target_properties(${NAME} PROPERTIES
-      CUDA_RESOLVE_DEVICE_SYMBOLS ON)
+  if (MFEM_USE_HIP)
+    add_executable(${NAME} ${ARGN})
+  else()
+    add_executable(${NAME} ${ARGN})
   endif()
 endmacro()
 
-# Wrapper for add_library
+# Wrapper for add_library that calls the HIP wrapper if applicable
 macro(mfem_add_library NAME)
-  add_library(${NAME} ${ARGN})
+  if (MFEM_USE_HIP)
+    add_library(${NAME} ${ARGN})
+  else()
+    add_library(${NAME} ${ARGN})
+  endif()
 endmacro()
 
 # Simple shortcut to add_custom_target() with option to add the target to the
@@ -162,12 +166,27 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
 
   # Append the additional libraries and options
   if (LIBRARIES_LIST)
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${LIBRARIES_LIST})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${LIBRARIES_LIST})
+    endif()
   endif()
   if (EXTRA_OPTIONS_LIST)
     string(REPLACE ";" " " EXTRA_OPTIONS_STRING "${EXTRA_OPTIONS_LIST}")
     message(STATUS "${MFEM_EXE_NAME}: add flags \"${EXTRA_OPTIONS_STRING}\"")
-    target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${EXTRA_OPTIONS_LIST})
+    else()
+      get_target_property(THIS_COMPILE_FLAGS ${MFEM_EXE_NAME} COMPILE_FLAGS)
+      if (THIS_COMPILE_FLAGS)
+        set(THIS_COMPILE_FLAGS "${THIS_COMPILE_FLAGS} ${EXTRA_OPTIONS_STRING}")
+      else()
+        set(THIS_COMPILE_FLAGS "${EXTRA_OPTIONS_STRING}")
+      endif()
+      set_target_properties(${MFEM_EXE_NAME}
+        PROPERTIES COMPILE_FLAGS ${THIS_COMPILE_FLAGS})
+    endif()
   endif()
   if (EXTRA_DEFINES_LIST)
     target_compile_definitions(${MFEM_EXE_NAME} PRIVATE ${EXTRA_DEFINES_LIST})
@@ -176,15 +195,17 @@ macro(add_mfem_miniapp MFEM_EXE_NAME)
   # Handle the MPI separately
   if (MFEM_USE_MPI)
     # Add MPI_CXX_LIBRARIES, in case this target does not link with mfem.
-    target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    if(CMAKE_VERSION VERSION_GREATER 2.8.11)
+      target_link_libraries(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_LIBRARIES})
+    else()
+      target_link_libraries(${MFEM_EXE_NAME} ${MPI_CXX_LIBRARIES})
+    endif()
 
     if (MPI_CXX_INCLUDE_PATH)
       target_include_directories(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_INCLUDE_PATH})
     endif()
     if (MPI_CXX_COMPILE_FLAGS)
-      separate_arguments(MPI_CXX_COMPILE_ARGS UNIX_COMMAND
-        "${MPI_CXX_COMPILE_FLAGS}")
-      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_ARGS})
+      target_compile_options(${MFEM_EXE_NAME} PRIVATE ${MPI_CXX_COMPILE_FLAGS})
     endif()
 
     if (MPI_CXX_LINK_FLAGS)
@@ -865,7 +886,7 @@ function(mfem_export_mk_files)
   set(CONFIG_MK_BOOL_VARS MFEM_USE_MPI MFEM_USE_METIS MFEM_USE_METIS_5
       MFEM_DEBUG MFEM_USE_EXCEPTIONS MFEM_USE_ZLIB MFEM_USE_LIBUNWIND
       MFEM_USE_LAPACK MFEM_THREAD_SAFE MFEM_USE_LEGACY_OPENMP MFEM_USE_OPENMP
-      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_SUITESPARSE
+      MFEM_USE_MEMALLOC MFEM_USE_SUNDIALS MFEM_USE_MESQUITE MFEM_USE_SUITESPARSE
       MFEM_USE_SUPERLU MFEM_USE_SUPERLU5 MFEM_USE_MUMPS MFEM_USE_STRUMPACK
       MFEM_USE_GINKGO MFEM_USE_AMGX MFEM_USE_GNUTLS MFEM_USE_NETCDF
       MFEM_USE_PETSC MFEM_USE_SLEPC MFEM_USE_MPFR MFEM_USE_SIDRE MFEM_USE_FMS
@@ -873,7 +894,7 @@ function(mfem_export_mk_files)
       MFEM_USE_HIP MFEM_USE_RAJA MFEM_USE_OCCA MFEM_USE_CEED MFEM_USE_CALIPER
       MFEM_USE_UMPIRE MFEM_USE_SIMD MFEM_USE_ADIOS2 MFEM_USE_MKL_CPARDISO
       MFEM_USE_ADFORWARD MFEM_USE_CODIPACK MFEM_USE_BENCHMARK MFEM_USE_PARELAG
-      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM MFEM_USE_ENZYME)
+      MFEM_USE_MOONOLITH MFEM_USE_ALGOIM)
   foreach(var ${CONFIG_MK_BOOL_VARS})
     if (${var})
       set(${var} YES)

config/config.hpp

@@ -31,11 +31,9 @@
 
 // Windows specific options
 #ifdef _WIN32
-#ifndef _USE_MATH_DEFINES
 // Macro needed to get defines like M_PI from <cmath>. (Visual Studio C++ only?)
 #define _USE_MATH_DEFINES
 #endif
-#endif
 // On Cygwin the option -std=c++11 prevents the definition of M_PI. Defining
 // the following macro allows us to get M_PI and some needed functions, e.g.
 // posix_memalign(), strdup(), strerror_r().

config/config.hpp.in

@@ -85,6 +85,9 @@
 // Enable MFEM functionality based on the SUNDIALS libraries.
 // #define MFEM_USE_SUNDIALS
 
+// Enable MFEM functionality based on the Mesquite library.
+// #define MFEM_USE_MESQUITE
+
 // Enable MFEM functionality based on the SuiteSparse library.
 // #define MFEM_USE_SUITESPARSE
 
@@ -192,7 +195,4 @@
 // Enable functionality based on the Google Benchmark library.
 // #define MFEM_USE_BENCHMARK
 
-// Enable the Enzyme LLVM plugin
-// #define MFEM_USE_ENZYME
-
 #endif // MFEM_CONFIG_HEADER

config/config.mk.in

@@ -29,6 +29,7 @@ MFEM_USE_OPENMP        = @MFEM_USE_OPENMP@
 MFEM_USE_MEMALLOC      = @MFEM_USE_MEMALLOC@
 MFEM_TIMER_TYPE        = @MFEM_TIMER_TYPE@
 MFEM_USE_SUNDIALS      = @MFEM_USE_SUNDIALS@
+MFEM_USE_MESQUITE      = @MFEM_USE_MESQUITE@
 MFEM_USE_SUITESPARSE   = @MFEM_USE_SUITESPARSE@
 MFEM_USE_SUPERLU       = @MFEM_USE_SUPERLU@
 MFEM_USE_SUPERLU5      = @MFEM_USE_SUPERLU5@
@@ -62,7 +63,6 @@ MFEM_USE_ADFORWARD     = @MFEM_USE_ADFORWARD@
 MFEM_USE_CODIPACK      = @MFEM_USE_CODIPACK@
 MFEM_USE_BENCHMARK     = @MFEM_USE_BENCHMARK@
 MFEM_USE_PARELAG       = @MFEM_USE_PARELAG@
-MFEM_USE_ENZYME        = @MFEM_USE_ENZYME@
 
 # Compiler, compile options, and link options
 MFEM_CXX       = @MFEM_CXX@

config/defaults.cmake

@@ -30,6 +30,7 @@ option(MFEM_USE_OPENMP "Enable the OpenMP backend" OFF)
 option(MFEM_USE_LEGACY_OPENMP "Enable legacy OpenMP usage" OFF)
 option(MFEM_USE_MEMALLOC "Enable the internal MEMALLOC option." ON)
 option(MFEM_USE_SUNDIALS "Enable SUNDIALS usage" OFF)
+option(MFEM_USE_MESQUITE "Enable MESQUITE usage" OFF)
 option(MFEM_USE_SUITESPARSE "Enable SuiteSparse usage" OFF)
 option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
 option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
@@ -63,7 +64,6 @@ option(MFEM_USE_ADFORWARD "Enable forward mode for AD" OFF)
 option(MFEM_USE_CODIPACK "Enable automatic differentiation (AD) using CoDiPack" OFF)
 option(MFEM_USE_BENCHMARK "Enable Google Benchmark" OFF)
 option(MFEM_USE_PARELAG "Enable ParELAG" OFF)
-option(MFEM_USE_ENZYME "Enable Enzyme" OFF)
 
 # Optional overrides for autodetected MPIEXEC and MPIEXEC_NUMPROC_FLAG
 # set(MFEM_MPIEXEC "mpirun" CACHE STRING "Command for running MPI tests")
@@ -123,6 +123,9 @@ set(SUNDIALS_DIR "${MFEM_DIR}/../sundials-5.0.0/instdir" CACHE PATH
 # set(SUNDIALS_REQUIRED_PACKAGES "SuiteSparse/KLU/AMD/BTF/COLAMD/config"
 #     CACHE STRING "Additional packages required by SUNDIALS.")
 
+set(MESQUITE_DIR "${MFEM_DIR}/../mesquite-2.99" CACHE PATH
+    "Path to the Mesquite library.")
+
 set(SuiteSparse_DIR "${MFEM_DIR}/../SuiteSparse" CACHE PATH
     "Path to the SuiteSparse library.")
 set(SuiteSparse_REQUIRED_PACKAGES "BLAS" "METIS"
@@ -184,7 +187,6 @@ set(GNUTLS_DIR "" CACHE PATH "Path to the GnuTLS library.")
 
 set(GSLIB_DIR "" CACHE PATH "Path to the GSLIB library.")
 
-set(HDF5_DIR "/usr" CACHE PATH "Path to the HDF5 library.")
 set(NETCDF_DIR "" CACHE PATH "Path to the NetCDF library.")
 set(NetCDF_REQUIRED_PACKAGES "HDF5/C/HL" CACHE STRING
     "Additional packages required by NetCDF.")

config/defaults.mk

@@ -42,9 +42,6 @@ STATIC = YES
 SHARED = NO
 
 # CUDA configuration options
-#
-# If you set MFEM_USE_ENZYME=YES, CUDA_CXX has to be configured to use cuda with
-# clang as its host compiler.
 CUDA_CXX = nvcc
 CUDA_ARCH = sm_60
 CUDA_FLAGS = -x=cu --expt-extended-lambda -arch=$(CUDA_ARCH)
@@ -131,6 +128,7 @@ MFEM_USE_LEGACY_OPENMP = NO
 MFEM_USE_MEMALLOC      = YES
 MFEM_TIMER_TYPE        = $(if $(NOTMAC),2,4)
 MFEM_USE_SUNDIALS      = NO
+MFEM_USE_MESQUITE      = NO
 MFEM_USE_SUITESPARSE   = NO
 MFEM_USE_SUPERLU       = NO
 MFEM_USE_SUPERLU5      = NO
@@ -165,7 +163,6 @@ MFEM_USE_ADFORWARD     = NO
 MFEM_USE_CODIPACK      = NO
 MFEM_USE_BENCHMARK     = NO
 MFEM_USE_PARELAG       = NO
-MFEM_USE_ENZYME        = NO
 
 # MPI library compile and link flags
 # These settings are used only when building MFEM with MPI + HIP
@@ -178,7 +175,7 @@ ifeq ($(MFEM_USE_MPI)$(MFEM_USE_HIP),YESYES)
 endif
 
 # ROCM/HIP directory such that ROCM/HIP libraries like rocsparse and rocrand are
-# found in $(HIP_DIR)/lib, usually as links. Typically, this directory is of
+# found in $(HIP_DIR)/lib, usually as links. Typically, this directoory is of
 # the form /opt/rocm-X.Y.Z which is called ROCM_PATH by hipconfig.
 ifeq ($(MFEM_USE_HIP),YES)
    HIP_DIR := $(patsubst %/,%,$(dir $(shell which $(HIP_CXX))))
@@ -206,7 +203,7 @@ HYPRE_OPT = -I$(HYPRE_DIR)/include
 HYPRE_LIB = -L$(HYPRE_DIR)/lib -lHYPRE
 ifeq (YES,$(MFEM_USE_CUDA))
    # This is only necessary when hypre is built with cuda:
-   HYPRE_LIB += -lcusparse -lcurand -lcublas
+   HYPRE_LIB += -lcusparse -lcurand
 endif
 ifeq (YES,$(MFEM_USE_HIP))
    # This is only necessary when hypre is built with hip:
@@ -250,16 +247,12 @@ POSIX_CLOCKS_LIB = -lrt
 # SUNDIALS library configuration
 # For sundials_nvecmpiplusx and nvecparallel remember to build with MPI_ENABLE=ON
 # and modify cmake variables for hypre for sundials
-SUNDIALS_DIR = @MFEM_DIR@/../sundials-5.0.0/instdir
-# SUNDIALS >= 6.4.0 requires C++14:
-ifeq ($(MFEM_USE_SUNDIALS),YES)
-   BASE_FLAGS = -std=c++14
-endif
-SUNDIALS_OPT = -I$(SUNDIALS_DIR)/include
-SUNDIALS_LIB = $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib64\
- $(XLINKER)-rpath,$(SUNDIALS_DIR)/lib\
- -L$(SUNDIALS_DIR)/lib64 -L$(SUNDIALS_DIR)/lib\
+SUNDIALS_DIR    = @MFEM_DIR@/../sundials-5.0.0/instdir
+SUNDIALS_OPT    = -I$(SUNDIALS_DIR)/include
+SUNDIALS_LIBDIR = $(wildcard $(SUNDIALS_DIR)/lib*)
+SUNDIALS_LIB    = $(XLINKER)-rpath,$(SUNDIALS_LIBDIR) -L$(SUNDIALS_LIBDIR)\
  -lsundials_arkode -lsundials_cvodes -lsundials_nvecserial -lsundials_kinsol
+
 ifeq ($(MFEM_USE_MPI),YES)
    SUNDIALS_LIB += -lsundials_nvecparallel -lsundials_nvecmpiplusx
 endif
@@ -269,6 +262,11 @@ endif
 # If SUNDIALS was built with KLU:
 # MFEM_USE_SUITESPARSE = YES
 
+# MESQUITE library configuration
+MESQUITE_DIR = @MFEM_DIR@/../mesquite-2.99
+MESQUITE_OPT = -I$(MESQUITE_DIR)/include
+MESQUITE_LIB = -L$(MESQUITE_DIR)/lib -lmesquite
+
 # SuiteSparse library configuration
 LIB_RT = $(if $(NOTMAC),-lrt,)
 SUITESPARSE_DIR = @MFEM_DIR@/../SuiteSparse
@@ -307,7 +305,7 @@ SCALAPACK_LIB = -L$(SCALAPACK_DIR)/lib -lscalapack $(LAPACK_LIB)
 MPI_FORTRAN_LIB = -lmpifort
 # OpenMPI:
 # MPI_FORTRAN_LIB = -lmpi_mpifh
-# Additional Fortran library:
+# Additional Fortan library:
 # MPI_FORTRAN_LIB += -lgfortran
 
 # MUMPS library configuration
@@ -318,9 +316,6 @@ MUMPS_LIB = $(XLINKER)-rpath,$(MUMPS_DIR)/lib -L$(MUMPS_DIR)/lib -ldmumps\
 
 # STRUMPACK library configuration
 STRUMPACK_DIR = @MFEM_DIR@/../STRUMPACK-build
-ifeq ($(MFEM_USE_STRUMPACK),YES)
-   BASE_FLAGS = -std=c++14
-endif
 STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
 # If STRUMPACK was build with OpenMP support, the following may be need:
 # STRUMPACK_OPT += $(OPENMP_OPT)
@@ -473,17 +468,7 @@ OCCA_LIB = $(XLINKER)-rpath,$(OCCA_DIR)/lib -L$(OCCA_DIR)/lib -locca
 # CALIPER library configuration
 CALIPER_DIR = @MFEM_DIR@/../caliper
 CALIPER_OPT = -I$(CALIPER_DIR)/include
-CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 $(XLINKER)-rpath,$(CALIPER_DIR)/lib -L$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib -lcaliper
-
-ifdef ADIAK_DIR
-   CALIPER_OPT += -I$(ADIAK_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(ADIAK_DIR)/lib64 $(XLINKER)-rpath,$(ADIAK_DIR)/lib -L$(ADIAK_DIR)/lib64 -L$(ADIAK_DIR)/lib -ladiak
-endif
-ifdef GOTCHA_DIR
-   CALIPER_OPT += -I$(GOTCHA_DIR)/include
-   CALIPER_LIB += $(XLINKER)-rpath,$(GOTCHA_DIR)/lib64 $(XLINKER)-rpath,$(GOTCHA_DIR)/lib -L$(GOTCHA_DIR)/lib64 -L$(GOTCHA_DIR)/lib -lgotcha
-endif
-
+CALIPER_LIB = $(XLINKER)-rpath,$(CALIPER_DIR)/lib64 -L$(CALIPER_DIR)/lib64 -lcaliper
 
 # BLITZ library configuration
 BLITZ_DIR = @MFEM_DIR@/../blitz
@@ -535,22 +520,6 @@ PARELAG_DIR = @MFEM_DIR@/../parelag
 PARELAG_OPT = -I$(PARELAG_DIR)/src -I$(PARELAG_DIR)/build/src
 PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
 
-# Enzyme configuration
-
-# If you want to enable automatic differentiation at compile time, use the
-# options below, adapted to your configuration. To be more flexible, we
-# recommend using the Enzyme plugin during link time optimization. One option is
-# to add your options to the global compiler/linker flags like
-#
-# BASE_FLAGS += -flto
-# CXX_XLINKER += -fuse-ld=lld -Wl,--lto-legacy-pass-manager\
-#                -Wl,-mllvm=-load=$(ENZYME_DIR)/LLDEnzyme-$(ENZYME_VERSION).so -Wl,
-#
-ENZYME_DIR ?= @MFEM_DIR@/../enzyme
-ENZYME_VERSION ?= 14
-ENZYME_OPT = -fno-experimental-new-pass-manager -Xclang -load -Xclang $(ENZYME_DIR)/ClangEnzyme-$(ENZYME_VERSION).so
-ENZYME_LIB = ""
-
 # If YES, enable some informational messages
 VERBOSE = NO
 

config/docker/Dockerfile

@@ -1,27 +1,30 @@
-FROM ghcr.io/mfem/mfem-ubuntu-base:latest as builder
+FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
 
-# docker build -t ghcr.io/mfem/mfem-ubuntu .
-
-COPY ./config/docker/spack.yaml /opt/mfem-env/spack.yaml
-RUN apt-get install -y python3 && \
-    cd /opt/mfem-env && \
-    . /opt/spack/share/spack/setup-env.sh && \
-    spack env activate . && \
-    spack env view regenerate
-
-FROM ubuntu:22.04
-
-COPY --from=builder /opt/view /opt/view
-COPY --from=builder /opt/mfem-view /opt/mfem-view
+# docker build -t ghcr.io/mfem/mfem-ubuntu-base .
 
 RUN apt-get update && \
     apt-get install -y unzip gfortran && \
+    spack compiler find && \
     apt-get install -y libcurl4-openssl-dev libssl-dev
 
-ENV PATH=$PATH:/opt/mfem-view/bin
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/mfem-view/lib:/opt/mfem-view/lib64
-ENV DEBIAN_FRONTEND=noninteractive
+# /code is the working directory for code
+WORKDIR /code
+COPY . /code
+
+# This is for a spack environment/view to install from there
+WORKDIR /opt/mfem-env
+RUN . /opt/spack/share/spack/setup-env.sh && \
+    spack env create -d . && \
+    echo "  concretization: together" >> spack.yaml && \
+    spack env activate . && \
+    spack develop --path /code mfem@master+examples+miniapps && \
+    spack add mfem@master+examples+miniapps && \
+    spack install
+
+# ensure mfem always on various paths
+RUN cd /opt/mfem-env && \
+    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
 
 # The user will see the view on shell into the container
-WORKDIR /opt/mfem-view
-ENTRYPOINT ["/bin/bash"]
+WORKDIR /opt/mfem-env/.spack-env/view/
+ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]

config/docker/Dockerfile.base

@@ -1,47 +0,0 @@
-FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.0.3
-
-# docker build -f Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
-
-RUN apt-get update && \
-    apt-get install -y unzip gfortran && \
-    spack compiler find && \
-    apt-get install -y libcurl4-openssl-dev libssl-dev
-
-# /code is the working directory for code
-WORKDIR /code
-COPY . /code
-
-# This is for a spack environment/view to install from there
-RUN mkdir -p /opt/mfem-env \
-&&  (echo "spack:" \
-&&   echo "  view:" \
-&&   echo "    mfem:" \
-&&   echo "      root: /opt/mfem-view" \
-&&   echo "      link_type: copy" \
-&&   echo "  packages:" \
-&&   echo "    all:" \
-&&   echo "      target:" \
-&&   echo "      - x86_64_v3" \
-&&   echo "  config:" \
-&&   echo "    concretizer: clingo" \
-&&   echo "    compiler:" \
-&&   echo "      target:" \
-&&   echo "      - x86_64_v3" \
-&&   echo "    install_missing_compilers: true" \
-&&   echo "  concretization: together") > /opt/mfem-env/spack.yaml
-
-RUN cd /opt/mfem-env && \
-    . /opt/spack/share/spack/setup-env.sh && \
-    spack env activate . && \
-    spack develop --path /code mfem@master+examples+miniapps && \
-    spack add mfem@master+examples+miniapps # && \
-#    spack install
-
-# ensure mfem always on various paths
-#RUN cd /opt/mfem-env && \
-#    spack env activate --sh -d . >> /etc/profile.d/z10_spack_environment.sh
-
-# Present the software install when we shell in
-# The view is at /opt/mfem-env/.spack-env/view
-#WORKDIR /opt/software
-#ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c"]

config/docker/README.md

@@ -1,8 +1,7 @@
 # mfem Docker
 
-We provide a [Dockerfile.base](Dockerfile.base) to build an ubuntu base image,
-and a [Dockerfile](Dockerfile) to build a smaller one with a multi-stage build.
-You can use this image for a demo of using mfem! 🎉️
+We provide a [Dockerfile](Dockerfile) to build an ubuntu base image. You can use
+this image for a demo of using mfem! 🎉️
 
 Updated containers are built and deployed on merges to the main branch and releases.
 If you want to request a build on demand, you can [manually run the workflow](https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow) thanks to the workflow dispatch event.
@@ -15,33 +14,18 @@ is the [GitHub packages](https://github.com/features/packages) registry that sup
  Docker images and other OCI artifacts. From the root of the repository:
 
 ```bash
-$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu .
-$ docker build -f config/docker/Dockerfile.base -t ghcr.io/mfem/mfem-ubuntu-base .
+$ docker build -f config/docker/Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base .
 ```
 
-### Shell Ubuntu
-
-To shell into the container:
+or this directory:
 
 ```bash
-$ docker run -it ghcr.io/mfem/mfem-ubuntu
+$ docker build -f Dockerfile -t ghcr.io/mfem/mfem-ubuntu-base ../../
 ```
 
-This smaller image has a view where everything is installed.
+### Shell
 
-```bash
-$ ls
-bin  etc  include  lib  libexec  sbin  share  var
-```
-
- - Examples are in share/mfem/examples
- - Examples are in share/mfem/miniapps
-
-You can read more about interaction with these examples and miniapps below.
-
-### Shell Ubuntu Base
-
-To shell into the container:
+To shell into a container (here is an example with ubuntu):
 
 ```bash
 $ docker run -it ghcr.io/mfem/mfem-ubuntu-base bash
@@ -144,4 +128,3 @@ $ docker run -it ghcr.io/mfem/mfem-ubuntu-base -v $PWD:/src bash
 In the above, we can pretend your project is in the present working directory (PWD) and we are
 binding to source. You can then use the mfem in the container for development, and if you
 want to distribute your library or app in a container, you can use the mfem container as the base.
-

config/docker/spack.yaml

@@ -1,11 +0,0 @@
-spack:
-  specs: [mfem@master+examples+miniapps]
-  view: 
-    mfem:
-      root: /opt/mfem-view 
-      link_type: copy
-  concretization: together
-  develop:
-    mfem:
-      path: /code
-      spec: mfem@master+examples+miniapps

config/sample-runs.sh

@@ -554,14 +554,15 @@ function go()
    local cmd_line="${1##+( )}"
    cmd_line="${cmd_line%%+( )}"
    shopt -u extglob
+   eval local cmd=(${cmd_line})
    local res=""
    echo $sep
    echo "<${group}>" "${cmd_line}"
    echo $sep
    if [ "${timing}" == "yes" ]; then
-      timed_run eval "${cmd_line}"
+      timed_run "${cmd[@]}"
    else
-      eval "${cmd_line}"
+      "${cmd[@]}"
    fi
    if [ "$?" -eq 0 ]; then
       res="${green}  OK  ${none}"

config/vcpkg/ports/metis-mfem/disable-programs.patch

@@ -1,8 +0,0 @@
---- a/CMakeLists.txt	Wed Dec 21 18:24:22 2016
-+++ b/CMakeLists.txt	Wed Dec 21 18:24:26 2016
-@@ -20,4 +20,4 @@
- # Recursively look for CMakeLists.txt in subdirs.
- add_subdirectory("include")
- add_subdirectory("libmetis")
--add_subdirectory("programs")
-+# add_subdirectory("programs")

config/vcpkg/ports/metis-mfem/enable-install.patch

@@ -1,15 +0,0 @@
---- a/CMakeLists.txt	Sat Mar 30 17:24:45 2013
-+++ b/CMakeLists.txt	Wed Dec 21 18:23:43 2016
-@@ -4,11 +4,7 @@
- set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
- set(SHARED FALSE CACHE BOOL "build a shared library")
- 
--if(MSVC)
--  set(METIS_INSTALL FALSE)
--else()
--  set(METIS_INSTALL TRUE)
--endif()
-+set(METIS_INSTALL TRUE)
- 
- # Configure libmetis library.
- if(SHARED)

config/vcpkg/ports/metis-mfem/fix-INT_MIN_define.patch

@@ -1,34 +0,0 @@
-diff --git a/include/metis.h b/include/metis.h
-index dc5406a..7732437 100644
---- a/include/metis.h
-+++ b/include/metis.h
-@@ -72,10 +72,14 @@ typedef __int64 int64_t;
- #define PRId64       "I64d"
- #define SCNd32       "ld"
- #define SCNd64       "I64d"
-+#ifdef _WIN32
-+#include <stdint.h>
-+#else
- #define INT32_MIN    ((int32_t)_I32_MIN)
- #define INT32_MAX    _I32_MAX
- #define INT64_MIN    ((int64_t)_I64_MIN)
- #define INT64_MAX    _I64_MAX
-+#endif
- #else
- #include <inttypes.h>
- #endif
-diff --git a/GKlib/gk_arch.h b/GKlib/gk_arch.h
-index 78b1431..7258763 100644
---- a/GKlib/gk_arch.h
-+++ b/GKlib/gk_arch.h
-@@ -32,8 +32,8 @@
- 
- 
- #ifdef __MSC__ 
--  #include "ms_stdint.h"
--  #include "ms_inttypes.h"
-+  #include <stdint.h>
-+  #include <inttypes.h>
-   #include "ms_stat.h"
- #else
- #ifndef SUNOS

config/vcpkg/ports/metis-mfem/fix-gklib-vs14-math.patch

@@ -1,11 +0,0 @@
---- a/GKlib/gk_arch.h	Wed Dec 21 18:34:18 2016
-+++ b/GKlib/gk_arch.h	Wed Dec 21 18:30:49 2016
-@@ -58,7 +58,7 @@
- #define PTRDIFF_MAX  INT64_MAX
- #endif
- 
--#ifdef __MSC__
-+#if defined(__MSC__) && (_MSC_VER < 1900)
- /* MSC does not have rint() function */
- #define rint(x) ((int)((x)+0.5))  
- 

config/vcpkg/ports/metis-mfem/fix-linux-build-error.patch

@@ -1,14 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index e94f050..b9613a7 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -1,7 +1,8 @@
- cmake_minimum_required(VERSION 2.8)
- project(METIS)
- 
--set(GKLIB_PATH "GKlib" CACHE PATH "path to GKlib")
-+set(GKLIB_PATH "${CMAKE_SOURCE_DIR}/GKlib" CACHE PATH "path to GKlib")
-+
- set(SHARED FALSE CACHE BOOL "build a shared library")
- 
- set(METIS_INSTALL TRUE)

config/vcpkg/ports/metis-mfem/fix-metis-vs14-math.patch

@@ -1,11 +0,0 @@
---- a/libmetis/metislib.h	Sat Mar 30 17:24:45 2013
-+++ b/libmetis/metislib.h	Wed Dec 21 18:30:59 2016
-@@ -31,7 +31,7 @@
- #include <proto.h>
- 
- 
--#if defined(COMPILER_MSC)
-+#if defined(COMPILER_MSC) && (_MSC_VER < 1900)
- #if defined(rint)
-   #undef rint
- #endif

config/vcpkg/ports/metis-mfem/fix-runtime-install-destination.patch

@@ -1,10 +0,0 @@
---- a/libmetis/CMakeLists.txt	Sat Mar 30 17:24:45 2013
-+++ b/libmetis/CMakeLists.txt	Wed Dec 21 17:41:37 2016
-@@ -11,6 +11,6 @@
- if(METIS_INSTALL)
-   install(TARGETS metis
-     LIBRARY DESTINATION lib
--    RUNTIME DESTINATION lib
-+    RUNTIME DESTINATION bin
-     ARCHIVE DESTINATION lib)
- endif()

config/vcpkg/ports/metis-mfem/install-metisConfig.patch

@@ -1,44 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index b9613a7..e43ffee 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -22,3 +22,23 @@ include_directories(include)
- add_subdirectory("include")
- add_subdirectory("libmetis")
- # add_subdirectory("programs")
-+
-+if(METIS_INSTALL)
-+  set(PRJ_NAME metis)
-+  set(PRJ_VER 5.1.0)
-+  install(EXPORT metisTargets 
-+    FILE ${PRJ_NAME}Targets.cmake
-+    DESTINATION lib/cmake/${PRJ_NAME})
-+  include(CMakePackageConfigHelpers)
-+  write_basic_package_version_file(
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
-+    VERSION ${PRJ_VER}
-+    COMPATIBILITY SameMajorVersion)
-+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
-+    "include(\${CMAKE_CURRENT_LIST_DIR}/${PRJ_NAME}Targets.cmake)")
-+  install(FILES 
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}ConfigVersion.cmake
-+    ${CMAKE_CURRENT_BINARY_DIR}/${PRJ_NAME}Config.cmake
-+    DESTINATION lib/cmake/${PRJ_NAME})
-+endif()
-+
-diff --git a/libmetis/CMakeLists.txt b/libmetis/CMakeLists.txt
-index 7a5fc74..5a68cf0 100644
---- a/libmetis/CMakeLists.txt
-+++ b/libmetis/CMakeLists.txt
-@@ -9,8 +9,9 @@ if(UNIX)
- endif()
- 
- if(METIS_INSTALL)
--  install(TARGETS metis
-+  install(TARGETS metis EXPORT metisTargets
-     LIBRARY DESTINATION lib
-     RUNTIME DESTINATION bin
--    ARCHIVE DESTINATION lib)
-+    ARCHIVE DESTINATION lib
-+    INCLUDES DESTINATION include)
- endif()

config/vcpkg/ports/metis-mfem/portfile.cmake

@@ -1,41 +0,0 @@
-vcpkg_check_linkage(ONLY_STATIC_LIBRARY)
-set(OPTIONS -DSHARED=OFF)
-
-set(METIS_VERSION 5.1.0)
-
-vcpkg_download_distfile(ARCHIVE
-    URLS "https://github.com/mfem/tpls/raw/gh-pages/metis-${METIS_VERSION}.tar.gz"
-    FILENAME "metis-${METIS_VERSION}.tar.gz"
-    SHA512 deea47749d13bd06fbeaf98a53c6c0b61603ddc17a43dae81d72c8015576f6495fd83c11b0ef68d024879ed5415c14ebdbd87ce49c181bdac680573bea8bdb25
-)
-
-vcpkg_extract_source_archive_ex(
-    OUT_SOURCE_PATH SOURCE_PATH
-    ARCHIVE ${ARCHIVE}
-    REF ${METIS_VERSION}
-    PATCHES
-        enable-install.patch
-        disable-programs.patch
-        fix-runtime-install-destination.patch
-        fix-metis-vs14-math.patch
-        fix-gklib-vs14-math.patch
-        fix-linux-build-error.patch
-        install-metisConfig.patch
-        fix-INT_MIN_define.patch
-)
-
-vcpkg_configure_cmake(
-    SOURCE_PATH ${SOURCE_PATH}
-    PREFER_NINJA
-    OPTIONS ${OPTIONS}
-)
-
-vcpkg_install_cmake()
-vcpkg_copy_pdbs()
-vcpkg_fixup_cmake_targets(CONFIG_PATH lib/cmake/metis)
-
-file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
-
-# Handle copyright
-file(COPY ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/metis)
-file(INSTALL ${SOURCE_PATH}/LICENSE.txt DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)

config/vcpkg/ports/metis-mfem/vcpkg.json

@@ -1,7 +0,0 @@
-{
-  "name": "metis-mfem",
-  "version-string": "5.1.0",
-  "port-version": 0,
-  "description": "Serial Graph Partitioning and Fill-reducing Matrix Ordering",
-  "homepage": "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview"
-}

data/cube-hex-ro.mesh

@@ -0,0 +1,86 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+3
+
+elements
+8
+1 5  0   1   10  9   3   4   13  12 
+1 5  1   2   11  10  4   5   14  13 
+1 5  3   4   13  12  6   7   16  15
+1 5  4   5   14  13  7   8   17  16
+1 5  9   10  19  18  12  13  22  21
+1 5  10  11  20  19  13  14  23  22
+1 5  12  13  22  21  15  16  25  24
+1 5  13  14  23  22  16  17  26  25
+
+#
+
+boundary
+24
+1 3 1  0  9  10 
+1 3 2  1  10 11 
+1 3 10 9  18 19 
+1 3 11 10 19 20 
+2 3 0  3  12 9
+2 3 9  12 21 18
+2 3 3  6  15 12
+2 3 12 15 24 21
+3 3 0  1  4  3
+3 3 1  2  5  4
+3 3 3  4  7  6
+3 3 4  5  8  7 
+4 3 6  7  16 15
+4 3 7  8  17 16
+4 3 15 16 25 24
+4 3 16 17 26 25 
+5 3 18 21 22 19 
+5 3 19 22 23 20 
+5 3 21 24 25 22 
+5 3 22 25 26 23 
+6 3 2  11 14 5
+6 3 11 20 23 14
+6 3 5  14 17 8 
+6 3 14 23 26 17
+
+vertices
+27
+3
+0.0 0.0 0.0
+0.5 0.0 0.0
+1.0 0.0 0.0
+0.0 0.0 0.5
+0.5 0.0 0.5
+1.0 0.0 0.5
+0.0 0.0 1.0
+0.5 0.0 1.0
+1.0 0.0 1.0
+0.0 0.5 0.0
+0.5 0.5 0.0
+1.0 0.5 0.0
+0.0 0.5 0.5
+0.5 0.5 0.5
+1.0 0.5 0.5
+0.0 0.5 1.0
+0.5 0.5 1.0
+1.0 0.5 1.0
+0.0 1.0 0.0
+0.5 1.0 0.0
+1.0 1.0 0.0
+0.0 1.0 0.5
+0.5 1.0 0.5
+1.0 1.0 0.5
+0.0 1.0 1.0
+0.5 1.0 1.0
+1.0 1.0 1.0

data/cube-hex.mesh

@@ -0,0 +1,84 @@
+MFEM mesh v1.0
+
+#
+# MFEM Geometry Types (see mesh/geom.hpp):
+#
+# POINT       = 0
+# SEGMENT     = 1
+# TRIANGLE    = 2
+# SQUARE      = 3
+# TETRAHEDRON = 4
+# CUBE        = 5
+#
+
+dimension
+3
+
+elements
+8
+1 5 0  1  4  3  9  10 13 12 
+1 5 1  2  5  4  10 11 14 13
+1 5 9  10 13 12 18 19 22 21
+1 5 10 11 14 13 19 20 23 22
+1 5 3  4  7  6  12 13 16 15
+1 5 4  5  8  7  13 14 17 16
+1 5 12 13 16 15 21 22 25 24
+1 5 13 14 17 16 22 23 26 25
+
+boundary
+24
+1 3 0  1  10 9 
+1 3 1  2  11 10 
+1 3 9  10 19 18 
+1 3 10 11 20 19 
+3 3 0  3  4  1 
+3 3 1  4  5  2 
+3 3 3  6  7  4 
+3 3 4  7  8  5 
+3 3 18 19 22 21 
+3 3 19 20 23 22 
+3 3 21 22 25 24 
+3 3 22 23 26 25 
+3 3 2  5  14 11 
+3 3 11 14 23 20 
+3 3 5  8  17 14 
+3 3 14 17 26 23 
+3 3 0  9  12 3 
+3 3 9  18 21 12 
+3 3 3  12 15 6 
+3 3 12 21 24 15 
+2 3 6  15 16 7 
+2 3 7  16 17 8 
+2 3 15 24 25 16 
+2 3 16 25 26 17 
+
+vertices
+27
+3
+0.0 0.0 0.0
+0.5 0.0 0.0
+1.0 0.0 0.0
+0.0 0.0 0.5
+0.5 0.0 0.5
+1.0 0.0 0.5
+0.0 0.0 1.0
+0.5 0.0 1.0
+1.0 0.0 1.0
+0.0 0.5 0.0
+0.5 0.5 0.0
+1.0 0.5 0.0
+0.0 0.5 0.5
+0.5 0.5 0.5
+1.0 0.5 0.5
+0.0 0.5 1.0
+0.5 0.5 1.0
+1.0 0.5 1.0
+0.0 1.0 0.0
+0.5 1.0 0.0
+1.0 1.0 0.0
+0.0 1.0 0.5
+0.5 1.0 0.5
+1.0 1.0 0.5
+0.0 1.0 1.0
+0.5 1.0 1.0
+1.0 1.0 1.0

data/tinyzoo-3d.mesh

@@ -1,7 +1,7 @@
 MFEM mesh v1.0
 
 #
-# MFEM Geometry Types (see mesh/geom.hpp):
+# MFEM Geomety Types (see mesh/geom.hpp):
 #
 # POINT       = 0
 # SEGMENT     = 1

doc/CodeDocumentation.conf.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MFEM"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v4.5.1
+PROJECT_NUMBER         = v4.4.1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -763,54 +763,36 @@ INPUT                  = @MFEM_SOURCE_DIR@/doc/CodeDocumentation.dox \
                          @MFEM_SOURCE_DIR@/config \
                          @MFEM_SOURCE_DIR@/general \
                          @MFEM_SOURCE_DIR@/linalg \
-                         @MFEM_SOURCE_DIR@/linalg/simd \
                          @MFEM_SOURCE_DIR@/mesh \
-                         @MFEM_SOURCE_DIR@/mesh/submesh \
                          @MFEM_SOURCE_DIR@/fem \
-                         @MFEM_SOURCE_DIR@/fem/ceed \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/convection \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/diffusion \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/mass \
-                         @MFEM_SOURCE_DIR@/fem/ceed/integrators/nlconvection \
-                         @MFEM_SOURCE_DIR@/fem/ceed/interface \
-                         @MFEM_SOURCE_DIR@/fem/ceed/solvers \
+                         @MFEM_SOURCE_DIR@/fem/moonolith \
                          @MFEM_SOURCE_DIR@/fem/fe \
                          @MFEM_SOURCE_DIR@/fem/lor \
-                         @MFEM_SOURCE_DIR@/fem/moonolith \
-                         @MFEM_SOURCE_DIR@/fem/qinterp \
-                         @MFEM_SOURCE_DIR@/fem/tmop \
                          @MFEM_SOURCE_DIR@/examples \
-                         @MFEM_SOURCE_DIR@/examples/amgx \
                          @MFEM_SOURCE_DIR@/examples/caliper \
+                         @MFEM_SOURCE_DIR@/examples/amgx \
                          @MFEM_SOURCE_DIR@/examples/ginkgo \
-                         @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/moonolith \
+                         @MFEM_SOURCE_DIR@/examples/hiop \
                          @MFEM_SOURCE_DIR@/examples/petsc \
                          @MFEM_SOURCE_DIR@/examples/pumi \
                          @MFEM_SOURCE_DIR@/examples/sundials \
                          @MFEM_SOURCE_DIR@/examples/superlu \
                          @MFEM_SOURCE_DIR@/miniapps/adjoint \
-                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                          @MFEM_SOURCE_DIR@/miniapps/common \
                          @MFEM_SOURCE_DIR@/miniapps/electromagnetics \
                          @MFEM_SOURCE_DIR@/miniapps/gslib \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/kernels \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/materials \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/operators \
-                         @MFEM_SOURCE_DIR@/miniapps/hooke/preconditioners \
                          @MFEM_SOURCE_DIR@/miniapps/meshing \
                          @MFEM_SOURCE_DIR@/miniapps/mtop \
-                         @MFEM_SOURCE_DIR@/miniapps/multidomain \
+                         @MFEM_SOURCE_DIR@/miniapps/autodiff \
                          @MFEM_SOURCE_DIR@/miniapps/navier \
                          @MFEM_SOURCE_DIR@/miniapps/nurbs \
-                         @MFEM_SOURCE_DIR@/miniapps/parelag \
                          @MFEM_SOURCE_DIR@/miniapps/performance \
                          @MFEM_SOURCE_DIR@/miniapps/shifted \
                          @MFEM_SOURCE_DIR@/miniapps/solvers \
                          @MFEM_SOURCE_DIR@/miniapps/tools \
-                         @MFEM_SOURCE_DIR@/miniapps/toys
+                         @MFEM_SOURCE_DIR@/miniapps/toys \
+                         @MFEM_SOURCE_DIR@/miniapps/parelag
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -2367,7 +2349,7 @@ PLANTUML_INCLUDE_PATH  =
 # Minimum value: 0, maximum value: 10000, default value: 50.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_GRAPH_MAX_NODES    = 100
+DOT_GRAPH_MAX_NODES    = 50
 
 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
 # generated by dot. A depth value of 3 means that only nodes reachable from the

examples/caliper/CMakeLists.txt

@@ -33,13 +33,13 @@ add_mfem_examples(CALIPER_EXE_SRCS ${PREFIX})
 if (MFEM_ENABLE_TESTING)
   foreach(SRC_FILE ${CALIPER_EXE_SRCS})
     get_filename_component(SRC_FILENAME ${SRC_FILE} NAME)
-    string(REPLACE ".cpp" "" TEST_NAME ${PREFIX}${SRC_FILENAME})
+    string(REPLACE ".cpp" "" TEST_NAME ${SRC_FILENAME})
 
     set(THIS_TEST_OPTIONS "-no-vis")
 
     if (NOT (${TEST_NAME} MATCHES ".*p$"))
       add_test(NAME ${TEST_NAME}_ser
-        COMMAND $<TARGET_FILE:${TEST_NAME}> ${THIS_TEST_OPTIONS})
+        COMMAND ${TEST_NAME} ${THIS_TEST_OPTIONS})
     else()
       add_test(NAME ${TEST_NAME}_np=${MFEM_MPI_NP}
         COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}

examples/ex1.cpp

@@ -30,7 +30,6 @@
 //
 // Device sample runs:
 //               ex1 -pa -d cuda
-//               ex1 -fa -d cuda
 //               ex1 -pa -d raja-cuda
 //             * ex1 -pa -d raja-hip
 //               ex1 -pa -d occa-cuda
@@ -38,13 +37,9 @@
 //               ex1 -pa -d occa-omp
 //               ex1 -pa -d ceed-cpu
 //               ex1 -pa -d ceed-cpu -o 4 -a
-//               ex1 -pa -d ceed-cpu -m ../data/square-mixed.mesh
-//               ex1 -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
 //             * ex1 -pa -d ceed-cuda
 //             * ex1 -pa -d ceed-hip
 //               ex1 -pa -d ceed-cuda:/gpu/cuda/shared
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
-//               ex1 -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
 //               ex1 -m ../data/beam-hex.mesh -pa -d cuda
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //               ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
@@ -78,7 +73,6 @@ int main(int argc, char *argv[])
    int order = 1;
    bool static_cond = false;
    bool pa = false;
-   bool fa = false;
    const char *device_config = "cpu";
    bool visualization = true;
    bool algebraic_ceed = false;
@@ -93,8 +87,6 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
@@ -192,14 +184,6 @@ int main(int argc, char *argv[])
    //    domain integrator.
    BilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 10. Assemble the bilinear form and the corresponding linear system,

examples/ex1p.cpp

@@ -30,18 +30,13 @@
 //
 // Device sample runs:
 //               mpirun -np 4 ex1p -pa -d cuda
-//               mpirun -np 4 ex1p -fa -d cuda
 //               mpirun -np 4 ex1p -pa -d occa-cuda
 //               mpirun -np 4 ex1p -pa -d raja-omp
 //               mpirun -np 4 ex1p -pa -d ceed-cpu
 //               mpirun -np 4 ex1p -pa -d ceed-cpu -o 4 -a
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/square-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/fichera-mixed.mesh
 //             * mpirun -np 4 ex1p -pa -d ceed-cuda
 //             * mpirun -np 4 ex1p -pa -d ceed-hip
 //               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
-//               mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
 //               mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
 //
 // Description:  This example code demonstrates the use of MFEM to define a
@@ -79,7 +74,6 @@ int main(int argc, char *argv[])
    int order = 1;
    bool static_cond = false;
    bool pa = false;
-   bool fa = false;
    const char *device_config = "cpu";
    bool visualization = true;
    bool algebraic_ceed = false;
@@ -94,8 +88,6 @@ int main(int argc, char *argv[])
                   "--no-static-condensation", "Enable static condensation.");
    args.AddOption(&pa, "-pa", "--partial-assembly", "-no-pa",
                   "--no-partial-assembly", "Enable Partial Assembly.");
-   args.AddOption(&fa, "-fa", "--full-assembly", "-no-fa",
-                  "--no-full-assembly", "Enable Full Assembly.");
    args.AddOption(&device_config, "-d", "--device",
                   "Device configuration string, see Device::Configure().");
 #ifdef MFEM_USE_CEED
@@ -219,14 +211,6 @@ int main(int argc, char *argv[])
    //     Diffusion domain integrator.
    ParBilinearForm a(&fespace);
    if (pa) { a.SetAssemblyLevel(AssemblyLevel::PARTIAL); }
-   if (fa)
-   {
-      a.SetAssemblyLevel(AssemblyLevel::FULL);
-      // Sort the matrix column indices when running on GPU or with OpenMP (i.e.
-      // when Device::IsEnabled() returns true). This makes the results
-      // bit-for-bit deterministic at the cost of somewhat longer run time.
-      a.EnableSparseMatrixSorting(Device::IsEnabled());
-   }
    a.AddDomainIntegrator(new DiffusionIntegrator(one));
 
    // 12. Assemble the parallel bilinear form and the corresponding linear

examples/ex26.cpp

@@ -182,7 +182,7 @@ int main(int argc, char *argv[])
    }
    for (int level = 0; level < order_refinements; ++level)
    {
-      collections.Append(new H1_FECollection((int)std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
       fespaces.AddOrderRefinedLevel(collections.Last());
    }
 

examples/ex26p.cpp

@@ -219,7 +219,7 @@ int main(int argc, char *argv[])
    }
    for (int level = 0; level < order_refinements; ++level)
    {
-      collections.Append(new H1_FECollection((int)std::pow(2, level+1), dim));
+      collections.Append(new H1_FECollection(std::pow(2, level+1), dim));
       fespaces->AddOrderRefinedLevel(collections.Last());
    }
 

examples/ex27p.cpp

@@ -195,7 +195,7 @@ int main(int argc, char *argv[])
    Array<int> ess_tdof_list(0);
    if (h1 && pmesh.bdr_attributes.Size())
    {
-      // For a continuous basis the linear system must be modified to enforce an
+      // For a continuous basis the linear system must be modifed to enforce an
       // essential (Dirichlet) boundary condition. In the DG case this is not
       // necessary as the boundary condition will only be enforced weakly.
       fespace.GetEssentialTrueDofs(dbc_bdr, ess_tdof_list);

examples/ex33.cpp

@@ -3,63 +3,34 @@
 // Compile with: make ex33
 //
 // Sample runs:  ex33 -m ../data/square-disc.mesh -alpha 0.33 -o 2
-//               ex33 -m ../data/square-disc.mesh -alpha 4.5 -o 3
-//               ex33 -m ../data/star.mesh -alpha 1.4 -o 3
 //               ex33 -m ../data/star.mesh -alpha 0.99 -o 3
 //               ex33 -m ../data/inline-quad.mesh -alpha 0.5 -o 3
-//               ex33 -m ../data/amr-quad.mesh -alpha 1.5 -o 3
 //               ex33 -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
-//               ex33 -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
 //               ex33 -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
-//               ex33 -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
-//
-// Verification runs:
-//    ex33 -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
-//    ex33 -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
-//    ex33 -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
-//    ex33 -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
-//
-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
 //
 // Description:
 //
 //  In this example we solve the following fractional PDE with MFEM:
 //
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
 //
-//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
-//  N is given by floor(α). By doing so, we obtain
+//  To solve this FPDE, we rely on a rational approximation [2] of the normal
+//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
+//  boundary conditions). Namely, we first approximate the operator
 //
-//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
-//
-//  We first compute the right hand side by solving the integer order PDE
-//
-//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
-//
-//  The remaining FPDE is then given by
-//
-//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
-//
-//  We rely on a rational approximation [2] of the normal linear operator
-//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
-//  and (a-N) in (0,1). We approximate the operator
-//
-//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
 //
 //  where I is the L2-identity operator and the coefficients c_i and d_i
 //  are generated offline to a prescribed accuracy in a pre-processing step.
 //  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve M+1
+//  that this partial fractional expansion derives from. We then solve N+1
 //  independent integer-order PDEs,
 //
-//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
+//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
 //
 //  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
 //
-//    u ≈ Σ_{i=0}^M u_i.
-//
-//  (If alpha is an integer, we stop after the first PDE was solved.)
+//    u ≈ Σ_{i=0}^N u_i.
 //
 // References:
 //
@@ -76,8 +47,6 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
-#include <math.h>
-#include <string>
 
 #include "ex33.hpp"
 
@@ -90,9 +59,8 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    int num_refs = 3;
-   double alpha = 0.5;
    bool visualization = true;
-   bool verification = false;
+   double alpha = 0.5;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -107,9 +75,6 @@ int main(int argc, char *argv[])
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
                   "Enable or disable GLVis visualization.");
-   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
-                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
    args.Parse();
    if (!args.Good())
    {
@@ -119,31 +84,9 @@ int main(int argc, char *argv[])
    args.PrintOptions(cout);
 
    Array<double> coeffs, poles;
-   int progress_steps = 1;
 
-   // 2. Compute the rational expansion coefficients that define the
-   //    integer-order PDEs.
-   const int power_of_laplace = floor(alpha);
-   double exponent_to_approximate = alpha - power_of_laplace;
-   bool integer_order = false;
-   // Check if alpha is an integer or not.
-   if (abs(exponent_to_approximate) > 1e-12)
-   {
-      mfem::out << "Approximating the fractional exponent "
-                << exponent_to_approximate
-                << endl;
-      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
-                                          poles);
-
-      // If the example is build without LAPACK, the exponent_to_approximate
-      // might be modified by the function call above.
-      alpha = exponent_to_approximate + power_of_laplace;
-   }
-   else
-   {
-      integer_order = true;
-      mfem::out << "Treating integer order PDE." << endl;
-   }
+   // 2. Compute the coefficients that define the integer-order PDEs.
+   ComputePartialFractionApproximation(alpha,coeffs,poles);
 
    // 3. Read the mesh from the given mesh file.
    Mesh mesh(mesh_file, 1, 1);
@@ -156,8 +99,8 @@ int main(int argc, char *argv[])
    }
 
    // 5. Define a finite element space on the mesh.
-   H1_FECollection fec(order, dim);
-   FiniteElementSpace fespace(&mesh, &fec);
+   FiniteElementCollection *fec = new H1_FECollection(order, dim);
+   FiniteElementSpace fespace(&mesh, fec);
    cout << "Number of finite element unknowns: "
         << fespace.GetTrueVSize() << endl;
 
@@ -171,234 +114,79 @@ int main(int argc, char *argv[])
    }
 
    // 7. Define diffusion coefficient, load, and solution GridFunction.
-   auto func = [&alpha](const Vector &x)
-   {
-      double val = 1.0;
-      for (int i=0; i<x.Size(); i++)
-      {
-         val *= sin(M_PI*x(i));
-      }
-      return pow(x.Size()*pow(M_PI,2), alpha) * val;
-   };
-   FunctionCoefficient f(func);
+   ConstantCoefficient f(1.0);
    ConstantCoefficient one(1.0);
    GridFunction u(&fespace);
-   GridFunction x(&fespace);
-   GridFunction g(&fespace);
-   u = 0.0;
-   x = 0.0;
-   g = 0.0;
+   u = 0.;
 
    // 8. Prepare for visualization.
    char vishost[] = "localhost";
    int  visport   = 19916;
-
-   // 9. Set up the linear form b(.) for integer-order PDE solves.
-   LinearForm b(&fespace);
-   if (verification)
+   socketstream xout, uout;
+   ostringstream oss_x, oss_u;
+   if (visualization)
    {
-      // This statement is only relevant for the verification of the code. It
-      // uses a different f such that an analytic solution is known and easy
-      // to compare with the numerical one. The FPDE becomes:
-      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
-      // -> u(x,y) = sin(\pi x) sin(\pi y)
-      b.AddDomainIntegrator(new DomainLFIntegrator(f));
+      xout.open(vishost, visport);
+      xout.precision(8);
+      uout.open(vishost, visport);
+      uout.precision(8);
    }
-   else
+
+   for (int i = 0; i < coeffs.Size(); i++)
    {
-      b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   }
-   b.Assemble();
+      // 9. Set up the linear form b(.) for integer-order PDE solve.
+      LinearForm b(&fespace);
+      ProductCoefficient cf(coeffs[i], f);
+      b.AddDomainIntegrator(new DomainLFIntegrator(cf));
+      b.Assemble();
 
-   // ------------------------------------------------------------------------
-   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
-   // ------------------------------------------------------------------------
+      // 10. Define GridFunction for integer-order PDE solve.
+      GridFunction x(&fespace);
+      x = 0.0;
 
-   if (power_of_laplace > 0)
-   {
-      // 10.1 Compute Stiffnes Matrix
-      BilinearForm k(&fespace);
-      k.AddDomainIntegrator(new DiffusionIntegrator(one));
-      k.Assemble();
+      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
+      BilinearForm a(&fespace);
+      a.AddDomainIntegrator(new DiffusionIntegrator(one));
+      ConstantCoefficient c2(-poles[i]);
+      a.AddDomainIntegrator(new MassIntegrator(c2));
+      a.Assemble();
 
-      // 10.2 Compute Mass Matrix
-      BilinearForm m(&fespace);
-      m.AddDomainIntegrator(new MassIntegrator(one));
-      m.Assemble();
-      SparseMatrix mass;
-      Array<int> empty;
-      m.FormSystemMatrix(empty, mass);
-
-      // 10.3 Form the system of equations
+      // 12. Assemble the bilinear form and the corresponding linear system.
+      OperatorPtr A;
       Vector B, X;
-      OperatorPtr Op;
-      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
-      GSSmoother M((SparseMatrix&)(*Op));
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
 
-      mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
-                << " ( f ) " << endl;
-      for (int i = 0; i < power_of_laplace; i++)
-      {
-         // 10.4 Solve the linear system Op X = B (N times).
-         PCG(*Op, M, B, X, 3, 300, 1e-12, 0.0);
+      // 13. Solve the linear system A X = B.
+      GSSmoother M((SparseMatrix&)(*A));
 
-         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
-         if (i == power_of_laplace - 1)
-         {
-            // Needed for visualization and solution verification.
-            k.RecoverFEMSolution(X, b, g);
-            if (integer_order && verification)
-            {
-               // For an integer order PDE, g is also our solution u.
-               u+=g;
-            }
-            if (visualization)
-            {
-               socketstream fout;
-               ostringstream oss_f;
-               fout.open(vishost, visport);
-               fout.precision(8);
-               oss_f.str(""); oss_f.clear();
-               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
-                     << power_of_laplace
-                     << " g = f";
-               fout << "solution\n" << mesh << g
-                    << "window_title '" << oss_f.str() << "'" << flush;
-            }
-         }
+      mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
+                << " u = " << coeffs[i] << " f " << endl;
+      PCG(*A, M, B, X, 3, 200, 1e-12, 0.0);
 
-         // 10.6 Prepare for next iteration (primal / dual space)
-         mass.Mult(X, B);
-         X.SetSubVectorComplement(ess_tdof_list,0.0);
-      }
+      // 14. Recover the solution as a finite element grid function.
+      a.RecoverFEMSolution(X, b, x);
 
-      // 10.7 Extract solution for the next step. The b now corresponds to the
-      //      function g in the PDE.
-      const SparseMatrix * R = fespace.GetRestrictionMatrix();
-      if (R)
-      {
-         R->MultTranspose(B,b);
-      }
-      else
-      {
-         b = B;
-      }
-   }
+      // 15. Accumulate integer-order PDE solutions.
+      u+=x;
 
-   // ------------------------------------------------------------------------
-   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
-   //     up the solutions.
-   // ------------------------------------------------------------------------
-   if (!integer_order)
-   {
-      // Setup visualization.
-      socketstream xout, uout;
-      ostringstream oss_x, oss_u;
+      // 16. Send the solutions by socket to a GLVis server.
       if (visualization)
       {
-         xout.open(vishost, visport);
-         xout.precision(8);
-         uout.open(vishost, visport);
-         uout.precision(8);
-      }
-      // Iterate over all expansion coefficient that contribute to the
-      // solution.
-      for (int i = 0; i < coeffs.Size(); i++)
-      {
-         mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
-                   << " u = " << coeffs[i] << " g " << endl;
+         oss_x.str(""); oss_x.clear();
+         oss_x << "Solution of PDE -Δ u + " << -poles[i]
+               << " u = " << coeffs[i] << " f";
+         xout << "solution\n" << mesh << x
+              << "window_title '" << oss_x.str() << "'" << flush;
 
-
-         // 11.1 Reset GridFunction for integer-order PDE solve.
-         x = 0.0;
-
-         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
-         BilinearForm a(&fespace);
-         a.AddDomainIntegrator(new DiffusionIntegrator(one));
-         ConstantCoefficient d_i(-poles[i]);
-         a.AddDomainIntegrator(new MassIntegrator(d_i));
-         a.Assemble();
-
-         // 11.3 Assemble the bilinear form and the corresponding linear system.
-         OperatorPtr A;
-         Vector B, X;
-         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-         // 11.4 Solve the linear system A X = B.
-         GSSmoother M((SparseMatrix&)(*A));
-
-         PCG(*A, M, B, X, 3, 300, 1e-12, 0.0);
-
-         // 11.5 Recover the solution as a finite element grid function.
-         a.RecoverFEMSolution(X, b, x);
-
-         // 11.6 Accumulate integer-order PDE solutions.
-         x *= coeffs[i];
-         u += x;
-
-         // 11.7 Send fractional PDE solution to a GLVis server.
-         if (visualization)
-         {
-            oss_x.str(""); oss_x.clear();
-            oss_x << "Step " << progress_steps
-                  << ": Solution of PDE -Δ u + " << -poles[i]
-                  << " u = " << coeffs[i] << " g";
-            xout << "solution\n" << mesh << x
-                 << "window_title '" << oss_x.str() << "'" << flush;
-
-            oss_u.str(""); oss_u.clear();
-            oss_u << "Step " << progress_steps + 1
-                  << ": Solution of fractional PDE (-Δ)^" << alpha
-                  << " u = f";
-            uout << "solution\n" << mesh << u
-                 << "window_title '" << oss_u.str() << "'"
-                 << flush;
-         }
+         oss_u.str(""); oss_u.clear();
+         oss_u << "Solution of fractional PDE -Δ^" << alpha
+               << " u = f";
+         uout << "solution\n" << mesh << u
+              << "window_title '" << oss_u.str() << "'" << flush;
       }
    }
 
-   // ------------------------------------------------------------------------
-   // 12. (optional) Verify the solution.
-   // ------------------------------------------------------------------------
-   if (verification)
-   {
-      auto solution = [] (const Vector &x)
-      {
-         double val = 1.0;
-         for (int i=0; i<x.Size(); i++)
-         {
-            val *= sin(M_PI*x(i));
-         }
-         return val;
-      };
-      FunctionCoefficient sol(solution);
-      double l2_error = u.ComputeL2Error(sol);
-
-      string analytic_solution,expected_mesh;
-      switch (dim)
-      {
-         case 1:
-            analytic_solution = "sin(π x)";
-            expected_mesh = "inline_segment.mesh";
-            break;
-         case 2:
-            analytic_solution = "sin(π x) sin(π y)";
-            expected_mesh = "inline_quad.mesh";
-            break;
-         default:
-            analytic_solution = "sin(π x) sin(π y) sin(π z)";
-            expected_mesh = "inline_hex.mesh";
-            break;
-      }
-
-      mfem::out << "\n" << string(80,'=')
-                << "\n\nSolution Verification in "<< dim << "D \n\n"
-                << "Analytic solution : " << analytic_solution << "\n"
-                << "Expected mesh     : " << expected_mesh <<"\n"
-                << "Your mesh         : " << mesh_file << "\n"
-                << "L2 error          : " << l2_error << "\n\n"
-                << string(80,'=') << endl;
-   }
-
+   // 17. Free the used memory.
+   delete fec;
    return 0;
 }

examples/ex33.hpp

@@ -32,7 +32,6 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
-#include <string>
 
 using namespace std;
 using namespace mfem;
@@ -250,13 +249,6 @@ void PartialFractionExpansion(double scale, Array<double> & poles,
    coeffs.SetSize(psize);
    coeffs = scale;
 
-   // Note: C p(z)/q(z) = Σ_i c_i / (z - p_i) results in an system of equations
-   // where the N unknowns are the coefficients c_i. After multiplying the
-   // system with q(z), the coefficients c_i can be computed analytically by
-   // choosing N values for z. Choosing z_j = = p_j diagonalizes the system and
-   // one can obtain an analytic form for the c_i coefficients. The result is
-   // implemented in the code block below.
-
    for (int i=0; i<psize; i++)
    {
       double tmp_numer=1.0;
@@ -313,12 +305,9 @@ void ComputePartialFractionApproximation(double & alpha,
    if (print_warning)
    {
       mfem::out
-            << "\n" << string(80, '=')
-            << "\nMFEM is compiled without LAPACK."
-            << "\nUsing precomputed values for PartialFractionApproximation."
-            << "\nOnly alpha = 0.33, 0.5, and 0.99 are available."
-            << "\nThe default is alpha = 0.5.\n" << string(80, '=') << "\n"
-            << endl;
+            << "\nMFEM is compiled without LAPACK.\nUsing precomputed values for PartialFractionApproximation. \n"
+            << "Only alpha = 0.33, 0.5, and 0.99 are available.\nThe default is alpha = 0.5."
+            << std::endl;
    }
    const double eps = std::numeric_limits<double>::epsilon();
 
@@ -362,7 +351,7 @@ void ComputePartialFractionApproximation(double & alpha,
 
    if (print_warning)
    {
-      mfem::out << "=> Using precomputed values for alpha = "
+      mfem::out << "Using precomputed values for alpha = "
                 << alpha << "\n" << std::endl;
    }
 

examples/ex33p.cpp

@@ -3,63 +3,34 @@
 // Compile with: make ex33p
 //
 // Sample runs:  mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 0.33 -o 2
-//               mpirun -np 4 ex33p -m ../data/square-disc.mesh -alpha 4.5 -o 3
-//               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 1.4 -o 3
 //               mpirun -np 4 ex33p -m ../data/star.mesh -alpha 0.99 -o 3
 //               mpirun -np 4 ex33p -m ../data/inline-quad.mesh -alpha 0.5 -o 3
-//               mpirun -np 4 ex33p -m ../data/amr-quad.mesh -alpha 1.5 -o 3
-//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3 -r 2
-//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 2.4 -o 3 -r 4
+//               mpirun -np 4 ex33p -m ../data/disc-nurbs.mesh -alpha 0.33 -o 3
 //               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 0.33 -o 3 -r 4
-//               mpirun -np 4 ex33p -m ../data/l-shape.mesh -alpha 1.7 -o 3 -r 5
-//
-// Verification runs:
-//    mpirun -np 4 ex33p -m ../data/inline-segment.mesh -ver -alpha 1.7 -o 2 -r 2
-//    mpirun -np 4 ex33p -m ../data/inline-quad.mesh -ver -alpha 1.2 -o 2 -r 2
-//    mpirun -np 4 ex33p -m ../data/amr-quad.mesh -ver -alpha 2.6 -o 2 -r 2
-//    mpirun -np 4 ex33p -m ../data/inline-hex.mesh -ver -alpha 0.3 -o 2 -r 1
-
-//  Note: the analytic solution to this problem is u = ∏_{i=0}^{dim-1} sin(π x_i)
-//        for all alpha.
 //
 // Description:
 //
 //  In this example we solve the following fractional PDE with MFEM:
 //
-//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α,
+//    ( - Δ )^α u = f  in Ω,      u = 0  on ∂Ω,      0 < α < 1,
 //
-//  To solve this FPDE, we apply the operator ( - Δ )^(-N), where the integer
-//  N is given by floor(α). By doing so, we obtain
+//  To solve this FPDE, we rely on a rational approximation [2] of the normal
+//  linear operator A^{-α}, where A = - Δ (with associated homogeneous
+//  boundary conditions). Namely, we first approximate the operator
 //
-//    ( - Δ )^(α-N) u = ( - Δ )^(-N) f  in Ω,      u = 0  on ∂Ω,      0 < α.
-//
-//  We first compute the right hand side by solving the integer order PDE
-//
-//   ( - Δ )^N g = f  in Ω, g = ( - Δ )^k g = 0 on ∂Ω, k = 1,..,N-1
-//
-//  The remaining FPDE is then given by
-//
-//  ( - Δ )^(α-N) u = g  in Ω,      u = 0  on ∂Ω.
-//
-//  We rely on a rational approximation [2] of the normal linear operator
-//  A^{-α + N}, where A = - Δ (with associated homogeneous boundary conditions)
-//  and (a-N) in (0,1). We approximate the operator
-//
-//    A^{-α+N} ≈ Σ_{i=0}^M c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
+//    A^{-α} ≈ Σ_{i=0}^N c_i (A + d_i I)^{-1},      d_0 = 0,   d_i > 0,
 //
 //  where I is the L2-identity operator and the coefficients c_i and d_i
 //  are generated offline to a prescribed accuracy in a pre-processing step.
 //  We use the triple-A algorithm [1] to generate the rational approximation
-//  that this partial fractional expansion derives from. We then solve M+1
+//  that this partial fractional expansion derives from. We then solve N+1
 //  independent integer-order PDEs,
 //
-//    A u_i + d_i u_i = c_i g  in Ω,      u_i = 0  on ∂Ω,      i=0,...,M,
+//    A u_i + d_i u_i = c_i f  in Ω,      u_i = 0  on ∂Ω,      i=0,...,N,
 //
 //  using MFEM and sum u_i to arrive at an approximate solution of the FPDE
 //
-//    u ≈ Σ_{i=0}^M u_i.
-//
-//  (If alpha is an integer, we stop after the first PDE was solved.)
+//    u ≈ Σ_{i=0}^N u_i.
 //
 // References:
 //
@@ -76,8 +47,6 @@
 #include "mfem.hpp"
 #include <fstream>
 #include <iostream>
-#include <math.h>
-#include <string>
 
 #include "ex33.hpp"
 
@@ -96,9 +65,9 @@ int main(int argc, char *argv[])
    const char *mesh_file = "../data/star.mesh";
    int order = 1;
    int num_refs = 3;
-   double alpha = 0.5;
    bool visualization = true;
-   bool verification = false;
+   bool visualize_x = false;
+   double alpha = 0.5;
 
    OptionsParser args(argc, argv);
    args.AddOption(&mesh_file, "-m", "--mesh",
@@ -110,12 +79,12 @@ int main(int argc, char *argv[])
                   "Number of uniform refinements");
    args.AddOption(&alpha, "-alpha", "--alpha",
                   "Fractional exponent");
+   args.AddOption(&visualize_x, "-vis_x", "--visualize_x", "-no-vis_x",
+                  "--no-visualization_x",
+                  "Enable or disable GLVis visualization of each integer-order PDE solution.");
    args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
                   "--no-visualization",
-                  "Enable or disable GLVis visualization.");
-   args.AddOption(&verification, "-ver", "--verification", "-no-ver",
-                  "--no-verification",
-                  "Use sinusoidal function (f) for analytic comparison.");
+                  "Enable or disable GLVis visualization of the fractional PDE solution.");
    args.Parse();
    if (!args.Good())
    {
@@ -128,51 +97,61 @@ int main(int argc, char *argv[])
    }
 
    Array<double> coeffs, poles;
-   int progress_steps = 1;
 
-   // 2. Compute the rational expansion coefficients that define the
-   //    integer-order PDEs.
-   const int power_of_laplace = floor(alpha);
-   double exponent_to_approximate = alpha - power_of_laplace;
-   bool integer_order = false;
-   // Check if alpha is an integer or not.
-   if (abs(exponent_to_approximate) > 1e-12)
-   {
-      if (Mpi::Root())
-      {
-         mfem::out << "Approximating the fractional exponent "
-                   << exponent_to_approximate
-                   << endl;
-      }
-      ComputePartialFractionApproximation(exponent_to_approximate, coeffs,
-                                          poles);
+   // 2. Compute the coefficients that define the integer-order PDEs.
+   ComputePartialFractionApproximation(alpha,coeffs,poles);
 
-      // If the example is build without LAPACK, the exponent_to_approximate
-      // might be modified by the function call above.
-      alpha = exponent_to_approximate + power_of_laplace;
-   }
-   else
+   int num_par_solves;
+   int max_par_solves = max(1,num_procs/2);
+   for (num_par_solves=max_par_solves; num_par_solves>0; num_par_solves--)
    {
-      integer_order = true;
-      if (Mpi::Root())
+      if (num_procs%num_par_solves==0 && num_par_solves<coeffs.Size())
       {
-         mfem::out << "Treating integer order PDE." << endl;
+         break;
       }
    }
+   if (num_par_solves == 1) {num_par_solves = num_procs;}
 
-   // 3. Read the mesh from the given mesh file.
+   int solver_ranks = num_procs/num_par_solves;
+
+   // 3. Split the MPI communicator:
+   //    row_comm is used for parallel partition of the mesh
+   //    col_comm is used for independent integer-order solves
+   int row_color = myid / solver_ranks; // Determine color based on row
+   int col_color = myid % solver_ranks; // Determine color based on col
+
+   MPI_Comm row_comm, col_comm;
+   MPI_Comm_split(MPI_COMM_WORLD, row_color, myid, &row_comm);
+   MPI_Comm_split(MPI_COMM_WORLD, col_color, myid, &col_comm);
+
+   int row_rank, row_size, col_rank, col_size;
+   MPI_Comm_rank(row_comm, &row_rank);
+   MPI_Comm_size(row_comm, &row_size);
+   MPI_Comm_rank(col_comm, &col_rank);
+   MPI_Comm_size(col_comm, &col_size);
+
+   if (Mpi::Root())
+   {
+      mfem::out << "\nTotal number of MPI ranks = " << num_procs << endl;
+      mfem::out << "Number of independent parallel solves = " << col_size << endl;
+      mfem::out << "Number of MPI ranks within each solve = " << row_size
+                <<"\n" << endl;
+   }
+
+   // 4. Read the mesh from the given mesh file.
    Mesh mesh(mesh_file, 1, 1);
    int dim = mesh.Dimension();
 
-   // 4. Refine the mesh to increase the resolution.
+   // 5. Refine the mesh to increase the resolution.
    for (int i = 0; i < num_refs; i++)
    {
       mesh.UniformRefinement();
    }
-   ParMesh pmesh(MPI_COMM_WORLD, mesh);
+
+   ParMesh pmesh(row_comm, mesh);
    mesh.Clear();
 
-   // 5. Define a finite element space on the mesh.
+   // 6. Define a finite element space on the mesh.
    H1_FECollection fec(order, dim);
    ParFiniteElementSpace fespace(&pmesh, &fec);
    if (Mpi::Root())
@@ -181,7 +160,7 @@ int main(int argc, char *argv[])
            << fespace.GetTrueVSize() << endl;
    }
 
-   // 6. Determine the list of true (i.e. conforming) essential boundary dofs.
+   // 7. Determine the list of true (i.e. conforming) essential boundary dofs.
    Array<int> ess_tdof_list;
    if (pmesh.bdr_attributes.Size())
    {
@@ -190,250 +169,120 @@ int main(int argc, char *argv[])
       fespace.GetEssentialTrueDofs(ess_bdr, ess_tdof_list);
    }
 
-   // 7. Define diffusion coefficient, load, and solution GridFunction.
-   auto func = [&alpha](const Vector &x)
-   {
-      double val = 1.0;
-      for (int i=0; i<x.Size(); i++)
-      {
-         val *= sin(M_PI*x(i));
-      }
-      return pow(x.Size()*pow(M_PI,2), alpha) * val;
-   };
-   FunctionCoefficient f(func);
+   // 8. Define diffusion coefficient, load, and solution GridFunction.
+   ConstantCoefficient f(1.0);
    ConstantCoefficient one(1.0);
    ParGridFunction u(&fespace);
    ParGridFunction x(&fespace);
-   ParGridFunction g(&fespace);
    u = 0.0;
-   x = 0.0;
-   g = 0.0;
-
-   // 8. Prepare for visualization.
-   char vishost[] = "localhost";
-   int  visport   = 19916;
 
    // 9. Set up the linear form b(.) for integer-order PDE solves.
    ParLinearForm b(&fespace);
-   if (verification)
-   {
-      // This statement is only relevant for the verification of the code. It
-      // uses a different f such that an analytic solution is known and easy
-      // to compare with the numerical one. The FPDE becomes:
-      // (-Δ)^α u = (2\pi ^2)^α sin(\pi x) sin(\pi y) on [0,1]^2
-      // -> u(x,y) = sin(\pi x) sin(\pi y)
-      b.AddDomainIntegrator(new DomainLFIntegrator(f));
-   }
-   else
-   {
-      b.AddDomainIntegrator(new DomainLFIntegrator(one));
-   }
+   b.AddDomainIntegrator(new DomainLFIntegrator(f));
    b.Assemble();
 
-   // ------------------------------------------------------------------------
-   // 10. Solve the PDE (-Δ)^N g = f, i.e. compute g = (-Δ)^{-1}^N f.
-   // ------------------------------------------------------------------------
-
-   if (power_of_laplace > 0)
+   int my_coeff_size = max(coeffs.Size()/col_size,1);
+   int ibeg = col_rank*my_coeff_size;
+   if (ibeg + 2*my_coeff_size > coeffs.Size())
    {
-      // 10.1 Compute Stiffnes Matrix
-      ParBilinearForm k(&fespace);
-      k.AddDomainIntegrator(new DiffusionIntegrator(one));
-      k.Assemble();
+      my_coeff_size = coeffs.Size()-col_rank*my_coeff_size;
+   }
+   else if (ibeg > coeffs.Size() - 1)
+   {
+      my_coeff_size = 0;
+   }
 
-      // 10.2 Compute Mass Matrix
-      ParBilinearForm m(&fespace);
-      m.AddDomainIntegrator(new MassIntegrator(one));
-      m.Assemble();
-      HypreParMatrix mass;
-      Array<int> empty;
-      m.FormSystemMatrix(empty, mass);
+   int iend = ibeg+my_coeff_size;
 
-      // 10.3 Form the system of equations
+
+   for (int i = ibeg; i < iend; i++)
+   {
+      // 10. Reset GridFunction for integer-order PDE solve.
+      x = 0.0;
+
+      // 11. Set up the bilinear form a(.,.) for integer-order PDE solve.
+      ParBilinearForm a(&fespace);
+      a.AddDomainIntegrator(new DiffusionIntegrator(one));
+      ConstantCoefficient d_i(-poles[i]);
+      a.AddDomainIntegrator(new MassIntegrator(d_i));
+      a.Assemble();
+
+      // 12. Assemble the bilinear form and the corresponding linear system.
+      OperatorPtr A;
       Vector B, X;
-      OperatorPtr Op;
-      k.FormLinearSystem(ess_tdof_list, g, b, Op, X, B);
-      HypreBoomerAMG prec;
-      prec.SetPrintLevel(-1);
-      CGSolver cg(MPI_COMM_WORLD);
+      a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
+
+      // 13. Solve the linear system A X = B.
+      HypreBoomerAMG * prec = new HypreBoomerAMG;
+      prec->SetPrintLevel(-1);
+
+      int print_level = (col_rank==0) ? 3 : 0;
+      if (Mpi::Root())
+      {
+         mfem::out << "\nMPI rank " <<  myid
+                   << ": Solving PDE -Δ u + " << -poles[i]
+                   << " u = " << coeffs[i] << " f " << endl;
+      }
+      CGSolver cg(row_comm);
       cg.SetRelTol(1e-12);
       cg.SetMaxIter(2000);
-      cg.SetPrintLevel(3);
-      cg.SetPreconditioner(prec);
-      cg.SetOperator(*Op);
+      cg.SetPrintLevel(print_level);
+      cg.SetPreconditioner(*prec);
+      cg.SetOperator(*A);
+      cg.Mult(B, X);
+      delete prec;
 
-      if (Mpi::Root())
+      // 14. Recover the solution as a finite element grid function.
+      a.RecoverFEMSolution(X, b, x);
+
+      // 15. Accumulate integer-order PDE solutions.
+      x *= coeffs[i];
+      u += x;
+
+      // 16. Send integer-order PDE solutions to a GLVis server.
+      if (visualize_x)
       {
-         mfem::out << "\nComputing (-Δ) ^ -" << power_of_laplace
-                   << " ( f ) " << endl;
-      }
-      for (int i = 0; i < power_of_laplace; i++)
-      {
-         // 10.4 Solve the linear system Op X = B (N times).
-         cg.Mult(B, X);
-         // 10.5 Visualize the solution g of -Δ ^ N g = f in the last step
-         if (i == power_of_laplace - 1)
+         if (col_rank > 0 && i < iend-1)
          {
-            // Needed for visualization and solution verification.
-            k.RecoverFEMSolution(X, b, g);
-            if (integer_order && verification)
-            {
-               // For an integer order PDE, g is also our solution u.
-               u+=g;
-            }
-            if (visualization)
-            {
-               socketstream fout;
-               ostringstream oss_f;
-               fout.open(vishost, visport);
-               fout.precision(8);
-               oss_f.str(""); oss_f.clear();
-               oss_f << "Step " << progress_steps++ << ": Solution of PDE -Δ ^ "
-                     << power_of_laplace
-                     << " g = f";
-               fout << "parallel " << num_procs << " " << myid << "\n"
-                    << "solution\n" << pmesh << g
-                    << "window_title '" << oss_f.str() << "'" << flush;
-            }
+            MPI_Status status;
+            MPI_Recv(nullptr,0,MPI_INT, col_rank-1,0,col_comm,&status);
          }
-
-         // 10.6 Prepare for next iteration (primal / dual space)
-         mass.Mult(X, B);
-         X.SetSubVectorComplement(ess_tdof_list,0.0);
-      }
-
-      // 10.7 Extract solution for the next step. The b now corresponds to the
-      //      function g in the PDE.
-      const SparseMatrix* rm = fespace.GetRestrictionMatrix();
-      rm->MultTranspose(B, b);
-   }
-
-   // ------------------------------------------------------------------------
-   // 11. Solve the fractional PDE by solving M integer order PDEs and adding
-   //     up the solutions.
-   // ------------------------------------------------------------------------
-   if (!integer_order)
-   {
-      // Setup visualization.
-      socketstream xout, uout;
-      ostringstream oss_x, oss_u;
-      if (visualization)
-      {
-         xout.open(vishost, visport);
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream xout(vishost, visport);
          xout.precision(8);
-         uout.open(vishost, visport);
-         uout.precision(8);
-      }
-      // Iterate over all expansion coefficient that contribute to the
-      // solution.
-      for (int i = 0; i < coeffs.Size(); i++)
-      {
-         if (Mpi::Root())
+         ostringstream oss;
+         oss << "Solution of PDE -Δ u + " << -poles[i]
+             << " u = " << coeffs[i] << " f" ;
+         xout << "parallel " << row_size << " " << row_rank << "\n";
+         xout << "solution\n" << pmesh << x
+              << "window_title '" << oss.str() << "'" << flush;
+         if (col_rank < col_size-1)
          {
-            mfem::out << "\nSolving PDE -Δ u + " << -poles[i]
-                      << " u = " << coeffs[i] << " g " << endl;
-         }
-
-         // 11.1 Reset GridFunction for integer-order PDE solve.
-         x = 0.0;
-
-         // 11.2 Set up the bilinear form a(.,.) for integer-order PDE solve.
-         ParBilinearForm a(&fespace);
-         a.AddDomainIntegrator(new DiffusionIntegrator(one));
-         ConstantCoefficient d_i(-poles[i]);
-         a.AddDomainIntegrator(new MassIntegrator(d_i));
-         a.Assemble();
-
-         // 11.3 Assemble the bilinear form and the corresponding linear system.
-         OperatorPtr A;
-         Vector B, X;
-         a.FormLinearSystem(ess_tdof_list, x, b, A, X, B);
-
-         // 11.4 Solve the linear system A X = B.
-         HypreBoomerAMG prec;
-         prec.SetPrintLevel(-1);
-
-         CGSolver cg(MPI_COMM_WORLD);
-         cg.SetRelTol(1e-12);
-         cg.SetMaxIter(2000);
-         cg.SetPrintLevel(3);
-         cg.SetPreconditioner(prec);
-         cg.SetOperator(*A);
-         cg.Mult(B, X);
-
-         // 11.5 Recover the solution as a finite element grid function.
-         a.RecoverFEMSolution(X, b, x);
-
-         // 11.6 Accumulate integer-order PDE solutions.
-         x *= coeffs[i];
-         u += x;
-
-         // 11.7 Send fractional PDE solution to a GLVis server.
-         if (visualization)
-         {
-            oss_x.str(""); oss_x.clear();
-            oss_x << "Step " << progress_steps
-                  << ": Solution of PDE -Δ u + " << -poles[i]
-                  << " u = " << coeffs[i] << " g";
-            xout << "parallel " << num_procs << " " << myid << "\n"
-                 << "solution\n" << pmesh << x
-                 << "window_title '" << oss_x.str() << "'" << flush;
-
-            oss_u.str(""); oss_u.clear();
-            oss_u << "Step " << progress_steps + 1
-                  << ": Solution of fractional PDE (-Δ)^" << alpha
-                  << " u = f";
-            uout << "parallel " << num_procs << " " << myid << "\n"
-                 << "solution\n" << pmesh << u
-                 << "window_title '" << oss_u.str() << "'"
-                 << flush;
+            MPI_Send(nullptr,0,MPI_INT,col_rank+1,0,col_comm);
          }
       }
    }
 
-   // ------------------------------------------------------------------------
-   // 12. (optional) Verify the solution.
-   // ------------------------------------------------------------------------
-   if (verification)
+   // 17. Accumulate for the fractional PDE solution
+   MPI_Allreduce(MPI_IN_PLACE, u.GetData(), u.Size(),
+                 MPI_DOUBLE, MPI_SUM,col_comm);
+
+   // 18. Send fractional PDE solution to a GLVis server.
+   if (visualization)
    {
-      auto solution = [] (const Vector &x)
+      if (col_rank == 0)
       {
-         double val = 1.0;
-         for (int i=0; i<x.Size(); i++)
-         {
-            val *= sin(M_PI*x(i));
-         }
-         return val;
-      };
-      FunctionCoefficient sol(solution);
-      double l2_error = u.ComputeL2Error(sol);
-
-      if (Mpi::Root())
-      {
-         string analytic_solution,expected_mesh;
-         switch (dim)
-         {
-            case 1:
-               analytic_solution = "sin(π x)";
-               expected_mesh = "inline_segment.mesh";
-               break;
-            case 2:
-               analytic_solution = "sin(π x) sin(π y)";
-               expected_mesh = "inline_quad.mesh";
-               break;
-            default:
-               analytic_solution = "sin(π x) sin(π y) sin(π z)";
-               expected_mesh = "inline_hex.mesh";
-               break;
-         }
-
-         mfem::out << "\n" << string(80,'=')
-                   << "\n\nSolution Verification in "<< dim << "D \n\n"
-                   << "Analytic solution : " << analytic_solution << "\n"
-                   << "Expected mesh     : " << expected_mesh <<"\n"
-                   << "Your mesh         : " << mesh_file << "\n"
-                   << "L2 error          : " << l2_error << "\n\n"
-                   << string(80,'=') << endl;
+         char vishost[] = "localhost";
+         int  visport   = 19916;
+         socketstream uout(vishost, visport);
+         uout.precision(8);
+         ostringstream oss;
+         oss << "Solution of fractional PDE -Δ^" << alpha
+             << " u = f" ;
+         uout << "parallel " << row_size << " " << row_rank << "\n";
+         uout << "solution\n" << pmesh << u
+              << "window_title '" << oss.str() << "'" << flush;
       }
    }
 

examples/ex5.cpp

@@ -197,6 +197,7 @@ int main(int argc, char *argv[])
       SparseMatrix &M(mVarf->SpMat());
       SparseMatrix &B(bVarf->SpMat());
       B *= -1.;
+      B.EnsureMultTranspose();
       Bt = new TransposeOperator(&B);
 
       darcyOp.SetBlock(0,0, &M);

examples/petsc/ex10p.cpp

@@ -187,6 +187,7 @@ int main(int argc, char *argv[])
 {
    // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
+   int num_procs = Mpi::WorldSize();
    int myid = Mpi::WorldRank();
    Hypre::Init();
 

examples/petsc/ex2p.cpp

@@ -248,10 +248,7 @@ int main(int argc, char *argv[])
    //     constraints for non-conforming AMR, static condensation, etc.
    if (myid == 0) { cout << "matrix ... " << flush; }
    if (static_cond) { a->EnableStaticCondensation(); }
-   // Here we want to try out block-size aware AMG solver in PETSc.
-   // For that to work properly, we need a fully-compliant block-size
-   // structure and we do not skip zeros when assembling.
-   a->Assemble(use_petsc ? 0 : 1);
+   a->Assemble();
 
    Vector B, X;
    if (!use_petsc)
@@ -297,14 +294,13 @@ int main(int argc, char *argv[])
          cout << "done." << endl;
          cout << "Size of linear system: " << A.M() << endl;
       }
-      // Tell PETSc the matrix has a block structure
-      A.SetBlockSize(dim);
-
-      // The preconditioner for the PCG solver can be specified in the
-      // PETSc config file
       PetscPCGSolver *pcg = new PetscPCGSolver(A);
+
+      // The preconditioner for the PCG solver defined below is specified in the
+      // PETSc config file, rc_ex2p, since a Krylov solver in PETSc can also
+      // customize its preconditioner.
       PetscPreconditioner *prec = NULL;
-      if (use_nonoverlapping) // Specialized BDDC construction
+      if (use_nonoverlapping)
       {
          // Compute dofs belonging to the natural boundary
          Array<int> nat_tdof_list, nat_bdr(pmesh->bdr_attributes.Max());

examples/petsc/ex9p.cpp

@@ -450,7 +450,7 @@ int main(int argc, char *argv[])
       for (int ti = 0; !done; )
       {
          // We cannot match exactly the time history of the Run method
-         // since we are explicitly telling PETSc to use a time step
+         // since we are explictly telling PETSc to use a time step
          double dt_real = min(dt, t_final - t);
          ode_solver->Step(*U, t, dt_real);
          ti++;

examples/petsc/makefile

@@ -78,7 +78,6 @@ EX1_ARGS_CUDA         := -m ../../data/star.mesh --usepetsc --partial-assembly -
 EX1_ARGS_CUDAAMG      := -m ../../data/star.mesh --usepetsc --device cuda --petscopts rc_ex1p_cudaamg
 EX2_ARGS              := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p
 EX2_ARGS_BDDC         := -m ../../data/beam-tri.mesh --usepetsc --nonoverlapping --petscopts rc_ex2p_bddc
-EX2_ARGS_ASM          := -m ../../data/beam-quad.mesh --usepetsc --petscopts rc_ex2p_asm
 EX3_ARGS              := -m ../../data/klein-bottle.mesh -o 2 -f 0.1 --usepetsc --petscopts rc_ex3p_bddc --nonoverlapping
 EX4_ARGS              := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping
 EX4_HYB_ARGS          := -m ../../data/klein-bottle.mesh -o 2 --usepetsc --petscopts rc_ex4p_bddc --nonoverlapping --hybridization
@@ -110,7 +109,6 @@ endif
 ex2p-test-par: ex2p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS))
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_BDDC))
-	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX2_ARGS_ASM))
 ex3p-test-par: ex3p
 	@$(call mfem-test,$<, $(RUN_MPI), $(TESTNAME),$(EX3_ARGS))
 ex4p-test-par: ex4p

examples/petsc/rc_ex1p

@@ -14,5 +14,4 @@
 -mg_levels_esteig_ksp_type cg
 -mg_levels_esteig_ksp_max_it 10
 -mg_levels_ksp_chebyshev_esteig 0,0.05,0,1.05
--pc_gamg_use_sa_esteig 0
 -mg_levels_pc_type sor

examples/petsc/rc_ex2p

@@ -1,7 +1,8 @@
 -ksp_converged_reason
 
 # GAMG is still not used at its best,
-# since we are not exploiting the RBMs
+# since we are not exploiting the
+# block size (Ordering::byVDIM) and the RBMs
 
 -ksp_view
 -pc_type gamg

examples/petsc/rc_ex2p_asm

@@ -1,10 +0,0 @@
-# Additive Schwarz with Overlap
-# This is not a good solver for elasticity
-# These options are here only to describe
-# the setup of the solver
--ksp_converged_reason
--ksp_view
--ksp_max_it 10
--pc_type asm
--pc_asm_overlap 1
--sub_pc_type icc

examples/sundials/ex10.cpp

@@ -210,9 +210,6 @@ void visualize(ostream &os, Mesh *mesh, GridFunction *deformed_nodes,
 
 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/beam-quad.mesh";
    int ref_levels = 2;

examples/sundials/ex10p.cpp

@@ -215,11 +215,10 @@ void visualize(ostream &os, ParMesh *mesh, ParGridFunction *deformed_nodes,
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
    int myid = Mpi::WorldRank();
    Hypre::Init();
-   Sundials::Init();
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/beam-quad.mesh";

examples/sundials/ex16.cpp

@@ -109,9 +109,6 @@ double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
    // 1. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
    int ref_levels = 2;
@@ -293,10 +290,7 @@ int main(int argc, char *argv[])
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
          ode_solver = arkode; break;
       case 12:
          arkode = new ARKStepSolver(ARKStepSolver::IMPLICIT);

examples/sundials/ex16p.cpp

@@ -101,12 +101,11 @@ double InitialTemperature(const Vector &x);
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
    int num_procs = Mpi::WorldSize();
    int myid = Mpi::WorldRank();
    Hypre::Init();
-   Sundials::Init();
 
    // 2. Parse command-line options.
    const char *mesh_file = "../../data/star.mesh";
@@ -328,10 +327,7 @@ int main(int argc, char *argv[])
          arkode->Init(oper);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 11)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 11) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
          ode_solver = arkode; break;
       case 12:
          arkode = new ARKStepSolver(MPI_COMM_WORLD, ARKStepSolver::IMPLICIT);

examples/sundials/ex9.cpp

@@ -140,9 +140,6 @@ public:
 
 int main(int argc, char *argv[])
 {
-   // 0. Initialize SUNDIALS.
-   Sundials::Init();
-
    // 1. Parse command-line options.
    problem = 0;
    const char *mesh_file = "../../data/periodic-hexagon.mesh";
@@ -411,7 +408,7 @@ int main(int argc, char *argv[])
          arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
+         arkode->SetERKTableNum(FEHLBERG_13_7_8);
          ode_solver = arkode; break;
    }
 

examples/sundials/ex9p.cpp

@@ -152,12 +152,11 @@ public:
 
 int main(int argc, char *argv[])
 {
-   // 1. Initialize MPI, HYPRE, and SUNDIALS.
+   // 1. Initialize MPI and HYPRE.
    Mpi::Init(argc, argv);
    int num_procs = Mpi::WorldSize();
    int myid = Mpi::WorldRank();
    Hypre::Init();
-   Sundials::Init();
 
    // 2. Parse command-line options.
    problem = 0;
@@ -488,10 +487,7 @@ int main(int argc, char *argv[])
          arkode->Init(adv);
          arkode->SetSStolerances(reltol, abstol);
          arkode->SetMaxStep(dt);
-         if (ode_solver_type == 9)
-         {
-            arkode->SetERKTableNum(ARKODE_FEHLBERG_13_7_8);
-         }
+         if (ode_solver_type == 9) { arkode->SetERKTableNum(FEHLBERG_13_7_8); }
          ode_solver = arkode; break;
    }
 

examples/superlu/CMakeLists.txt

@@ -35,7 +35,7 @@ add_mfem_examples(SUPERLU_EXAMPLES_SRCS ${PFX} "" test_superlu)
 if (MFEM_ENABLE_TESTING)
   # Command line options for the tests.
   # Example 1: Test SuperLU on the simple Poisson problem
-  set(EX1_COMMON_OPTS -m ../../data/star.mesh)
+  set(EX1_COMMON_OPTS -m ../../data/star.mesh -p 2)
   set(EX1P_TEST_OPTS ${EX1_COMMON_OPTS})
 
   # Add the tests: one test per source file.

fem/CMakeLists.txt

@@ -39,7 +39,6 @@ set(SRCS
   complex_fem.cpp
   convergence.cpp
   datacollection.cpp
-  dgmassinv.cpp
   doftrans.cpp
   eltrans.cpp
   estimators.cpp
@@ -73,7 +72,6 @@ set(SRCS
   linearform.cpp
   linearform_ext.cpp
   lininteg.cpp
-  lininteg_boundary.cpp
   lininteg_domain.cpp
   lininteg_domain_grad.cpp
   lor/lor.cpp
@@ -90,7 +88,6 @@ set(SRCS
   fespacehierarchy.cpp
   nonlininteg_vectorconvection.cpp
   nonlininteg_vectorconvection_mf.cpp
-  qfunction.cpp
   qinterp/det.cpp
   qinterp/eval_by_nodes.cpp
   qinterp/eval_by_vdim.cpp
@@ -98,7 +95,6 @@ set(SRCS
   qinterp/grad_by_vdim.cpp
   qinterp/grad_phys_by_nodes.cpp
   qinterp/grad_phys_by_vdim.cpp
-  qspace.cpp
   quadinterpolator.cpp
   quadinterpolator_face.cpp
   restriction.cpp
@@ -140,13 +136,10 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
-  bilininteg_mass_pa.hpp
   coefficient.hpp
   complex_fem.hpp
   convergence.hpp
   datacollection.hpp
-  dgmassinv.hpp
-  dgmassinv_kernels.hpp
   doftrans.hpp
   eltrans.hpp
   estimators.hpp
@@ -196,11 +189,9 @@ set(HDRS
   nonlinearform.hpp
   nonlinearform_ext.hpp
   nonlininteg.hpp
-  qfunction.hpp
   qinterp/dispatch.hpp
   qinterp/eval.hpp
   qinterp/grad.hpp
-  qspace.hpp
   quadinterpolator.hpp
   quadinterpolator_face.hpp
   restriction.hpp

fem/bilinearform.cpp

@@ -124,7 +124,6 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
       case AssemblyLevel::LEGACY:
          break;
       case AssemblyLevel::FULL:
-         SetDiagonalPolicy( DIAG_ONE ); // Only diagonal policy supported on device
          ext = new FABilinearFormExtension(this);
          break;
       case AssemblyLevel::ELEMENT:
@@ -137,7 +136,7 @@ void BilinearForm::SetAssemblyLevel(AssemblyLevel assembly_level)
          ext = new MFBilinearFormExtension(this);
          break;
       default:
-         MFEM_ABORT("BilinearForm: unknown assembly level");
+         mfem_error("Unknown assembly level");
    }
 }
 
@@ -993,7 +992,6 @@ void BilinearForm::EliminateVDofs(const Array<int> &vdofs_,
       mat_e = new SparseMatrix(height);
    }
 
-   vdofs_.HostRead();
    for (int i = 0; i < vdofs_.Size(); i++)
    {
       int vdof = vdofs_[i];

fem/bilinearform.hpp

@@ -26,8 +26,7 @@ namespace mfem
 {
 
 /** @brief Enumeration defining the assembly level for bilinear and nonlinear
-    form classes derived from Operator. For more details, see
-    https://mfem.org/howto/assembly_levels */
+    form classes derived from Operator. */
 enum class AssemblyLevel
 {
    /// In the case of a BilinearForm LEGACY corresponds to a fully assembled
@@ -80,9 +79,6 @@ protected:
    /** @brief Extension for supporting Full Assembly (FA), Element Assembly (EA),
        Partial Assembly (PA), or Matrix Free assembly (MF). */
    BilinearFormExtension *ext;
-   /** Indicates if the sparse matrix is sorted after assembly when using
-       Full Assembly (FA). */
-   bool sort_sparse_matrix = false;
 
    /** @brief Indicates the Mesh::sequence corresponding to the current state of
        the BilinearForm. */
@@ -181,24 +177,9 @@ public:
        - AssemblyLevel::ELEMENT
        - AssemblyLevel::NONE
 
-       If used, this method must be called before assembly. */
+       This method must be called before assembly. */
    void SetAssemblyLevel(AssemblyLevel assembly_level);
 
-   /** @brief Force the sparse matrix column indices to be sorted when using
-       AssemblyLevel::FULL.
-
-       When assembling on device the assembly algorithm uses atomic operations
-       to insert values in the sparse matrix, which can result in different
-       column index orderings across runs. Calling this method with @a enable_it
-       set to @a true forces a sorting algorithm to be called at the end of the
-       assembly procedure to ensure sorted column indices (and therefore
-       deterministic results).
-   */
-   void EnableSparseMatrixSorting(bool enable_it)
-   {
-      sort_sparse_matrix = enable_it;
-   }
-
    /// Returns the assembly level
    AssemblyLevel GetAssemblyLevel() const { return assembly; }
 
@@ -324,66 +305,38 @@ public:
    /// Finalizes the matrix initialization.
    virtual void Finalize(int skip_zeros = 1);
 
-   /** @brief Returns a const reference to the sparse matrix:  \f$ M \f$
-
-       This will fail if HasSpMat() is false. */
+   /// Returns a const reference to the sparse matrix.
    const SparseMatrix &SpMat() const
    {
       MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
       return *mat;
    }
 
-   /** @brief Returns a reference to the sparse matrix:  \f$ M \f$
-
-       This will fail if HasSpMat() is false. */
+   /// Returns a reference to the sparse matrix:  \f$ M \f$
    SparseMatrix &SpMat()
    {
       MFEM_VERIFY(mat, "mat is NULL and can't be dereferenced");
       return *mat;
    }
 
-   /** @brief Returns true if the sparse matrix is not null, false otherwise.
-
-       @sa SpMat(). */
-   bool HasSpMat()
-   {
-      return mat != nullptr;
-   }
-
-
    /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transferring ownership. */
+        to it.  Used for transfering ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
-   /** @brief Returns a const reference to the sparse matrix of eliminated b.c.:
-       \f$ M_e \f$
-
-       This will fail if HasSpMatElim() is false. */
+   /// Returns a const reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
    const SparseMatrix &SpMatElim() const
    {
       MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
       return *mat_e;
    }
 
-   /** @brief Returns a reference to the sparse matrix of eliminated b.c.:
-       \f$ M_e \f$
-
-       This will fail if HasSpMatElim() is false. */
+   /// Returns a reference to the sparse matrix of eliminated b.c.: \f$ M_e \f$
    SparseMatrix &SpMatElim()
    {
       MFEM_VERIFY(mat_e, "mat_e is NULL and can't be dereferenced");
       return *mat_e;
    }
 
-   /**  @brief Returns true if the sparse matrix of eliminated b.c.s is not null,
-        false otherwise.
-
-        @sa SpMatElim(). */
-   bool HasSpMatElim()
-   {
-      return mat_e != nullptr;
-   }
-
    /// Adds new Domain Integrator. Assumes ownership of @a bfi.
    void AddDomainIntegrator(BilinearFormIntegrator *bfi);
    /// Adds new Domain Integrator restricted to certain elements specified by
@@ -457,14 +410,6 @@ public:
    virtual const Operator *GetOutputRestriction() const
    { return GetRestriction(); }
 
-   /// @brief Compute serial RAP operator and store it in @a A as a SparseMatrix.
-   void SerialRAP(OperatorHandle &A)
-   {
-      MFEM_ASSERT(mat, "SerialRAP requires the SparseMatrix to be assembled.");
-      ConformingAssemble();
-      A.Reset(mat, false);
-   }
-
    /** @brief Form the linear system A X = B, corresponding to this bilinear
        form and the linear form @a b(.). */
    /** This method applies any necessary transformations to the linear system
@@ -793,7 +738,7 @@ public:
    SparseMatrix &SpMat() { return *mat; }
 
    /**  @brief Nullifies the internal matrix \f$ M \f$ and returns a pointer
-        to it.  Used for transferring ownership. */
+        to it.  Used for transfering ownership. */
    SparseMatrix *LoseMat() { SparseMatrix *tmp = mat; mat = NULL; return tmp; }
 
    /// Adds a domain integrator. Assumes ownership of @a bfi.

fem/bilinearform_ext.cpp

@@ -160,7 +160,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultMF(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
       }
    }
 
@@ -176,7 +176,7 @@ void MFBilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultMF(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
       }
    }
 }
@@ -217,7 +217,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultTransposeMF(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
       }
    }
 
@@ -233,7 +233,7 @@ void MFBilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultTransposeMF(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
       }
    }
 }
@@ -251,7 +251,6 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
 
 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 {
-   if ( Device::Allows(Backend::CEED_MASK) ) { return; }
    ElementDofOrdering ordering = UsesTensorBasis(*a->FESpace())?
                                  ElementDofOrdering::LEXICOGRAPHIC:
                                  ElementDofOrdering::NATIVE;
@@ -418,7 +417,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultPA(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
       }
    }
 
@@ -434,7 +433,7 @@ void PABilinearFormExtension::Mult(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultPA(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
       }
    }
 }
@@ -475,7 +474,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             intFaceIntegrators[i]->AddMultTransposePA(int_face_X, int_face_Y);
          }
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
       }
    }
 
@@ -491,7 +490,7 @@ void PABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
          {
             bdrFaceIntegrators[i]->AddMultTransposePA(bdr_face_X, bdr_face_Y);
          }
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
       }
    }
 }
@@ -668,7 +667,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             Y(j, 0, f) += res;
          });
          // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
       }
    }
 
@@ -699,7 +698,7 @@ void EABilinearFormExtension::Mult(const Vector &x, Vector &y) const
             Y(j, f) += res;
          });
          // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
       }
    }
 }
@@ -796,7 +795,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             Y(j, 0, f) += res;
          });
          // Apply the Interior Face Restriction transposed
-         int_face_restrict_lex->AddMultTransposeInPlace(int_face_Y, y);
+         int_face_restrict_lex->AddMultTranspose(int_face_Y, y);
       }
    }
 
@@ -827,7 +826,7 @@ void EABilinearFormExtension::MultTranspose(const Vector &x, Vector &y) const
             Y(j, f) += res;
          });
          // Apply the Boundary Face Restriction transposed
-         bdr_face_restrict_lex->AddMultTransposeInPlace(bdr_face_Y, y);
+         bdr_face_restrict_lex->AddMultTranspose(bdr_face_Y, y);
       }
    }
 }
@@ -955,64 +954,6 @@ void FABilinearFormExtension::Assemble()
       }
       a->mat = mat;
    }
-   if ( a->sort_sparse_matrix )
-   {
-      a->mat->SortColumnIndices();
-   }
-}
-
-
-void FABilinearFormExtension::RAP(OperatorHandle &A)
-{
-#ifdef MFEM_USE_MPI
-   if ( auto pa = dynamic_cast<ParBilinearForm*>(a) )
-   {
-      pa->ParallelRAP(*pa->mat, A);
-   }
-   else
-#endif
-   {
-      a->SerialRAP(A);
-   }
-}
-
-void FABilinearFormExtension::EliminateBC(const Array<int> &ess_dofs,
-                                          OperatorHandle &A)
-{
-   MFEM_VERIFY(a->diag_policy == DiagonalPolicy::DIAG_ONE,
-               "Only DiagonalPolicy::DIAG_ONE supported with"
-               " FABilinearFormExtension.");
-#ifdef MFEM_USE_MPI
-   if ( dynamic_cast<ParBilinearForm*>(a) )
-   {
-      A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
-                                          DiagonalPolicy::DIAG_ONE);
-   }
-   else
-#endif
-   {
-      A.As<SparseMatrix>()->EliminateBC(ess_dofs,
-                                        DiagonalPolicy::DIAG_ONE);
-   }
-}
-
-void FABilinearFormExtension::FormSystemMatrix(const Array<int> &ess_dofs,
-                                               OperatorHandle &A)
-{
-   RAP(A);
-   EliminateBC(ess_dofs, A);
-}
-
-void FABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
-                                               Vector &x, Vector &b,
-                                               OperatorHandle &A,
-                                               Vector &X, Vector &B,
-                                               int copy_interior)
-{
-   Operator *A_out;
-   Operator::FormLinearSystem(ess_tdof_list, x, b, A_out, X, B, copy_interior);
-   delete A_out;
-   FormSystemMatrix(ess_tdof_list, A);
 }
 
 void FABilinearFormExtension::DGMult(const Vector &x, Vector &y) const

fem/bilinearform_ext.hpp

@@ -125,15 +125,6 @@ public:
    FABilinearFormExtension(BilinearForm *form);
 
    void Assemble();
-   void RAP(OperatorHandle &A);
-   /** @note Always does `DIAG_ONE` policy to be consistent with
-       `Operator::FormConstrainedSystemOperator`. */
-   void EliminateBC(const Array<int> &ess_dofs, OperatorHandle &A);
-   void FormSystemMatrix(const Array<int> &ess_tdof_list, OperatorHandle &A);
-   void FormLinearSystem(const Array<int> &ess_tdof_list,
-                         Vector &x, Vector &b,
-                         OperatorHandle &A, Vector &X, Vector &B,
-                         int copy_interior = 0);
    void Mult(const Vector &x, Vector &y) const;
    void MultTranspose(const Vector &x, Vector &y) const;
 

fem/bilininteg.cpp

@@ -2003,83 +2003,6 @@ void CurlCurlIntegrator::AssembleElementMatrix
    }
 }
 
-void CurlCurlIntegrator::AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                                const FiniteElement &test_fe,
-                                                ElementTransformation &Trans,
-                                                DenseMatrix &elmat)
-{
-   int tr_nd = trial_fe.GetDof();
-   int te_nd = test_fe.GetDof();
-   dim = trial_fe.GetDim();
-   int dimc = trial_fe.GetCurlDim();
-   double w;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector D;
-   DenseMatrix curlshape(tr_nd,dimc), curlshape_dFt(tr_nd,dimc), M;
-   DenseMatrix te_curlshape(te_nd,dimc), te_curlshape_dFt(te_nd,dimc);
-#else
-   curlshape.SetSize(tr_nd,dimc);
-   curlshape_dFt.SetSize(tr_nd,dimc);
-   te_curlshape.SetSize(te_nd,dimc);
-   te_curlshape_dFt.SetSize(te_nd,dimc);
-#endif
-   elmat.SetSize(te_nd, tr_nd);
-
-   if (MQ) { M.SetSize(dimc); }
-   if (DQ) { D.SetSize(dimc); }
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order;
-      if (trial_fe.Space() == FunctionSpace::Pk)
-      {
-         order = test_fe.GetOrder() + trial_fe.GetOrder() - 2;
-      }
-      else
-      {
-         order = test_fe.GetOrder() + trial_fe.GetOrder() + trial_fe.GetDim() - 1;
-      }
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-   }
-
-   elmat = 0.0;
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Trans.SetIntPoint(&ip);
-
-      w = ip.weight * Trans.Weight();
-      trial_fe.CalcPhysCurlShape(Trans, curlshape_dFt);
-      test_fe.CalcPhysCurlShape(Trans, te_curlshape_dFt);
-
-      if (MQ)
-      {
-         MQ->Eval(M, Trans, ip);
-         M *= w;
-         Mult(te_curlshape_dFt, M, te_curlshape);
-         AddMultABt(te_curlshape, curlshape_dFt, elmat);
-      }
-      else if (DQ)
-      {
-         DQ->Eval(D, Trans, ip);
-         D *= w;
-         AddMultADBt(te_curlshape_dFt,D,curlshape_dFt,elmat);
-      }
-      else
-      {
-         if (Q)
-         {
-            w *= Q->Eval(Trans, ip);
-         }
-         curlshape_dFt *= w;
-         AddMultABt(te_curlshape_dFt, curlshape_dFt, elmat);
-      }
-   }
-}
-
 void CurlCurlIntegrator
 ::ComputeElementFlux(const FiniteElement &el, ElementTransformation &Trans,
                      Vector &u, const FiniteElement &fluxelem, Vector &flux,
@@ -2317,84 +2240,6 @@ double VectorCurlCurlIntegrator::GetElementEnergy(
    return 0.5 * energy;
 }
 
-void MixedCurlIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe, const FiniteElement &test_fe,
-   ElementTransformation &Trans, DenseMatrix &elmat)
-{
-   int dim = trial_fe.GetDim();
-   int trial_dof = trial_fe.GetDof();
-   int test_dof = test_fe.GetDof();
-   int dimc = (dim == 3) ? 3 : 1;
-
-   MFEM_VERIFY(trial_fe.GetMapType() == mfem::FiniteElement::H_CURL ||
-               (dim == 2 && trial_fe.GetMapType() == mfem::FiniteElement::VALUE),
-               "Trial finite element must be either 2D/3D H(Curl) or 2D H1");
-   MFEM_VERIFY(test_fe.GetMapType() == mfem::FiniteElement::VALUE ||
-               test_fe.GetMapType() == mfem::FiniteElement::INTEGRAL,
-               "Test finite element must be in H1/L2");
-
-   bool spaceH1 = (trial_fe.GetMapType() == mfem::FiniteElement::VALUE);
-
-   if (spaceH1)
-   {
-      dshape.SetSize(trial_dof,dim);
-      curlshape.SetSize(trial_dof,dim);
-      dimc = dim;
-   }
-   else
-   {
-      curlshape.SetSize(trial_dof,dimc);
-      elmat_comp.SetSize(test_dof, trial_dof);
-   }
-   elmat.SetSize(dimc * test_dof, trial_dof);
-   shape.SetSize(test_dof);
-   elmat = 0.0;
-
-   double c;
-   Vector d_col;
-   const IntegrationRule *ir = IntRule;
-
-   if (ir == NULL)
-   {
-      int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderJ();
-      ir = &IntRules.Get(trial_fe.GetGeomType(), order);
-   }
-
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-      Trans.SetIntPoint(&ip);
-      if (spaceH1)
-      {
-         trial_fe.CalcPhysDShape(Trans, dshape);
-         dshape.GradToVectorCurl2D(curlshape);
-      }
-      else
-      {
-         trial_fe.CalcPhysCurlShape(Trans, curlshape);
-      }
-      test_fe.CalcPhysShape(Trans, shape);
-      c = ip.weight*Trans.Weight();
-      if (Q)
-      {
-         c *= Q->Eval(Trans, ip);
-      }
-      shape *= c;
-
-      for (int d = 0; d < dimc; ++d)
-      {
-         double * curldata = &(curlshape.GetData())[d*trial_dof];
-         for (int jj = 0; jj < trial_dof; ++jj)
-         {
-            for (int ii = 0; ii < test_dof; ++ii)
-            {
-               elmat(d * test_dof + ii, jj) += shape(ii) * curldata[jj];
-            }
-         }
-      }
-   }
-}
-
 
 void VectorFEMassIntegrator::AssembleElementMatrix(
    const FiniteElement &el,
@@ -2741,54 +2586,6 @@ void DivDivIntegrator::AssembleElementMatrix(
    }
 }
 
-void DivDivIntegrator::AssembleElementMatrix2(
-   const FiniteElement &trial_fe,
-   const FiniteElement &test_fe,
-   ElementTransformation &Trans,
-   DenseMatrix &elmat)
-{
-   int tr_nd = trial_fe.GetDof();
-   int te_nd = test_fe.GetDof();
-   double c;
-
-#ifdef MFEM_THREAD_SAFE
-   Vector divshape(tr_nd);
-   Vector te_divshape(te_nd);
-#else
-   divshape.SetSize(tr_nd);
-   te_divshape.SetSize(te_nd);
-#endif
-   elmat.SetSize(te_nd,tr_nd);
-
-   const IntegrationRule *ir = IntRule;
-   if (ir == NULL)
-   {
-      int order = 2 * max(test_fe.GetOrder(),
-                          trial_fe.GetOrder()) - 2; // <--- OK for RTk
-      ir = &IntRules.Get(test_fe.GetGeomType(), order);
-   }
-
-   elmat = 0.0;
-
-   for (int i = 0; i < ir -> GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      trial_fe.CalcDivShape(ip,divshape);
-      test_fe.CalcDivShape(ip,te_divshape);
-
-      Trans.SetIntPoint (&ip);
-      c = ip.weight / Trans.Weight();
-
-      if (Q)
-      {
-         c *= Q -> Eval (Trans, ip);
-      }
-
-      te_divshape *= c;
-      AddMultVWt(te_divshape, divshape, elmat);
-   }
-}
 
 void VectorDiffusionIntegrator::AssembleElementMatrix(
    const FiniteElement &el,
@@ -3983,7 +3780,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
       for (i = 0; i < ndof1; i++)
          for (j = 0; j < face_ndof; j++)
          {
-            elmat(i, j) += shape1_n(i) * face_shape(j);
+            elmat(i, j) -= shape1_n(i) * face_shape(j);
          }
       if (ndof2)
       {
@@ -3991,7 +3788,7 @@ void NormalTraceJumpIntegrator::AssembleFaceMatrix(
          for (i = 0; i < ndof2; i++)
             for (j = 0; j < face_ndof; j++)
             {
-               elmat(ndof1+i, j) -= shape2_n(i) * face_shape(j);
+               elmat(ndof1+i, j) += shape2_n(i) * face_shape(j);
             }
       }
    }

fem/bilininteg.hpp

@@ -215,10 +215,10 @@ public:
                              function by any coefficients describing the
                              integrator.
        @param[in] ir  If passed (the default value is NULL), the implementation
-                      of the method will ignore the integration rule provided
-                      by the @a fluxelem parameter and, instead, compute the
-                      discrete flux at the points specified by the integration
-                      rule @a ir.
+                        of the method will ignore the integration rule provided
+                        by the @a fluxelem parameter and, instead, compute the
+                        discrete flux at the points specified by the integration
+                        rule @a ir.
     */
    virtual void ComputeElementFlux(const FiniteElement &el,
                                    ElementTransformation &Trans,
@@ -2174,7 +2174,6 @@ public:
 /** Class for local mass matrix assembling a(u,v) := (Q u, v) */
 class MassIntegrator: public BilinearFormIntegrator
 {
-   friend class DGMassInverse;
 protected:
 #ifndef MFEM_THREAD_SAFE
    Vector shape, te_shape;
@@ -2525,7 +2524,6 @@ private:
 #ifndef MFEM_THREAD_SAFE
    Vector D;
    DenseMatrix curlshape, curlshape_dFt, M;
-   DenseMatrix te_curlshape, te_curlshape_dFt;
    DenseMatrix vshape, projcurl;
 #endif
 
@@ -2559,11 +2557,6 @@ public:
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
 
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-
    virtual void ComputeElementFlux(const FiniteElement &el,
                                    ElementTransformation &Trans,
                                    Vector &u, const FiniteElement &fluxelem,
@@ -2609,35 +2602,6 @@ public:
                                    const Vector &elfun);
 };
 
-/** Class for integrating the bilinear form a(u,v) := (Q curl u, v) where Q is
-    an optional scalar coefficient, and v is a vector with components v_i in
-    the L2 or H1 space. This integrator handles 3 cases:
-    (a) u ∈ H(curl) in 3D, v is a 3D vector with components v_i in L^2 or H^1
-    (b) u ∈ H(curl) in 2D, v is a scalar field in L^2 or H^1
-    (c) u is a scalar field in H^1, i.e, curl u := [0 1;-1 0]grad u and v is a
-        2D vector field with components v_i in L^2 or H^1 space.
-    Note: Case (b) can also be handled by MixedScalarCurlIntegrator  */
-class MixedCurlIntegrator : public BilinearFormIntegrator
-{
-protected:
-   Coefficient *Q;
-
-private:
-   Vector shape;
-   DenseMatrix dshape;
-   DenseMatrix curlshape;
-   DenseMatrix elmat_comp;
-public:
-   MixedCurlIntegrator() : Q{NULL} { }
-   MixedCurlIntegrator(Coefficient *q_) :  Q{q_} { }
-   MixedCurlIntegrator(Coefficient &q) :  Q{&q} { }
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-};
-
 /** Integrator for (Q u, v), where Q is an optional coefficient (of type scalar,
     vector (diagonal matrix), or matrix), trial function u is in H(Curl) or
     H(Div), and test function v is in H(Curl), H(Div), or v=(v1,...,vn), where
@@ -2761,7 +2725,7 @@ protected:
 
 private:
 #ifndef MFEM_THREAD_SAFE
-   Vector divshape, te_divshape;
+   Vector divshape;
 #endif
 
    // PA extension
@@ -2773,18 +2737,11 @@ private:
 
 public:
    DivDivIntegrator() { Q = NULL; }
-   DivDivIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
-      BilinearFormIntegrator(ir), Q(&q) { }
+   DivDivIntegrator(Coefficient &q) : Q(&q) { }
 
    virtual void AssembleElementMatrix(const FiniteElement &el,
                                       ElementTransformation &Trans,
                                       DenseMatrix &elmat);
-
-   virtual void AssembleElementMatrix2(const FiniteElement &trial_fe,
-                                       const FiniteElement &test_fe,
-                                       ElementTransformation &Trans,
-                                       DenseMatrix &elmat);
-
    const Coefficient *GetCoefficient() const { return Q; }
 };
 
@@ -3059,8 +3016,8 @@ public:
 
 /** Integrator for the DG form:
 
-        - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
-        + kappa < {h^{-1} Q} [u], [v] >
+    - < {(Q grad(u)).n}, [v] > + sigma < [u], {(Q grad(v)).n} >
+    + kappa < {h^{-1} Q} [u], [v] >,
 
     where Q is a scalar or matrix diffusion coefficient and u, v are the trial
     and test spaces, respectively. The parameters sigma and kappa determine the

fem/bilininteg_convection_mf.cpp

@@ -30,16 +30,7 @@ void ConvectionIntegrator::AssembleMF(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedMFConvectionIntegrator(*this, fes, Q, alpha);
-      }
-      else
-      {
-         ceedOp = new ceed::MFConvectionIntegrator(fes, *ir, Q, alpha);
-      }
+      ceedOp = new ceed::MFConvectionIntegrator(fes, *ir, Q, alpha);
       return;
    }
    MFEM_ABORT("Error: ConvectionIntegrator::AssembleMF only implemented with"

fem/bilininteg_convection_pa.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/convection/convection.hpp"
 #include "quadinterpolator.hpp"
 
@@ -1387,16 +1386,7 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPAConvectionIntegrator(*this, fes, Q, alpha);
-      }
-      else
-      {
-         ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
-      }
+      ceedOp = new ceed::PAConvectionIntegrator(fes, *ir, Q, alpha);
       return;
    }
    const int dims = el.GetDim();
@@ -1409,10 +1399,66 @@ void ConvectionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * ne, mt);
+   Vector vel;
+   if (VectorConstantCoefficient *cQ =
+          dynamic_cast<VectorConstantCoefficient*>(Q))
+   {
+      vel = cQ->GetVec();
+   }
+   else if (VectorGridFunctionCoefficient *vgfQ =
+               dynamic_cast<VectorGridFunctionCoefficient*>(Q))
+   {
+      vel.SetSize(dim * nq * ne, mt);
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector vel(*Q, qs, CoefficientStorage::COMPRESSED);
+      const GridFunction *gf = vgfQ->GetGridFunction();
+      const FiniteElementSpace &gf_fes = *gf->FESpace();
+      const QuadratureInterpolator *qi(gf_fes.GetQuadratureInterpolator(*ir));
+      const bool use_tensor_products = UsesTensorBasis(gf_fes);
+      const ElementDofOrdering ordering = use_tensor_products ?
+                                          ElementDofOrdering::LEXICOGRAPHIC :
+                                          ElementDofOrdering::NATIVE;
+      const Operator *R = gf_fes.GetElementRestriction(ordering);
 
+      Vector xe(R->Height(), mt);
+      xe.UseDevice(true);
+
+      R->Mult(*gf, xe);
+      qi->SetOutputLayout(QVectorLayout::byVDIM);
+      qi->DisableTensorProducts(!use_tensor_products);
+      qi->Values(xe,vel);
+   }
+   else if (VectorQuadratureFunctionCoefficient* vqfQ =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = vqfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+
+      qFun.Read();
+      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      vel.SetSize(dim * nq * ne);
+      auto C = Reshape(vel.HostWrite(), dim, nq, ne);
+      DenseMatrix MQ_ir;
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         Q->Eval(MQ_ir, T, *ir);
+         for (int q = 0; q < nq; ++q)
+         {
+            for (int i = 0; i < dim; ++i)
+            {
+               C(i,q,e) = MQ_ir(i,q);
+            }
+         }
+      }
+   }
    PAConvectionSetup(dim, nq, ne, ir->GetWeights(), geom->J,
                      vel, alpha, pa_data);
 }
@@ -1451,7 +1497,6 @@ static void PAConvectionApply(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x22: return SmemPAConvectionApply3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
          case 0x23: return SmemPAConvectionApply3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
          case 0x24: return SmemPAConvectionApply3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
          case 0x26: return SmemPAConvectionApply3D<2,6>(NE,B,G,Bt,Gt,op,x,y);
@@ -1503,7 +1548,6 @@ static void PAConvectionApplyT(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x22: return SmemPAConvectionApplyT3D<2,2>(NE,B,G,Bt,Gt,op,x,y);
          case 0x23: return SmemPAConvectionApplyT3D<2,3>(NE,B,G,Bt,Gt,op,x,y);
          case 0x24: return SmemPAConvectionApplyT3D<2,4>(NE,B,G,Bt,Gt,op,x,y);
          case 0x26: return SmemPAConvectionApplyT3D<2,6>(NE,B,G,Bt,Gt,op,x,y);

fem/bilininteg_dgtrace_pa.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "restriction.hpp"
 
 using namespace std;
@@ -137,9 +136,6 @@ static void PADGTraceSetup(const int dim,
 
 void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
 {
-   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
-                         Device::GetDeviceMemoryType() : pa_mt;
-
    nf = fes.GetNFbyType(type);
    if (nf==0) { return; }
    // Assumes tensor-product elements
@@ -157,29 +153,93 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
    geom = mesh->GetFaceGeometricFactors(
              *ir,
              FaceGeometricFactors::DETERMINANTS |
-             FaceGeometricFactors::NORMALS, type, mt);
+             FaceGeometricFactors::NORMALS, type);
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(symmDims * nq * nf, Device::GetMemoryType());
-
-   FaceQuadratureSpace qs(*mesh, *ir, type);
-   CoefficientVector vel(*u, qs, CoefficientStorage::COMPRESSED);
-
-   CoefficientVector r(qs, CoefficientStorage::COMPRESSED);
-   if (rho == nullptr)
+   Vector vel;
+   if (VectorConstantCoefficient *c_u = dynamic_cast<VectorConstantCoefficient*>
+                                        (u))
    {
-      r.SetConstant(1.0);
+      vel = c_u->GetVec();
    }
-   else if (ConstantCoefficient *const_rho = dynamic_cast<ConstantCoefficient*>
-                                             (rho))
+   else if (VectorQuadratureFunctionCoefficient* qf_u =
+               dynamic_cast<VectorQuadratureFunctionCoefficient*>(u))
    {
-      r.SetConstant(const_rho->constant);
+      // Assumed to be in lexicographical ordering
+      const QuadratureFunction &qFun = qf_u->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == dim * nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      vel.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      vel.SetSize(dim * nq * nf);
+      auto C = Reshape(vel.HostWrite(), dim, nq, nf);
+      Vector Vq(dim);
+      int f_ind = 0;
+      for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
+      {
+         Mesh::FaceInformation face = mesh->GetFaceInformation(f);
+         if (face.IsNonconformingCoarse())
+         {
+            // We skip nonconforming coarse faces as they are treated
+            // by the corresponding nonconforming fine faces.
+            continue;
+         }
+         else if ( face.IsOfFaceType(type) )
+         {
+            const int mask = FaceElementTransformations::HAVE_ELEM1 |
+                             FaceElementTransformations::HAVE_LOC1;
+            FaceElementTransformations &T =
+               *fes.GetMesh()->GetFaceElementTransformations(f, mask);
+            for (int q = 0; q < nq; ++q)
+            {
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
+               T.SetAllIntPoints(&ir->IntPoint(q));
+               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+               u->Eval(Vq, *T.Elem1, eip1);
+               for (int i = 0; i < dim; ++i)
+               {
+                  C(i,iq,f_ind) = Vq(i);
+               }
+            }
+            f_ind++;
+         }
+      }
+      MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
+   }
+   Vector r;
+   if (rho==nullptr)
+   {
+      r.SetSize(1);
+      r(0) = 1.0;
+   }
+   else if (ConstantCoefficient *c_rho = dynamic_cast<ConstantCoefficient*>(rho))
+   {
+      r.SetSize(1);
+      r(0) = c_rho->constant;
    }
    else if (QuadratureFunctionCoefficient* qf_rho =
                dynamic_cast<QuadratureFunctionCoefficient*>(rho))
    {
-      r.MakeRef(qf_rho->GetQuadFunction());
+      const QuadratureFunction &qFun = qf_rho->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * nf,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      r.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
    }
    else
    {
@@ -191,42 +251,45 @@ void DGTraceIntegrator::SetupPA(const FiniteElementSpace &fes, FaceType type)
       for (int f = 0; f < mesh->GetNumFacesWithGhost(); ++f)
       {
          Mesh::FaceInformation face = mesh->GetFaceInformation(f);
-         if (face.IsNonconformingCoarse() || !face.IsOfFaceType(type))
+         if (face.IsNonconformingCoarse())
          {
             // We skip nonconforming coarse faces as they are treated
             // by the corresponding nonconforming fine faces.
             continue;
          }
-         FaceElementTransformations &T =
-            *fes.GetMesh()->GetFaceElementTransformations(f);
-         for (int q = 0; q < nq; ++q)
+         else if ( face.IsOfFaceType(type) )
          {
-            // Convert to lexicographic ordering
-            int iq = ToLexOrdering(dim, face.element[0].local_face_id,
-                                   quad1D, q);
-
-            T.SetAllIntPoints(&ir->IntPoint(q));
-            const IntegrationPoint &eip1 = T.GetElement1IntPoint();
-            const IntegrationPoint &eip2 = T.GetElement2IntPoint();
-            double rq;
-
-            if (face.IsBoundary())
+            FaceElementTransformations &T =
+               *fes.GetMesh()->GetFaceElementTransformations(f);
+            for (int q = 0; q < nq; ++q)
             {
-               rq = rho->Eval(*T.Elem1, eip1);
-            }
-            else
-            {
-               double udotn = 0.0;
-               for (int d=0; d<dim; ++d)
+               // Convert to lexicographic ordering
+               int iq = ToLexOrdering(dim, face.element[0].local_face_id,
+                                      quad1D, q);
+
+               T.SetAllIntPoints(&ir->IntPoint(q));
+               const IntegrationPoint &eip1 = T.GetElement1IntPoint();
+               const IntegrationPoint &eip2 = T.GetElement2IntPoint();
+               double rq;
+
+               if ( face.IsBoundary() )
                {
-                  udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+                  rq = rho->Eval(*T.Elem1, eip1);
                }
-               if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
-               else { rq = rho->Eval(*T.Elem1, eip1); }
+               else
+               {
+                  double udotn = 0.0;
+                  for (int d=0; d<dim; ++d)
+                  {
+                     udotn += C_vel(d,iq,f_ind)*n(iq,d,f_ind);
+                  }
+                  if (udotn >= 0.0) { rq = rho->Eval(*T.Elem2, eip2); }
+                  else { rq = rho->Eval(*T.Elem1, eip1); }
+               }
+               C(iq,f_ind) = rq;
             }
-            C(iq,f_ind) = rq;
+            f_ind++;
          }
-         f_ind++;
       }
       MFEM_VERIFY(f_ind==nf, "Incorrect number of faces.");
    }
@@ -632,7 +695,6 @@ static void PADGTraceApply(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x22: return SmemPADGTraceApply3D<2,2,1>(NF,B,Bt,op,x,y);
          case 0x23: return SmemPADGTraceApply3D<2,3,1>(NF,B,Bt,op,x,y);
          case 0x34: return SmemPADGTraceApply3D<3,4,2>(NF,B,Bt,op,x,y);
          case 0x45: return SmemPADGTraceApply3D<4,5,2>(NF,B,Bt,op,x,y);
@@ -1062,7 +1124,6 @@ static void PADGTraceApplyTranspose(const int dim,
    {
       switch ((D1D << 4 ) | Q1D)
       {
-         case 0x22: return SmemPADGTraceApplyTranspose3D<2,2>(NF,B,Bt,op,x,y);
          case 0x23: return SmemPADGTraceApplyTranspose3D<2,3>(NF,B,Bt,op,x,y);
          case 0x34: return SmemPADGTraceApplyTranspose3D<3,4>(NF,B,Bt,op,x,y);
          case 0x45: return SmemPADGTraceApplyTranspose3D<4,5>(NF,B,Bt,op,x,y);

fem/bilininteg_diffusion_mf.cpp

@@ -33,16 +33,7 @@ void DiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
       MFEM_VERIFY(!VQ && !MQ,
                   "Only scalar coefficient supported for DiffusionIntegrator"
                   " with libCEED");
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedMFDiffusionIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
       return;
    }
    MFEM_ABORT("Error: DiffusionIntegrator::AssembleMF only implemented with"

fem/bilininteg_diffusion_pa.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"
 
 using namespace std;
@@ -272,21 +271,18 @@ void PADiffusionSetup3D(const int Q1D,
                   D(qx,qy,qz,1,e) = D12; // 1,2
                   D(qx,qy,qz,2,e) = w_detJ * (A11*R13 + A12*R23 + A13*R33); // 1,3
 
+                  const double D21 = w_detJ * (A21*R11 + A22*R21 + A23*R31);
                   const double D22 = w_detJ * (A21*R12 + A22*R22 + A23*R32);
                   const double D23 = w_detJ * (A21*R13 + A22*R23 + A23*R33);
 
                   const double D33 = w_detJ * (A31*R13 + A32*R23 + A33*R33);
 
+                  D(qx,qy,qz,3,e) = symmetric ? D22 : D21; // 2,2 or 2,1
                   D(qx,qy,qz,4,e) = symmetric ? D23 : D22; // 2,3 or 2,2
                   D(qx,qy,qz,5,e) = symmetric ? D33 : D23; // 3,3 or 2,3
 
-                  if (symmetric)
+                  if (!symmetric)
                   {
-                     D(qx,qy,qz,3,e) = D22; // 2,2
-                  }
-                  else
-                  {
-                     D(qx,qy,qz,3,e) = w_detJ * (A21*R11 + A22*R21 + A23*R31); // 2,1
                      D(qx,qy,qz,6,e) = w_detJ * (A31*R11 + A32*R21 + A33*R31); // 3,1
                      D(qx,qy,qz,7,e) = w_detJ * (A31*R12 + A32*R22 + A33*R32); // 3,2
                      D(qx,qy,qz,8,e) = D33; // 3,3
@@ -369,16 +365,7 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
       MFEM_VERIFY(!VQ && !MQ,
                   "Only scalar coefficient supported for DiffusionIntegrator"
                   " with libCEED");
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
       return;
    }
    const int dims = el.GetDim();
@@ -391,21 +378,120 @@ void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    maps = &el.GetDofToQuad(*ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
+   int coeffDim = 1;
+   Vector coeff;
+   const int MQfullDim = MQ ? MQ->GetHeight() * MQ->GetWidth() : 0;
+   if (auto *SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ))
+   {
+      MFEM_VERIFY(SMQ->GetSize() == dim, "");
+      coeffDim = symmDims;
+      coeff.SetSize(symmDims * nq * ne);
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::COMPRESSED);
+      DenseSymmetricMatrix sym_mat;
+      sym_mat.SetSize(dim);
 
-   if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (VQ) { coeff.Project(*VQ); }
-   else if (Q) { coeff.Project(*Q); }
-   else { coeff.SetConstant(1.0); }
+      auto C = Reshape(coeff.HostWrite(), symmDims, nq, ne);
 
-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dims*dims);
-   const int pa_size = symmetric ? symmDims : dims*dims;
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            SMQ->Eval(sym_mat, *tr, ir->IntPoint(p));
+            int cnt = 0;
+            for (int i=0; i<dim; ++i)
+               for (int j=i; j<dim; ++j, ++cnt)
+               {
+                  C(cnt, p, e) = sym_mat(i,j);
+               }
+         }
+      }
+   }
+   else if (MQ)
+   {
+      symmetric = false;
+      MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
 
-   pa_data.SetSize(pa_size * nq * ne, mt);
-   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeff_dim, ne, ir->GetWeights(),
+      coeffDim = MQfullDim;
+
+      coeff.SetSize(MQfullDim * nq * ne);
+
+      DenseMatrix mat;
+      mat.SetSize(dim);
+
+      auto C = Reshape(coeff.HostWrite(), MQfullDim, nq, ne);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            MQ->Eval(mat, *tr, ir->IntPoint(p));
+            for (int i=0; i<dim; ++i)
+               for (int j=0; j<dim; ++j)
+               {
+                  C(j+(i*dim), p, e) = mat(i,j);
+               }
+         }
+      }
+   }
+   else if (VQ)
+   {
+      MFEM_VERIFY(VQ->GetVDim() == dim, "");
+      coeffDim = VQ->GetVDim();
+      coeff.SetSize(coeffDim * nq * ne);
+      auto C = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+      Vector DM(coeffDim);
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            VQ->Eval(DM, *tr, ir->IntPoint(p));
+            for (int i=0; i<coeffDim; ++i)
+            {
+               C(i, p, e) = DM[i];
+            }
+         }
+      }
+   }
+   else if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
+   pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne, mt);
+   PADiffusionSetup(dim, sdim, dofs1D, quad1D, coeffDim, ne, ir->GetWeights(),
                     geom->J, coeff, pa_data);
 }
 

fem/bilininteg_gradient.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 
 using namespace std;
 
@@ -210,8 +209,44 @@ void GradientIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
                "PA requires test and trial space to have same number of quadrature points!");
    pa_data.SetSize(nq * dimsToStore * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+   Vector coeff;
+
+   if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");
+
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *trial_fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
 
    PAGradientSetup(dim, trial_dofs1D, test_dofs1D, quad1D,
                    ne, ir->GetWeights(), geom->J, coeff, pa_data);
@@ -830,3 +865,4 @@ void GradientIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 }
 
 } // namespace mfem
+

fem/bilininteg_hcurl.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qspace.hpp"
 
 using namespace std;
 
@@ -968,6 +967,8 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dim = mesh->Dimension();
    MFEM_VERIFY(dim == 2 || dim == 3, "");
 
+   const int dimc = (dim == 3) ? 3 : 1;
+
    ne = fes.GetNE();
    geom = mesh->GetGeometricFactors(*ir, GeometricFactors::JACOBIANS);
    mapsC = &el->GetDofToQuad(*ir, DofToQuad::TENSOR);
@@ -977,19 +978,88 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    MFEM_VERIFY(dofs1D == mapsO->ndof + 1 && quad1D == mapsO->nqpt, "");
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
-   if (Q) { coeff.Project(*Q); }
-   else if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
+   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
 
-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dim*dim);
-   const int sym_dims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
-   const int ndata = (dim == 2) ? 1 : (symmetric ? sym_dims : dim*dim);
+   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+
+   symmetric = (SMQ || MQ == NULL);
+
+   const int symmDims = (dims * (dims + 1)) / 2; // 1x1: 1, 2x2: 3, 3x3: 6
+   const int ndata = (dim == 2) ? 1 : (symmetric ? symmDims : MQfullDim);
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
+   Vector coeff(coeffDim * ne * nq);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ || MQ)
+   {
+      Vector DM(DQ ? coeffDim : 0);
+      DenseMatrix GM;
+      DenseSymmetricMatrix SM;
+
+      if (DQ)
+      {
+         MFEM_VERIFY(coeffDim == dimc, "");
+      }
+      if (SMQ)
+      {
+         SM.SetSize(dimc);
+         MFEM_VERIFY(SMQ->GetSize() == dimc, "");
+      }
+      else if (MQ)
+      {
+         GM.SetSize(dimc);
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dimc && MQ->GetWidth() == dimc, "");
+      }
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            if (SMQ)
+            {
+               SMQ->Eval(SM, *tr, ir->IntPoint(p));
+
+               int cnt = 0;
+               for (int i=0; i<dimc; ++i)
+                  for (int j=i; j<dimc; ++j, ++cnt)
+                  {
+                     coeffh(cnt, p, e) = SM(i,j);
+                  }
+
+            }
+            else if (MQ)
+            {
+               MQ->Eval(GM, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dimc; ++i)
+                  for (int j=0; j<dimc; ++j)
+                  {
+                     coeffh(j+(i*dimc), p, e) = GM(i,j);
+                  }
+
+            }
+            else if (DQ)
+            {
+               DQ->Eval(DM, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = DM[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }
+
    if (el->GetDerivType() != mfem::FiniteElement::CURL)
    {
       MFEM_ABORT("Unknown kernel.");
@@ -997,7 +1067,7 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    if (dim == 3)
    {
-      PACurlCurlSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J, coeff,
+      PACurlCurlSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J, coeff,
                         pa_data);
    }
    else
@@ -2710,7 +2780,7 @@ void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
    }
 }
 
-// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply3D(const int D1D,
                       const int Q1D,
@@ -2900,7 +2970,7 @@ void PAHcurlH1Apply3D(const int D1D,
    }); // end of element loop
 }
 
-// Apply to x corresponding to DOFs in H(curl), integrated
+// Apply to x corresponding to DOF's in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose3D(const int D1D,
                                const int Q1D,
@@ -3099,7 +3169,7 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
    }); // end of element loop
 }
 
-// Apply to x corresponding to DOFs in H^1 (trial), whose gradients are
+// Apply to x corresponding to DOF's in H^1 (trial), whose gradients are
 // integrated against H(curl) test functions corresponding to y.
 void PAHcurlH1Apply2D(const int D1D,
                       const int Q1D,
@@ -3223,7 +3293,7 @@ void PAHcurlH1Apply2D(const int D1D,
    }); // end of element loop
 }
 
-// Apply to x corresponding to DOFs in H(curl), integrated
+// Apply to x corresponding to DOF's in H(curl), integrated
 // against gradients of H^1 functions corresponding to y.
 void PAHcurlH1ApplyTranspose2D(const int D1D,
                                const int Q1D,
@@ -3419,8 +3489,20 @@ void MixedScalarCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    pa_data.SetSize(nq * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), nq, ne);
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeffh(p, e) = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }
 
    if (dim == 2)
    {
@@ -3511,11 +3593,38 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    const int ndata = curlSpaces ? (coeffDim == 1 ? 1 : 9) : symmDims;
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::FULL);
-   if (Q) { coeff.Project(*Q); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
+   Vector coeff(coeffDim * nq * ne);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ)
+   {
+      Vector V(coeffDim);
+      if (DQ)
+      {
+         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
+      }
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+
+         for (int p=0; p<nq; ++p)
+         {
+            if (DQ)
+            {
+               DQ->Eval(V, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = V[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }
 
    if (testType == mfem::FiniteElement::CURL &&
        trialType == mfem::FiniteElement::CURL && dim == 3)
@@ -3543,7 +3652,7 @@ void MixedVectorCurlIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    }
 }
 
-// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3D(const int D1D,
@@ -3906,7 +4015,7 @@ static void PAHcurlL2Apply3D(const int D1D,
    }); // end of element loop
 }
 
-// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void SmemPAHcurlL2Apply3D(const int D1D,
@@ -4216,7 +4325,7 @@ static void SmemPAHcurlL2Apply3D(const int D1D,
    ForallWrap<3>(true, NE, device_kernel, host_kernel, Q1D, Q1D, Q1D);
 }
 
-// Apply to x corresponding to DOFs in H(curl) (trial), whose curl is
+// Apply to x corresponding to DOF's in H(curl) (trial), whose curl is
 // integrated against H(div) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3D(const int D1D,
@@ -4572,7 +4681,7 @@ static void PAHcurlHdivApply3D(const int D1D,
    }); // end of element loop
 }
 
-// Apply to x corresponding to DOFs in H(div) (test), integrated against the
+// Apply to x corresponding to DOF's in H(div) (test), integrated against the
 // curl of H(curl) trial functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlHdivApply3DTranspose(const int D1D,
@@ -5037,11 +5146,38 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
 
    pa_data.SetSize(ndata * nq * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::FULL);
-   if (Q) { coeff.Project(*Q); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
+   Vector coeff(coeffDim * nq * ne);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ)
+   {
+      Vector V(coeffDim);
+      if (DQ)
+      {
+         MFEM_VERIFY(DQ->GetVDim() == coeffDim, "");
+      }
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+
+         for (int p=0; p<nq; ++p)
+         {
+            if (DQ)
+            {
+               DQ->Eval(V, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = V[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }
 
    if (trialType == mfem::FiniteElement::CURL && dim == 3)
    {
@@ -5067,7 +5203,7 @@ void MixedVectorWeakCurlIntegrator::AssemblePA(const FiniteElementSpace
    }
 }
 
-// Apply to x corresponding to DOFs in H(curl) (trial), integrated against curl
+// Apply to x corresponding to DOF's in H(curl) (trial), integrated against curl
 // of H(curl) test functions corresponding to y.
 template<int MAX_D1D = HCURL_MAX_D1D, int MAX_Q1D = HCURL_MAX_Q1D>
 static void PAHcurlL2Apply3DTranspose(const int D1D,

fem/bilininteg_hdiv.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qspace.hpp"
 
 using namespace std;
 
@@ -25,20 +24,18 @@ namespace mfem
 
 // PA H(div) Mass Assemble 2D kernel
 void PAHdivSetup2D(const int Q1D,
-                   const int coeffDim,
                    const int NE,
                    const Array<double> &w,
                    const Vector &j,
                    Vector &coeff_,
                    Vector &op)
 {
-   const bool symmetric = (coeffDim != 4);
    const int NQ = Q1D*Q1D;
    auto W = w.Read();
 
    auto J = Reshape(j.Read(), NQ, 2, 2, NE);
-   auto C = Reshape(coeff_.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, symmetric ? 3 : 4, NE);
+   auto coeff = Reshape(coeff_.Read(), NQ, NE);
+   auto y = Reshape(op.Write(), NQ, 3, NE);
 
    MFEM_FORALL(e, NE,
    {
@@ -48,60 +45,28 @@ void PAHdivSetup2D(const int Q1D,
          const double J21 = J(q,1,0,e);
          const double J12 = J(q,0,1,e);
          const double J22 = J(q,1,1,e);
-         const double c_detJ = W[q] / ((J11*J22)-(J21*J12));
-
-         // (1/detJ) J^T C J
-         if (coeffDim == 3 || coeffDim == 4) // Matrix coefficient
-         {
-            const double C11 = C(0,q,e);
-            const double C12 = C(1,q,e);
-            const double C21 = symmetric ? C12 : C(2,q,e);
-            const double C22 = symmetric ? C(2,q,e) : C(3,q,e);
-            const double R11 = C11*J11 + C12*J21;
-            const double R21 = C21*J11 + C22*J21;
-            const double R12 = C11*J12 + C12*J22;
-            const double R22 = C21*J12 + C22*J22;
-
-            y(q,0,e) = c_detJ * (J11*R11 + J21*R21); // 1,1
-            y(q,1,e) = c_detJ * (J11*R12 + J21*R22); // 1,2
-
-            if (symmetric)
-            {
-               y(q,2,e) = c_detJ * (J12*R12 + J22*R22); // 2,2
-            }
-            else
-            {
-               y(q,2,e) = c_detJ * (J12*R11 + J22*R21); // 2,1
-               y(q,3,e) = c_detJ * (J12*R12 + J22*R22); // 2,2
-            }
-         }
-         else // Vector or scalar coefficient
-         {
-            const double C1 = C(0,q,e);
-            const double C2 = (coeffDim == 2 ? C(1,q,e) : C1);
-            y(q,0,e) = c_detJ * (J11*C1*J11 + J21*C2*J21); // 1,1
-            y(q,1,e) = c_detJ * (J11*C1*J12 + J21*C2*J22); // 1,2
-            y(q,2,e) = c_detJ * (J12*C1*J12 + J22*C2*J22); // 2,2
-         }
+         const double c_detJ = W[q] * coeff(q, e) / ((J11*J22)-(J21*J12));
+         // (c/detJ) J^T J
+         y(q,0,e) = c_detJ * (J11*J11 + J21*J21); // 1,1
+         y(q,1,e) = c_detJ * (J11*J12 + J21*J22); // 1,2
+         y(q,2,e) = c_detJ * (J12*J12 + J22*J22); // 2,2
       }
    });
 }
 
 // PA H(div) Mass Assemble 3D kernel
 void PAHdivSetup3D(const int Q1D,
-                   const int coeffDim,
                    const int NE,
                    const Array<double> &w,
                    const Vector &j,
                    Vector &coeff_,
                    Vector &op)
 {
-   const bool symmetric = (coeffDim != 9);
    const int NQ = Q1D*Q1D*Q1D;
    auto W = w.Read();
    auto J = Reshape(j.Read(), NQ, 3, 3, NE);
-   auto C = Reshape(coeff_.Read(), coeffDim, NQ, NE);
-   auto y = Reshape(op.Write(), NQ, symmetric ? 6 : 9, NE);
+   auto coeff = Reshape(coeff_.Read(), NQ, NE);
+   auto y = Reshape(op.Write(), NQ, 6, NE);
 
    MFEM_FORALL(e, NE,
    {
@@ -119,58 +84,14 @@ void PAHdivSetup3D(const int Q1D,
          const double detJ = J11 * (J22 * J33 - J32 * J23) -
          /* */               J21 * (J12 * J33 - J32 * J13) +
          /* */               J31 * (J12 * J23 - J22 * J13);
-         const double c_detJ = W[q] / detJ;
-
-         // (1/detJ) J^T C J
-         if (coeffDim == 6 || coeffDim == 9) // Matrix coefficient version
-         {
-            double M[3][3];
-            M[0][0] = C(0, q, e);
-            M[0][1] = C(1, q, e);
-            M[0][2] = C(2, q, e);
-            M[1][0] = (!symmetric) ? C(3, q, e) : M[0][1];
-            M[1][1] = (!symmetric) ? C(4, q, e) : C(3, q, e);
-            M[1][2] = (!symmetric) ? C(5, q, e) : C(4, q, e);
-            M[2][0] = (!symmetric) ? C(6, q, e) : M[0][2];
-            M[2][1] = (!symmetric) ? C(7, q, e) : M[1][2];
-            M[2][2] = (!symmetric) ? C(8, q, e) : C(5, q, e);
-
-            int idx = 0;
-            for (int i=0; i<3; ++i)
-               for (int j = (symmetric ? i : 0); j<3; ++j)
-               {
-                  y(q,idx,e) = 0.0;
-                  for (int k=0; k<3; ++k)
-                  {
-                     double MJ_kj = 0.0;
-                     for (int l=0; l<3; ++l)
-                     {
-                        MJ_kj += M[k][l] * J(q,l,j,e);
-                     }
-
-                     y(q,idx,e) += J(q,k,i,e) * MJ_kj;
-                  }
-
-                  y(q,idx,e) *= c_detJ;
-                  idx++;
-               }
-         }
-         else  // Vector or scalar coefficient version
-         {
-            int idx = 0;
-            for (int i=0; i<3; ++i)
-               for (int j=i; j<3; ++j)
-               {
-                  y(q,idx,e) = 0.0;
-                  for (int k=0; k<3; ++k)
-                  {
-                     y(q,idx,e) += J(q,k,i,e) * C(coeffDim == 3 ? k : 0, q, e) * J(q,k,j,e);
-                  }
-
-                  y(q,idx,e) *= c_detJ;
-                  idx++;
-               }
-         }
+         const double c_detJ = W[q] * coeff(q, e) / detJ;
+         // (c/detJ) J^T J
+         y(q,0,e) = c_detJ * (J11*J11 + J21*J21 + J31*J31); // 1,1
+         y(q,1,e) = c_detJ * (J12*J11 + J22*J21 + J32*J31); // 2,1
+         y(q,2,e) = c_detJ * (J13*J11 + J23*J21 + J33*J31); // 3,1
+         y(q,3,e) = c_detJ * (J12*J12 + J22*J22 + J32*J32); // 2,2
+         y(q,4,e) = c_detJ * (J13*J12 + J23*J22 + J33*J32); // 3,2
+         y(q,5,e) = c_detJ * (J13*J13 + J23*J23 + J33*J33); // 3,3
       }
    });
 }
@@ -178,7 +99,6 @@ void PAHdivSetup3D(const int Q1D,
 void PAHdivMassApply2D(const int D1D,
                        const int Q1D,
                        const int NE,
-                       const bool symmetric,
                        const Array<double> &Bo_,
                        const Array<double> &Bc_,
                        const Array<double> &Bot_,
@@ -195,7 +115,7 @@ void PAHdivMassApply2D(const int D1D,
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
    auto Bct = Reshape(Bct_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, 3, NE);
    auto x = Reshape(x_.Read(), 2*(D1D-1)*D1D, NE);
    auto y = Reshape(y_.ReadWrite(), 2*(D1D-1)*D1D, NE);
 
@@ -258,12 +178,11 @@ void PAHdivMassApply2D(const int D1D,
          {
             const double O11 = op(qx,qy,0,e);
             const double O12 = op(qx,qy,1,e);
-            const double O21 = symmetric ? O12 : op(qx,qy,2,e);
-            const double O22 = symmetric ? op(qx,qy,2,e) : op(qx,qy,3,e);
+            const double O22 = op(qx,qy,2,e);
             const double massX = mass[qy][qx][0];
             const double massY = mass[qy][qx][1];
             mass[qy][qx][0] = (O11*massX)+(O12*massY);
-            mass[qy][qx][1] = (O21*massX)+(O22*massY);
+            mass[qy][qx][1] = (O12*massX)+(O22*massY);
          }
       }
 
@@ -306,179 +225,9 @@ void PAHdivMassApply2D(const int D1D,
    }); // end of element loop
 }
 
-template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAHdivMassApply2D(const int NE,
-                           const bool symmetric,
-                           const Array<double> &Bo_,
-                           const Array<double> &Bc_,
-                           const Array<double> &Bot_,
-                           const Array<double> &Bct_,
-                           const Vector &op_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int d1d = 0,
-                           const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(Bot_);
-   MFEM_CONTRACT_VAR(Bct_);
-
-   static constexpr int VDIM = 2;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
-   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
-   const auto D = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
-   const auto x = Reshape(x_.Read(), D1D*(D1D-1), VDIM, NE);
-   auto y = y_.ReadWrite();
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, VDIM,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-      MFEM_SHARED double smo[MQ1*(MD1-1)];
-      DeviceMatrix Bo(smo, D1D-1, Q1D);
-
-      MFEM_SHARED double smc[MQ1*MD1];
-      DeviceMatrix Bc(smc, D1D, Q1D);
-
-      MFEM_SHARED double sm0[VDIM*MDQ*MDQ];
-      MFEM_SHARED double sm1[VDIM*MDQ*MDQ];
-      DeviceMatrix X(sm0, D1D*(D1D-1), VDIM);
-      DeviceCube QD(sm1, Q1D, D1D, VDIM);
-      DeviceCube QQ(sm0, Q1D, Q1D, VDIM);
-
-      // Load X, Bo and Bc into shared memory
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         MFEM_FOREACH_THREAD(dy,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               if (qx < D1D && dy < (D1D-1)) { X(qx + dy*D1D,vd) = x(qx+dy*D1D,vd,e); }
-               if (tidz == 0)
-               {
-                  if (dy < (D1D-1)) { Bo(dy,qx) = bo(qx,dy); }
-                  Bc(dy,qx) = bc(qx,dy);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Apply B operator
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         DeviceCube Xxy(X, nx, ny, VDIM);
-         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double dq = 0.0;
-               for (int dx = 0; dx < nx; ++dx)
-               {
-                  dq += Xxy(dx,dy,vd) * Bx(dx,qx);
-               }
-               QD(qx,dy,vd) = dq;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         DeviceMatrix By = (vd == 1) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double qq = 0.0;
-               for (int dy = 0; dy < ny; ++dy)
-               {
-                  qq += QD(qx,dy,vd) * By(dy,qy);
-               }
-               QQ(qx,qy,vd) = qq;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Apply D operator
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               const double Qx = QQ(qx,qy,0);
-               const double Qy = QQ(qx,qy,1);
-
-               const double D11 = D(qx,qy,0,e);
-               const double D12 = D(qx,qy,1,e);
-               const double D21 = symmetric ? D12 : D(qx,qy,2,e);
-               const double D22 = symmetric ? D(qx,qy,2,e) : D(qx,qy,3,e);
-
-               QQ(qx,qy,0) = D11*Qx + D12*Qy;
-               QQ(qx,qy,1) = D21*Qx + D22*Qy;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Apply Bt operator
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               double qd = 0.0;
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  qd += QQ(qx,qy,vd) * Btx(dx,qx);
-               }
-               QD(dx,qy,vd) = qd;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
-         DeviceTensor<4> Yxy(y, nx, ny, VDIM, NE);
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               double dd = 0.0;
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  dd += QD(dx,qy,vd) * Bty(dy,qy);
-               }
-               Yxy(dx,dy,vd,e) += dd;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
                                   const int NE,
-                                  const bool symmetric,
                                   const Array<double> &Bo_,
                                   const Array<double> &Bc_,
                                   const Vector &op_,
@@ -489,7 +238,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, symmetric ? 3 : 4, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, 3, NE);
    auto diag = Reshape(diag_.ReadWrite(), 2*(D1D-1)*D1D, NE);
 
    MFEM_FORALL(e, NE,
@@ -510,7 +259,7 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
                for (int qy = 0; qy < Q1D; ++qy)
                {
                   const double wy = (c == 1) ? Bc(qy,dy) : Bo(qy,dy);
-                  mass[qx] += wy*wy*((c == 0) ? op(qx,qy,0,e) : op(qx,qy,symmetric ? 2 : 3,e));
+                  mass[qx] += wy*wy*((c == 0) ? op(qx,qy,0,e) : op(qx,qy,2,e));
                }
             }
 
@@ -534,7 +283,6 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const int Q1D,
                                   const int NE,
-                                  const bool symmetric,
                                   const Array<double> &Bo_,
                                   const Array<double> &Bc_,
                                   const Vector &op_,
@@ -546,7 +294,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
 
    auto Bo = Reshape(Bo_.Read(), Q1D, D1D-1);
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 6, NE);
    auto diag = Reshape(diag_.ReadWrite(), 3*(D1D-1)*(D1D-1)*D1D, NE);
 
    MFEM_FORALL(e, NE,
@@ -559,8 +307,7 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
          const int D1Dy = (c == 1) ? D1D : D1D - 1;
          const int D1Dx = (c == 0) ? D1D : D1D - 1;
 
-         const int opc = (c == 0) ? 0 : ((c == 1) ? (symmetric ? 3 : 4) :
-                                         (symmetric ? 5 : 8));
+         const int opc = (c == 0) ? 0 : ((c == 1) ? 3 : 5);
 
          double mass[HDIV_MAX_Q1D];
 
@@ -603,7 +350,6 @@ void PAHdivMassAssembleDiagonal3D(const int D1D,
 void PAHdivMassApply3D(const int D1D,
                        const int Q1D,
                        const int NE,
-                       const bool symmetric,
                        const Array<double> &Bo_,
                        const Array<double> &Bc_,
                        const Array<double> &Bot_,
@@ -620,7 +366,7 @@ void PAHdivMassApply3D(const int D1D,
    auto Bc = Reshape(Bc_.Read(), Q1D, D1D);
    auto Bot = Reshape(Bot_.Read(), D1D-1, Q1D);
    auto Bct = Reshape(Bct_.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
+   auto op = Reshape(op_.Read(), Q1D, Q1D, Q1D, 6, NE);
    auto x = Reshape(x_.Read(), 3*(D1D-1)*(D1D-1)*D1D, NE);
    auto y = Reshape(y_.ReadWrite(), 3*(D1D-1)*(D1D-1)*D1D, NE);
 
@@ -715,19 +461,15 @@ void PAHdivMassApply3D(const int D1D,
                const double O11 = op(qx,qy,qz,0,e);
                const double O12 = op(qx,qy,qz,1,e);
                const double O13 = op(qx,qy,qz,2,e);
-               const double O21 = symmetric ? O12 : op(qx,qy,qz,3,e);
-               const double O22 = symmetric ? op(qx,qy,qz,3,e) : op(qx,qy,qz,4,e);
-               const double O23 = symmetric ? op(qx,qy,qz,4,e) : op(qx,qy,qz,5,e);
-               const double O31 = symmetric ? O13 : op(qx,qy,qz,6,e);
-               const double O32 = symmetric ? O23 : op(qx,qy,qz,7,e);
-               const double O33 = symmetric ? op(qx,qy,qz,5,e) : op(qx,qy,qz,8,e);
-
+               const double O22 = op(qx,qy,qz,3,e);
+               const double O23 = op(qx,qy,qz,4,e);
+               const double O33 = op(qx,qy,qz,5,e);
                const double massX = mass[qz][qy][qx][0];
                const double massY = mass[qz][qy][qx][1];
                const double massZ = mass[qz][qy][qx][2];
                mass[qz][qy][qx][0] = (O11*massX)+(O12*massY)+(O13*massZ);
-               mass[qz][qy][qx][1] = (O21*massX)+(O22*massY)+(O23*massZ);
-               mass[qz][qy][qx][2] = (O31*massX)+(O32*massY)+(O33*massZ);
+               mass[qz][qy][qx][1] = (O12*massX)+(O22*massY)+(O23*massZ);
+               mass[qz][qy][qx][2] = (O13*massX)+(O23*massY)+(O33*massZ);
             }
          }
       }
@@ -795,337 +537,6 @@ void PAHdivMassApply3D(const int D1D,
    }); // end of element loop
 }
 
-template<int T_D1D = 0, int T_Q1D = 0>
-void SmemPAHdivMassApply3D(const int NE,
-                           const bool symmetric,
-                           const Array<double> &Bo_,
-                           const Array<double> &Bc_,
-                           const Array<double> &Bot_,
-                           const Array<double> &Bct_,
-                           const Vector &op_,
-                           const Vector &x_,
-                           Vector &y_,
-                           const int d1d = 0,
-                           const int q1d = 0)
-{
-   MFEM_CONTRACT_VAR(Bot_);
-   MFEM_CONTRACT_VAR(Bct_);
-
-   static constexpr int VDIM = 3;
-
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-   const auto bo = Reshape(Bo_.Read(), Q1D, D1D-1);
-   const auto bc = Reshape(Bc_.Read(), Q1D, D1D);
-   const auto D = Reshape(op_.Read(), Q1D, Q1D, Q1D, symmetric ? 6 : 9, NE);
-   const auto x = Reshape(x_.Read(), D1D*(D1D-1)*(D1D-1), VDIM, NE);
-   auto y = y_.ReadWrite();
-
-   MFEM_FORALL_3D(e, NE, Q1D, Q1D, VDIM,
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-
-      constexpr int MQ1 = T_Q1D ? T_Q1D : HDIV_MAX_Q1D;
-      constexpr int MD1 = T_D1D ? T_D1D : HDIV_MAX_D1D;
-      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-      MFEM_SHARED double smo[MQ1*(MD1-1)];
-      DeviceMatrix Bo(smo, D1D-1, Q1D);
-
-      MFEM_SHARED double smc[MQ1*MD1];
-      DeviceMatrix Bc(smc, D1D, Q1D);
-
-      MFEM_SHARED double sm0[VDIM*MDQ*MDQ*MDQ];
-      MFEM_SHARED double sm1[VDIM*MDQ*MDQ*MDQ];
-      DeviceMatrix X(sm0, D1D*(D1D-1)*(D1D-1), VDIM);
-      DeviceTensor<4> QDD(sm1, Q1D, D1D, D1D, VDIM);
-      DeviceTensor<4> QQD(sm0, Q1D, Q1D, D1D, VDIM);
-      DeviceTensor<4> QQQ(sm1, Q1D, Q1D, Q1D, VDIM);
-      DeviceTensor<4> DQQ(sm0, D1D, Q1D, Q1D, VDIM);
-      DeviceTensor<4> DDQ(sm1, D1D, D1D, Q1D, VDIM);
-
-      // Load X into shared memory
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         MFEM_FOREACH_THREAD(dz,y,D1D-1)
-         {
-            MFEM_FOREACH_THREAD(dy,x,D1D-1)
-            {
-               MFEM_UNROLL(MD1)
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  X(dx+(dy+dz*(D1D-1))*D1D,vd) = x(dx+(dy+dz*(D1D-1))*D1D,vd,e);
-               }
-            }
-         }
-      }
-      // Load Bo and Bc into shared memory
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(d,y,D1D-1)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bo(d,q) = bo(q,d);
-            }
-         }
-         MFEM_FOREACH_THREAD(d,y,D1D)
-         {
-            MFEM_FOREACH_THREAD(q,x,Q1D)
-            {
-               Bc(d,q) = bc(q,d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Apply B operator
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         const int nz = (vd == 2) ? D1D : D1D-1;
-         DeviceTensor<4> Xxyz(X, nx, ny, nz, VDIM);
-         DeviceMatrix Bx = (vd == 0) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u[D1D];
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
-               MFEM_UNROLL(MD1)
-               for (int dx = 0; dx < nx; ++dx)
-               {
-                  MFEM_UNROLL(MD1)
-                  for (int dz = 0; dz < nz; ++dz)
-                  {
-                     u[dz] += Xxyz(dx,dy,dz,vd) * Bx(dx,qx);
-                  }
-               }
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz) { QDD(qx,dy,dz,vd) = u[dz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         const int nz = (vd == 2) ? D1D : D1D-1;
-         DeviceMatrix By = (vd == 1) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u[D1D];
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
-               MFEM_UNROLL(MD1)
-               for (int dy = 0; dy < ny; ++dy)
-               {
-                  MFEM_UNROLL(MD1)
-                  for (int dz = 0; dz < nz; ++dz)
-                  {
-                     u[dz] += QDD(qx,dy,dz,vd) * By(dy,qy);
-                  }
-               }
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz) { QQD(qx,qy,dz,vd) = u[dz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nz = (vd == 2) ? D1D : D1D-1;
-         DeviceMatrix Bz = (vd == 2) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               double u[Q1D];
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz)
-               {
-                  MFEM_UNROLL(MQ1)
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u[qz] += QQD(qx,qy,dz,vd) * Bz(dz,qz);
-                  }
-               }
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz) { QQQ(qx,qy,qz,vd) = u[qz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Apply D operator
-      if (tidz == 0)
-      {
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(qx,x,Q1D)
-            {
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  const double Qx = QQQ(qx,qy,qz,0);
-                  const double Qy = QQQ(qx,qy,qz,1);
-                  const double Qz = QQQ(qx,qy,qz,2);
-
-                  const double D11 = D(qx,qy,qz,0,e);
-                  const double D12 = D(qx,qy,qz,1,e);
-                  const double D13 = D(qx,qy,qz,2,e);
-                  const double D21 = symmetric ? D12 : D(qx,qy,qz,3,e);
-                  const double D22 = symmetric ? D(qx,qy,qz,3,e) : D(qx,qy,qz,4,e);
-                  const double D23 = symmetric ? D(qx,qy,qz,4,e) : D(qx,qy,qz,5,e);
-                  const double D31 = symmetric ? D13 : D(qx,qy,qz,6,e);
-                  const double D32 = symmetric ? D23 : D(qx,qy,qz,7,e);
-                  const double D33 = symmetric ? D(qx,qy,qz,5,e) : D(qx,qy,qz,8,e);
-
-                  QQQ(qx,qy,qz,0) = D11*Qx + D12*Qy + D13*Qz;
-                  QQQ(qx,qy,qz,1) = D21*Qx + D22*Qy + D23*Qz;
-                  QQQ(qx,qy,qz,2) = D31*Qx + D32*Qy + D33*Qz;
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      // Apply Bt operator
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         DeviceMatrix Btx = (vd == 0) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(qy,y,Q1D)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               double u[Q1D];
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(MQ1)
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  MFEM_UNROLL(MQ1)
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u[qz] += QQQ(qx,qy,qz,vd) * Btx(dx,qx);
-                  }
-               }
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz) { DQQ(dx,qy,qz,vd) = u[qz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         DeviceMatrix Bty = (vd == 1) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               double u[Q1D];
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz) { u[qz] = 0.0; }
-               MFEM_UNROLL(MQ1)
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  MFEM_UNROLL(MQ1)
-                  for (int qz = 0; qz < Q1D; ++qz)
-                  {
-                     u[qz] += DQQ(dx,qy,qz,vd) * Bty(dy,qy);
-                  }
-               }
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz) { DDQ(dx,dy,qz,vd) = u[qz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(vd,z,VDIM)
-      {
-         const int nx = (vd == 0) ? D1D : D1D-1;
-         const int ny = (vd == 1) ? D1D : D1D-1;
-         const int nz = (vd == 2) ? D1D : D1D-1;
-         DeviceTensor<5> Yxyz(y, nx, ny, nz, VDIM, NE);
-         DeviceMatrix Btz = (vd == 2) ? Bc : Bo;
-         MFEM_FOREACH_THREAD(dy,y,ny)
-         {
-            MFEM_FOREACH_THREAD(dx,x,nx)
-            {
-               double u[D1D];
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz) { u[dz] = 0.0; }
-               MFEM_UNROLL(MQ1)
-               for (int qz = 0; qz < Q1D; ++qz)
-               {
-                  MFEM_UNROLL(MD1)
-                  for (int dz = 0; dz < nz; ++dz)
-                  {
-                     u[dz] += DDQ(dx,dy,qz,vd) * Btz(dz,qz);
-                  }
-               }
-               MFEM_UNROLL(MD1)
-               for (int dz = 0; dz < nz; ++dz) { Yxyz(dx,dy,dz,vd,e) += u[dz]; }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-   });
-}
-
-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<double> &Bo,
-                     const Array<double> &Bc,
-                     const Array<double> &Bot,
-                     const Array<double> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y)
-{
-   const int id = (D1D << 4) | Q1D;
-
-   if (dim == 2)
-   {
-      switch (id)
-      {
-         case 0x22: return SmemPAHdivMassApply2D<2,2>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x33: return SmemPAHdivMassApply2D<3,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x44: return SmemPAHdivMassApply2D<4,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x55: return SmemPAHdivMassApply2D<5,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply2D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-   else if (dim == 3)
-   {
-      switch (id)
-      {
-         case 0x23: return SmemPAHdivMassApply3D<2,3>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x34: return SmemPAHdivMassApply3D<3,4>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x45: return SmemPAHdivMassApply3D<4,5>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x56: return SmemPAHdivMassApply3D<5,6>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x67: return SmemPAHdivMassApply3D<6,7>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         case 0x78: return SmemPAHdivMassApply3D<7,8>(NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-         default: // fallback
-            return PAHdivMassApply3D(D1D,Q1D,NE,symmetric,Bo,Bc,Bot,Bct,op,x,y);
-      }
-   }
-}
-
 // PA H(div) div-div assemble 2D kernel
 // NOTE: this is identical to PACurlCurlSetup3D
 static void PADivDivSetup2D(const int Q1D,
@@ -1215,7 +626,7 @@ static void PADivDivApply2D(const int D1D,
    {
       double div[MAX_Q1D][MAX_Q1D];
 
-      // div[qy][qx] will be computed as du_x/dx + du_y/dy
+      // div[qy][qx] will be computed as du_x/dx + duy_/dy
 
       for (int qy = 0; qy < Q1D; ++qy)
       {
@@ -1514,8 +925,19 @@ void DivDivIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    pa_data.SetSize(nq * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }
 
    if (el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
    {
@@ -1773,14 +1195,18 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
 
    pa_data.SetSize(nq * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
-
-   if (test_el->GetMapType() == FiniteElement::INTEGRAL)
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   if (Q)
    {
-      const GeometricFactors *geom =
-         mesh->GetGeometricFactors(*ir, GeometricFactors::DETERMINANTS);
-      coeff /= geom->detJ;
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
    }
 
    if (trial_el->GetDerivType() == mfem::FiniteElement::DIV && dim == 3)
@@ -1797,7 +1223,7 @@ VectorFEDivergenceIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    }
 }
 
-// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply3D(const int D1D,
                             const int Q1D,
@@ -1960,7 +1386,7 @@ static void PAHdivL2Apply3D(const int D1D,
    }); // end of element loop
 }
 
-// Apply to x corresponding to DOFs in H(div) (trial), whose divergence is
+// Apply to x corresponding to DOF's in H(div) (trial), whose divergence is
 // integrated against L_2 test functions corresponding to y.
 static void PAHdivL2Apply2D(const int D1D,
                             const int Q1D,

fem/bilininteg_mass_mf.cpp

@@ -31,16 +31,7 @@ void MassIntegrator::AssembleMF(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedMFMassIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
       return;
    }
    MFEM_ABORT("Error: MassIntegrator::AssembleMF only implemented with"

fem/bilininteg_mass_pa.cpp

@@ -12,9 +12,7 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/mass/mass.hpp"
-#include "bilininteg_mass_pa.hpp"
 
 using namespace std;
 
@@ -40,16 +38,7 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
       return;
    }
    int map_type = el.GetMapType();
@@ -62,10 +51,43 @@ void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    dofs1D = maps->ndof;
    quad1D = maps->nqpt;
    pa_data.SetSize(ne*nq, mt);
+   Vector coeff;
+   if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == nq * ne,
+                  "Incompatible QuadratureFunction dimension \n");
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
-
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto C = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            C(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
    if (dim==1) { MFEM_ABORT("Not supported yet... stay tuned!"); }
    if (dim==2)
    {
@@ -559,18 +581,85 @@ static void PAMassApply2D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b_.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto X = Reshape(x_.Read(), D1D, D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL(e, NE,
    {
-      internal::PAMassApply2D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+      const int D1D = T_D1D ? T_D1D : d1d; // nvcc workaround
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      // the following variables are evaluated at compile time
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double sol_xy[max_Q1D][max_Q1D];
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] = 0.0;
+         }
+      }
+      for (int dy = 0; dy < D1D; ++dy)
+      {
+         double sol_x[max_Q1D];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            sol_x[qy] = 0.0;
+         }
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            const double s = X(dx,dy,e);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] += B(qx,dx)* s;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            const double d2q = B(qy,dy);
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] += d2q * sol_x[qx];
+            }
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            sol_xy[qy][qx] *= D(qx,qy,e);
+         }
+      }
+      for (int qy = 0; qy < Q1D; ++qy)
+      {
+         double sol_x[max_D1D];
+         for (int dx = 0; dx < D1D; ++dx)
+         {
+            sol_x[dx] = 0.0;
+         }
+         for (int qx = 0; qx < Q1D; ++qx)
+         {
+            const double s = sol_xy[qy][qx];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] += Bt(dx,qx) * s;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            const double q2d = Bt(dy,qy);
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               Y(dx,dy,e) += q2d * sol_x[dx];
+            }
+         }
+      }
    });
 }
 
@@ -592,13 +681,108 @@ static void SmemPAMassApply2D(const int NE,
    constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
    MFEM_VERIFY(D1D <= MD1, "");
    MFEM_VERIFY(Q1D <= MQ1, "");
-   const auto b = b_.Read();
-   const auto D = d_.Read();
-   const auto x = x_.Read();
-   auto Y = y_.ReadWrite();
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, NE);
    MFEM_FORALL_2D(e, NE, Q1D, Q1D, NBZ,
    {
-      internal::SmemPAMassApply2D_Element<T_D1D,T_Q1D,T_NBZ>(e, NE, b, D, x, Y, d1d, q1d);
+      const int tidz = MFEM_THREAD_ID(z);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      MFEM_SHARED double BBt[MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) BBt;
+      double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
+      MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
+      MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
+      double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
+      double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
+      double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
+      double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            X[dy][dx] = x(dx,dy,e);
+         }
+      }
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               B[q][dy] = b(q,dy);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double dq = 0.0;
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               dq += X[dy][dx] * B[qx][dx];
+            }
+            DQ[dy][qx] = dq;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double qq = 0.0;
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               qq += DQ[dy][qx] * B[qy][dy];
+            }
+            QQ[qy][qx] = qq * D(qx, qy, e);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      if (tidz == 0)
+      {
+         MFEM_FOREACH_THREAD(dy,y,D1D)
+         {
+            MFEM_FOREACH_THREAD(q,x,Q1D)
+            {
+               Bt[dy][q] = b(q,dy);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double dq = 0.0;
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               dq += QQ[qy][qx] * Bt[dx][qx];
+            }
+            QD[qy][dx] = dq;
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double dd = 0.0;
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               dd += (QD[qy][dx] * Bt[dy][qy]);
+            }
+            Y(dx, dy, e) += dd;
+         }
+      }
    });
 }
 
@@ -612,18 +796,134 @@ static void PAMassApply3D(const int NE,
                           const int d1d = 0,
                           const int q1d = 0)
 {
-   MFEM_VERIFY(T_D1D ? T_D1D : d1d <= MAX_D1D, "");
-   MFEM_VERIFY(T_Q1D ? T_Q1D : q1d <= MAX_Q1D, "");
-
-   const auto B = b_.Read();
-   const auto Bt = bt_.Read();
-   const auto D = d_.Read();
-   const auto X = x_.Read();
-   auto Y = y_.ReadWrite();
-
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= MAX_Q1D, "");
+   auto B = Reshape(b_.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt_.Read(), D1D, Q1D);
+   auto D = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto X = Reshape(x_.Read(), D1D, D1D, D1D, NE);
+   auto Y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_FORALL(e, NE,
    {
-      internal::PAMassApply3D_Element(e, NE, B, Bt, D, X, Y, d1d, q1d);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : MAX_Q1D;
+      double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] = 0.0;
+            }
+         }
+      }
+      for (int dz = 0; dz < D1D; ++dz)
+      {
+         double sol_xy[max_Q1D][max_Q1D];
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xy[qy][qx] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            double sol_x[max_Q1D];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_x[qx] = 0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const double s = X(dx,dy,dz,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_x[qx] += B(qx,dx) * s;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const double wy = B(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xy[qy][qx] += wy * sol_x[qx];
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            const double wz = B(qz,dz);
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
+               }
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
+            }
+         }
+      }
+      for (int qz = 0; qz < Q1D; ++qz)
+      {
+         double sol_xy[max_D1D][max_D1D];
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_xy[dy][dx] = 0;
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            double sol_x[max_D1D];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               sol_x[dx] = 0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const double s = sol_xyz[qz][qy][qx];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_x[dx] += Bt(dx,qx) * s;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const double wy = Bt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  sol_xy[dy][dx] += wy * sol_x[dx];
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            const double wz = Bt(dz,qz);
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
+               }
+            }
+         }
+      }
    });
 }
 
@@ -644,13 +944,213 @@ static void SmemPAMassApply3D(const int NE,
    constexpr int M1D = T_D1D ? T_D1D : MAX_D1D;
    MFEM_VERIFY(D1D <= M1D, "");
    MFEM_VERIFY(Q1D <= M1Q, "");
-   auto b = b_.Read();
-   auto d = d_.Read();
-   auto x = x_.Read();
-   auto y = y_.ReadWrite();
+   auto b = Reshape(b_.Read(), Q1D, D1D);
+   auto d = Reshape(d_.Read(), Q1D, Q1D, Q1D, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, NE);
    MFEM_FORALL_3D(e, NE, Q1D, Q1D, 1,
    {
-      internal::SmemPAMassApply3D_Element<T_D1D,T_Q1D>(e, NE, b, d, x, y, d1d, q1d);
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
+      constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
+      constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
+      MFEM_SHARED double sDQ[MQ1*MD1];
+      double (*B)[MD1] = (double (*)[MD1]) sDQ;
+      double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
+      MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
+      MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
+      double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
+      double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
+      double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
+      double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
+      double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
+      double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               X[dz][dy][dx] = x(dx,dy,dz,e);
+            }
+         }
+         MFEM_FOREACH_THREAD(dx,x,Q1D)
+         {
+            B[dx][dy] = b(dx,dy);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += X[dz][dy][dx] * B[qx][dx];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               DDQ[dz][dy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; dz++)
+               {
+                  u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; dz++)
+            {
+               DQQ[dz][qy][qx] = u[dz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(qx,x,Q1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; qz++)
+               {
+                  u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; qz++)
+            {
+               QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(d,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(q,x,Q1D)
+         {
+            Bt[d][q] = b(q,d);
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(qy,y,Q1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QQD[qz][qy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[Q1D];
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               u[qz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               MFEM_UNROLL(MQ1)
+               for (int qz = 0; qz < Q1D; ++qz)
+               {
+                  u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
+               }
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               QDD[qz][dy][dx] = u[qz];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+      MFEM_FOREACH_THREAD(dy,y,D1D)
+      {
+         MFEM_FOREACH_THREAD(dx,x,D1D)
+         {
+            double u[D1D];
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               u[dz] = 0;
+            }
+            MFEM_UNROLL(MQ1)
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               MFEM_UNROLL(MD1)
+               for (int dz = 0; dz < D1D; ++dz)
+               {
+                  u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
+               }
+            }
+            MFEM_UNROLL(MD1)
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               y(dx,dy,dz,e) += u[dz];
+            }
+         }
+      }
    });
 }
 

fem/bilininteg_mass_pa.hpp

@@ -1,632 +0,0 @@
-// Copyright (c) 2010-2022, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_MASS_PA_HPP
-#define MFEM_BILININTEG_MASS_PA_HPP
-
-#include "../config/config.hpp"
-#include "../general/forall.hpp"
-#include "../linalg/dtensor.hpp"
-
-namespace mfem
-{
-
-namespace internal
-{
-
-template <bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void PAMassApply2D_Element(const int e,
-                           const int NE,
-                           const double *b_,
-                           const double *bt_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_,
-                           const int d1d = 0,
-                           const int q1d = 0)
-{
-   const int D1D = d1d;
-   const int Q1D = q1d;
-   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
-   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
-   auto X = ConstDeviceCube(x_, D1D, D1D, NE);
-   auto Y = DeviceCube(y_, D1D, D1D, NE);
-
-   if (!ACCUMULATE)
-   {
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            Y(dx, dy, e) = 0.0;
-         }
-      }
-   }
-
-   constexpr int max_D1D = MAX_D1D;
-   constexpr int max_Q1D = MAX_Q1D;
-   double sol_xy[max_Q1D][max_Q1D];
-   for (int qy = 0; qy < Q1D; ++qy)
-   {
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         sol_xy[qy][qx] = 0.0;
-      }
-   }
-   for (int dy = 0; dy < D1D; ++dy)
-   {
-      double sol_x[max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         sol_x[qy] = 0.0;
-      }
-      for (int dx = 0; dx < D1D; ++dx)
-      {
-         const double s = X(dx,dy,e);
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_x[qx] += B(qx,dx)* s;
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         const double d2q = B(qy,dy);
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] += d2q * sol_x[qx];
-         }
-      }
-   }
-   for (int qy = 0; qy < Q1D; ++qy)
-   {
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         sol_xy[qy][qx] *= D(qx,qy,e);
-      }
-   }
-   for (int qy = 0; qy < Q1D; ++qy)
-   {
-      double sol_x[max_D1D];
-      for (int dx = 0; dx < D1D; ++dx)
-      {
-         sol_x[dx] = 0.0;
-      }
-      for (int qx = 0; qx < Q1D; ++qx)
-      {
-         const double s = sol_xy[qy][qx];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] += Bt(dx,qx) * s;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         const double q2d = Bt(dy,qy);
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            Y(dx,dy,e) += q2d * sol_x[dx];
-         }
-      }
-   }
-}
-
-template<int T_D1D, int T_Q1D, int T_NBZ, bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void SmemPAMassApply2D_Element(const int e,
-                               const int NE,
-                               const double *b_,
-                               const double *d_,
-                               const double *x_,
-                               double *y_,
-                               int d1d = 0,
-                               int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto D = ConstDeviceCube(d_, Q1D, Q1D, NE);
-   auto x = ConstDeviceCube(x_, D1D, D1D, NE);
-   auto Y = DeviceCube(y_, D1D, D1D, NE);
-
-   const int tidz = MFEM_THREAD_ID(z);
-
-   MFEM_SHARED double BBt[MQ1*MD1];
-   double (*B)[MD1] = (double (*)[MD1]) BBt;
-   double (*Bt)[MQ1] = (double (*)[MQ1]) BBt;
-   MFEM_SHARED double sm0[NBZ][MDQ*MDQ];
-   MFEM_SHARED double sm1[NBZ][MDQ*MDQ];
-   double (*X)[MD1] = (double (*)[MD1]) (sm0 + tidz);
-   double (*DQ)[MQ1] = (double (*)[MQ1]) (sm1 + tidz);
-   double (*QQ)[MQ1] = (double (*)[MQ1]) (sm0 + tidz);
-   double (*QD)[MD1] = (double (*)[MD1]) (sm1 + tidz);
-
-
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         X[dy][dx] = x(dx,dy,e);
-      }
-   }
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            B[q][dy] = b(q,dy);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double dq = 0.0;
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            dq += X[dy][dx] * B[qx][dx];
-         }
-         DQ[dy][qx] = dq;
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double qq = 0.0;
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            qq += DQ[dy][qx] * B[qy][dy];
-         }
-         QQ[qy][qx] = qq * D(qx, qy, e);
-      }
-   }
-   MFEM_SYNC_THREAD;
-   if (tidz == 0)
-   {
-      MFEM_FOREACH_THREAD(dy,y,D1D)
-      {
-         MFEM_FOREACH_THREAD(q,x,Q1D)
-         {
-            Bt[dy][q] = b(q,dy);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double dq = 0.0;
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            dq += QQ[qy][qx] * Bt[dx][qx];
-         }
-         QD[qy][dx] = dq;
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double dd = 0.0;
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            dd += (QD[qy][dx] * Bt[dy][qy]);
-         }
-         if (ACCUMULATE)
-         {
-            Y(dx, dy, e) += dd;
-         }
-         else
-         {
-            Y(dx, dy, e) = dd;
-         }
-      }
-   }
-}
-
-template <bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void PAMassApply3D_Element(const int e,
-                           const int NE,
-                           const double *b_,
-                           const double *bt_,
-                           const double *d_,
-                           const double *x_,
-                           double *y_,
-                           const int d1d,
-                           const int q1d)
-{
-   const int D1D = d1d;
-   const int Q1D = q1d;
-   auto B = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto Bt = ConstDeviceMatrix(bt_, D1D, Q1D);
-   auto D = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
-   auto X = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
-   auto Y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
-
-   if (!ACCUMULATE)
-   {
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx, dy, dz, e) = 0.0;
-            }
-         }
-      }
-   }
-
-   constexpr int max_D1D = MAX_D1D;
-   constexpr int max_Q1D = MAX_Q1D;
-   double sol_xyz[max_Q1D][max_Q1D][max_Q1D];
-   for (int qz = 0; qz < Q1D; ++qz)
-   {
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xyz[qz][qy][qx] = 0.0;
-         }
-      }
-   }
-   for (int dz = 0; dz < D1D; ++dz)
-   {
-      double sol_xy[max_Q1D][max_Q1D];
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xy[qy][qx] = 0.0;
-         }
-      }
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         double sol_x[max_Q1D];
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_x[qx] = 0;
-         }
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            const double s = X(dx,dy,dz,e);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_x[qx] += B(qx,dx) * s;
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            const double wy = B(qy,dy);
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xy[qy][qx] += wy * sol_x[qx];
-            }
-         }
-      }
-      for (int qz = 0; qz < Q1D; ++qz)
-      {
-         const double wz = B(qz,dz);
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               sol_xyz[qz][qy][qx] += wz * sol_xy[qy][qx];
-            }
-         }
-      }
-   }
-   for (int qz = 0; qz < Q1D; ++qz)
-   {
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            sol_xyz[qz][qy][qx] *= D(qx,qy,qz,e);
-         }
-      }
-   }
-   for (int qz = 0; qz < Q1D; ++qz)
-   {
-      double sol_xy[max_D1D][max_D1D];
-      for (int dy = 0; dy < D1D; ++dy)
-      {
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_xy[dy][dx] = 0;
-         }
-      }
-      for (int qy = 0; qy < Q1D; ++qy)
-      {
-         double sol_x[max_D1D];
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            sol_x[dx] = 0;
-         }
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            const double s = sol_xyz[qz][qy][qx];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_x[dx] += Bt(dx,qx) * s;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            const double wy = Bt(dy,qy);
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               sol_xy[dy][dx] += wy * sol_x[dx];
-            }
-         }
-      }
-      for (int dz = 0; dz < D1D; ++dz)
-      {
-         const double wz = Bt(dz,qz);
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               Y(dx,dy,dz,e) += wz * sol_xy[dy][dx];
-            }
-         }
-      }
-   }
-}
-
-template<int T_D1D, int T_Q1D, bool ACCUMULATE = true>
-MFEM_HOST_DEVICE inline
-void SmemPAMassApply3D_Element(const int e,
-                               const int NE,
-                               const double *b_,
-                               const double *d_,
-                               const double *x_,
-                               double *y_,
-                               const int d1d = 0,
-                               const int q1d = 0)
-{
-   constexpr int D1D = T_D1D ? T_D1D : d1d;
-   constexpr int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int MQ1 = T_Q1D ? T_Q1D : MAX_Q1D;
-   constexpr int MD1 = T_D1D ? T_D1D : MAX_D1D;
-   constexpr int MDQ = (MQ1 > MD1) ? MQ1 : MD1;
-
-   auto b = ConstDeviceMatrix(b_, Q1D, D1D);
-   auto d = DeviceTensor<4,const double>(d_, Q1D, Q1D, Q1D, NE);
-   auto x = DeviceTensor<4,const double>(x_, D1D, D1D, D1D, NE);
-   auto y = DeviceTensor<4,double>(y_, D1D, D1D, D1D, NE);
-
-   MFEM_SHARED double sDQ[MQ1*MD1];
-   double (*B)[MD1] = (double (*)[MD1]) sDQ;
-   double (*Bt)[MQ1] = (double (*)[MQ1]) sDQ;
-   MFEM_SHARED double sm0[MDQ*MDQ*MDQ];
-   MFEM_SHARED double sm1[MDQ*MDQ*MDQ];
-   double (*X)[MD1][MD1]   = (double (*)[MD1][MD1]) sm0;
-   double (*DDQ)[MD1][MQ1] = (double (*)[MD1][MQ1]) sm1;
-   double (*DQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm0;
-   double (*QQQ)[MQ1][MQ1] = (double (*)[MQ1][MQ1]) sm1;
-   double (*QQD)[MQ1][MD1] = (double (*)[MQ1][MD1]) sm0;
-   double (*QDD)[MD1][MD1] = (double (*)[MD1][MD1]) sm1;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            X[dz][dy][dx] = x(dx,dy,dz,e);
-         }
-      }
-      MFEM_FOREACH_THREAD(dx,x,Q1D)
-      {
-         B[dx][dy] = b(dx,dy);
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double u[D1D];
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; dz++)
-         {
-            u[dz] = 0;
-         }
-         MFEM_UNROLL(MD1)
-         for (int dx = 0; dx < D1D; ++dx)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] += X[dz][dy][dx] * B[qx][dx];
-            }
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            DDQ[dz][dy][qx] = u[dz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double u[D1D];
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; dz++)
-         {
-            u[dz] = 0;
-         }
-         MFEM_UNROLL(MD1)
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; dz++)
-            {
-               u[dz] += DDQ[dz][dy][qx] * B[qy][dy];
-            }
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; dz++)
-         {
-            DQQ[dz][qy][qx] = u[dz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(qx,x,Q1D)
-      {
-         double u[Q1D];
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; qz++)
-         {
-            u[qz] = 0;
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; qz++)
-            {
-               u[qz] += DQQ[dz][qy][qx] * B[qz][dz];
-            }
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; qz++)
-         {
-            QQQ[qz][qy][qx] = u[qz] * d(qx,qy,qz,e);
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(di,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(q,x,Q1D)
-      {
-         Bt[di][q] = b(q,di);
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(qy,y,Q1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u[Q1D];
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            u[qz] = 0;
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qx = 0; qx < Q1D; ++qx)
-         {
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] += QQQ[qz][qy][qx] * Bt[dx][qx];
-            }
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            QQD[qz][qy][dx] = u[qz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u[Q1D];
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            u[qz] = 0;
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            MFEM_UNROLL(MQ1)
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               u[qz] += QQD[qz][qy][dx] * Bt[dy][qy];
-            }
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            QDD[qz][dy][dx] = u[qz];
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-   MFEM_FOREACH_THREAD(dy,y,D1D)
-   {
-      MFEM_FOREACH_THREAD(dx,x,D1D)
-      {
-         double u[D1D];
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            u[dz] = 0;
-         }
-         MFEM_UNROLL(MQ1)
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            MFEM_UNROLL(MD1)
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               u[dz] += QDD[qz][dy][dx] * Bt[dz][qz];
-            }
-         }
-         MFEM_UNROLL(MD1)
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            if (ACCUMULATE)
-            {
-               y(dx,dy,dz,e) += u[dz];
-            }
-            else
-            {
-               y(dx,dy,dz,e) = u[dz];
-            }
-         }
-      }
-   }
-   MFEM_SYNC_THREAD;
-}
-
-} // namespace internal
-
-} // namespace mfem
-
-#endif

fem/bilininteg_vecdiffusion.cpp

@@ -12,7 +12,6 @@
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
 #include "gridfunc.hpp"
-#include "qfunction.hpp"
 #include "ceed/integrators/diffusion/diffusion.hpp"
 
 using namespace std;
@@ -150,16 +149,7 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPADiffusionIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::PADiffusionIntegrator(fes, *ir, Q);
       return;
    }
    const int dims = el.GetDim();
@@ -176,9 +166,43 @@ void VectorDiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
    MFEM_VERIFY(!VQ && !MQ,
                "Only scalar coefficient supported for partial assembly for VectorDiffusionIntegrator");
+   Vector coeff;
+   if (Q == nullptr)
+   {
+      coeff.SetSize(1);
+      coeff(0) = 1.0;
+   }
+   else if (ConstantCoefficient* cQ = dynamic_cast<ConstantCoefficient*>(Q))
+   {
+      coeff.SetSize(1);
+      coeff(0) = cQ->constant;
+   }
+   else if (QuadratureFunctionCoefficient* qfQ =
+               dynamic_cast<QuadratureFunctionCoefficient*>(Q))
+   {
+      const QuadratureFunction &qFun = qfQ->GetQuadFunction();
+      MFEM_VERIFY(qFun.Size() == ne*nq,
+                  "Incompatible QuadratureFunction dimension \n");
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::COMPRESSED);
+      MFEM_VERIFY(ir == &qFun.GetSpace()->GetElementIntRule(0),
+                  "IntegrationRule used within integrator and in"
+                  " QuadratureFunction appear to be different");
+      qFun.Read();
+      coeff.MakeRef(const_cast<QuadratureFunction &>(qFun),0);
+   }
+   else
+   {
+      coeff.SetSize(nq * ne);
+      auto Co = Reshape(coeff.HostWrite(), nq, ne);
+      for (int e = 0; e < ne; ++e)
+      {
+         ElementTransformation& T = *fes.GetElementTransformation(e);
+         for (int q = 0; q < nq; ++q)
+         {
+            Co(q,e) = Q->Eval(T, ir->IntPoint(q));
+         }
+      }
+   }
 
    const Array<double> &w = ir->GetWeights();
    const Vector &j = geom->J;

fem/bilininteg_vecdiffusion_mf.cpp

@@ -30,19 +30,7 @@ void VectorDiffusionIntegrator::AssembleMF(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      MFEM_VERIFY(!VQ && !MQ,
-                  "Only scalar coefficient supported for DiffusionIntegrator"
-                  " with libCEED");
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedMFDiffusionIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::MFDiffusionIntegrator(fes, *ir, Q);
       return;
    }
    MFEM_ABORT("Error: VectorDiffusionIntegrator::AssembleMF only implemented"

fem/bilininteg_vecmass.cpp

@@ -34,16 +34,7 @@ void VectorMassIntegrator::AssemblePA(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedPAMassIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::PAMassIntegrator(fes, *ir, Q);
       return;
    }
    dim = mesh->Dimension();

fem/bilininteg_vecmass_mf.cpp

@@ -34,16 +34,7 @@ void VectorMassIntegrator::AssembleMF(const FiniteElementSpace &fes)
    if (DeviceCanUseCeed())
    {
       delete ceedOp;
-      const bool mixed = mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
-                         fes.IsVariableOrder();
-      if (mixed)
-      {
-         ceedOp = new ceed::MixedMFMassIntegrator(*this, fes, Q);
-      }
-      else
-      {
-         ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
-      }
+      ceedOp = new ceed::MFMassIntegrator(fes, *ir, Q);
       return;
    }
    MFEM_ABORT("Error: VectorMassIntegrator::AssembleMF only implemented with"

fem/bilininteg_vectorfe.cpp

@@ -11,8 +11,6 @@
 
 #include "../general/forall.hpp"
 #include "bilininteg.hpp"
-#include "qspace.hpp"
-#include "gridfunc.hpp"
 
 namespace mfem
 {
@@ -91,7 +89,6 @@ void SmemPAHcurlMassApply3D(const int D1D,
                             Vector &y);
 
 void PAHdivSetup2D(const int Q1D,
-                   const int coeffDim,
                    const int NE,
                    const Array<double> &w,
                    const Vector &j,
@@ -99,7 +96,6 @@ void PAHdivSetup2D(const int Q1D,
                    Vector &op);
 
 void PAHdivSetup3D(const int Q1D,
-                   const int coeffDim,
                    const int NE,
                    const Array<double> &w,
                    const Vector &j,
@@ -153,7 +149,6 @@ void PAHcurlH1ApplyTranspose3D(const int D1D,
 void PAHdivMassAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
                                   const int NE,
-                                  const bool symmetric,
                                   const Array<double> &Bo_,
                                   const Array<double> &Bc_,
                                   const Vector &op_,
@@ -162,24 +157,32 @@ void PAHdivMassAssembleDiagonal2D(const int D1D,
 void PAHdivMassAssembleDiagonal3D(const int D1D,
                                   const int Q1D,
                                   const int NE,
-                                  const bool symmetric,
                                   const Array<double> &Bo_,
                                   const Array<double> &Bc_,
                                   const Vector &op_,
                                   Vector &diag_);
 
-void PAHdivMassApply(const int dim,
-                     const int D1D,
-                     const int Q1D,
-                     const int NE,
-                     const bool symmetric,
-                     const Array<double> &Bo,
-                     const Array<double> &Bc,
-                     const Array<double> &Bot,
-                     const Array<double> &Bct,
-                     const Vector &op,
-                     const Vector &x,
-                     Vector &y);
+void PAHdivMassApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &Bo_,
+                       const Array<double> &Bc_,
+                       const Array<double> &Bot_,
+                       const Array<double> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
+
+void PAHdivMassApply3D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<double> &Bo_,
+                       const Array<double> &Bc_,
+                       const Array<double> &Bot_,
+                       const Array<double> &Bct_,
+                       const Vector &op_,
+                       const Vector &x_,
+                       Vector &y_);
 
 void PAHcurlL2Setup(const int NQ,
                     const int coeffDim,
@@ -794,63 +797,129 @@ void VectorFEMassIntegrator::AssemblePA(const FiniteElementSpace &trial_fes,
    trial_fetype = trial_el->GetDerivType();
    test_fetype = test_el->GetDerivType();
 
+   auto SMQ = dynamic_cast<SymmetricMatrixCoefficient *>(MQ);
+
+   const int MQsymmDim = SMQ ? (SMQ->GetSize() * (SMQ->GetSize() + 1)) / 2 : 0;
+   const int MQfullDim = MQ ? (MQ->GetHeight() * MQ->GetWidth()) : 0;
+   const int MQdim = SMQ ? MQsymmDim : MQfullDim;
+   const int coeffDim = MQ ? MQdim : (DQ ? DQ->GetVDim() : 1);
+
+   symmetric = (SMQ || MQ == NULL);
+
    const bool trial_curl = (trial_fetype == mfem::FiniteElement::CURL);
    const bool trial_div = (trial_fetype == mfem::FiniteElement::DIV);
    const bool test_curl = (test_fetype == mfem::FiniteElement::CURL);
    const bool test_div = (test_fetype == mfem::FiniteElement::DIV);
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(qs, CoefficientStorage::SYMMETRIC);
-   if (Q) { coeff.Project(*Q); }
-   else if (MQ) { coeff.ProjectTranspose(*MQ); }
-   else if (DQ) { coeff.Project(*DQ); }
-   else { coeff.SetConstant(1.0); }
-
-   const int coeff_dim = coeff.GetVDim();
-   symmetric = (coeff_dim != dim*dim);
-
    if ((trial_curl && test_div) || (trial_div && test_curl))
-      pa_data.SetSize((coeff_dim == 1 ? 1 : dim*dim) * nq * ne,
+      pa_data.SetSize((coeffDim == 1 ? 1 : dim*dim) * nq * ne,
                       Device::GetMemoryType());
    else
-      pa_data.SetSize((symmetric ? symmDims : dims*dims) * nq * ne,
+      pa_data.SetSize((symmetric ? symmDims : MQfullDim) * nq * ne,
                       Device::GetMemoryType());
 
+   Vector coeff(coeffDim * ne * nq);
+   coeff = 1.0;
+   auto coeffh = Reshape(coeff.HostWrite(), coeffDim, nq, ne);
+   if (Q || DQ || MQ)
+   {
+      Vector DM(DQ ? coeffDim : 0);
+      DenseMatrix M;
+      DenseSymmetricMatrix SM;
+
+      if (DQ)
+      {
+         MFEM_VERIFY(coeffDim == dim, "");
+      }
+      if (SMQ)
+      {
+         MFEM_VERIFY(SMQ->GetSize() == dim, "");
+         SM.SetSize(dim);
+      }
+      else if (MQ)
+      {
+         MFEM_VERIFY(coeffDim == MQdim, "");
+         MFEM_VERIFY(MQ->GetHeight() == dim && MQ->GetWidth() == dim, "");
+         M.SetSize(dim);
+      }
+
+
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            if (SMQ)
+            {
+               SMQ->Eval(SM, *tr, ir->IntPoint(p));
+               int cnt = 0;
+               for (int i=0; i<dim; ++i)
+                  for (int j=i; j<dim; ++j, ++cnt)
+                  {
+                     coeffh(cnt, p, e) = SM(i,j);
+                  }
+            }
+            else if (MQ)
+            {
+               MQ->Eval(M, *tr, ir->IntPoint(p));
+
+               for (int i=0; i<dim; ++i)
+                  for (int j=0; j<dim; ++j)
+                  {
+                     coeffh(j+(i*dim), p, e) = M(i,j);
+                  }
+            }
+            else if (DQ)
+            {
+               DQ->Eval(DM, *tr, ir->IntPoint(p));
+               for (int i=0; i<coeffDim; ++i)
+               {
+                  coeffh(i, p, e) = DM[i];
+               }
+            }
+            else
+            {
+               coeffh(0, p, e) = Q->Eval(*tr, ir->IntPoint(p));
+            }
+         }
+      }
+   }
+
    if (trial_curl && test_curl && dim == 3)
    {
-      PADiffusionSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup3D(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                          coeff, pa_data);
    }
    else if (trial_curl && test_curl && dim == 2)
    {
-      PADiffusionSetup2D<2>(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PADiffusionSetup2D<2>(quad1D, coeffDim, ne, ir->GetWeights(), geom->J,
                             coeff, pa_data);
    }
    else if (trial_div && test_div && dim == 3)
    {
-      PAHdivSetup3D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup3D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
    else if (trial_div && test_div && dim == 2)
    {
-      PAHdivSetup2D(quad1D, coeff_dim, ne, ir->GetWeights(), geom->J,
+      PAHdivSetup2D(quad1D, ne, ir->GetWeights(), geom->J,
                     coeff, pa_data);
    }
    else if (((trial_curl && test_div) || (trial_div && test_curl)) &&
             test_fel->GetOrder() == trial_fel->GetOrder())
    {
-      if (coeff_dim == 1)
+      if (coeffDim == 1)
       {
-         PAHcurlL2Setup(nq, coeff_dim, ne, ir->GetWeights(), coeff, pa_data);
+         PAHcurlL2Setup(nq, coeffDim, ne, ir->GetWeights(), coeff, pa_data);
       }
       else
       {
          const bool tr = (trial_div && test_curl);
          if (dim == 3)
-            PAHcurlHdivSetup3D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup3D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
                                geom->J, coeff, pa_data);
          else
-            PAHcurlHdivSetup2D(quad1D, coeff_dim, ne, tr, ir->GetWeights(),
+            PAHcurlHdivSetup2D(quad1D, coeffDim, ne, tr, ir->GetWeights(),
                                geom->J, coeff, pa_data);
       }
    }
@@ -894,7 +963,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
       else if (trial_fetype == mfem::FiniteElement::DIV &&
                test_fetype == trial_fetype)
       {
-         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne, symmetric,
+         PAHdivMassAssembleDiagonal3D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
       }
       else
@@ -902,7 +971,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
          MFEM_ABORT("Unknown kernel.");
       }
    }
-   else // 2D
+   else
    {
       if (trial_fetype == mfem::FiniteElement::CURL && test_fetype == trial_fetype)
       {
@@ -912,7 +981,7 @@ void VectorFEMassIntegrator::AssembleDiagonalPA(Vector& diag)
       else if (trial_fetype == mfem::FiniteElement::DIV &&
                test_fetype == trial_fetype)
       {
-         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne, symmetric,
+         PAHdivMassAssembleDiagonal2D(dofs1D, quad1D, ne,
                                       mapsO->B, mapsC->B, pa_data, diag);
       }
       else
@@ -965,8 +1034,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
       }
       else if (trial_div && test_div)
       {
-         PAHdivMassApply(3, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
-                         mapsC->Bt, pa_data, x, y);
+         PAHdivMassApply3D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
+                           mapsC->Bt, pa_data, x, y);
       }
       else if (trial_curl && test_div)
       {
@@ -987,7 +1056,7 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
          MFEM_ABORT("Unknown kernel.");
       }
    }
-   else // 2D
+   else
    {
       if (trial_curl && test_curl)
       {
@@ -996,8 +1065,8 @@ void VectorFEMassIntegrator::AddMultPA(const Vector &x, Vector &y) const
       }
       else if (trial_div && test_div)
       {
-         PAHdivMassApply(2, dofs1D, quad1D, ne, symmetric, mapsO->B, mapsC->B, mapsO->Bt,
-                         mapsC->Bt, pa_data, x, y);
+         PAHdivMassApply2D(dofs1D, quad1D, ne, mapsO->B, mapsC->B, mapsO->Bt,
+                           mapsC->Bt, pa_data, x, y);
       }
       else if ((trial_curl && test_div) || (trial_div && test_curl))
       {
@@ -1042,11 +1111,6 @@ void VectorFEMassIntegrator::AddMultTransposePA(const Vector &x,
 
    if (symmetricSpaces)
    {
-      if (MQ && dynamic_cast<SymmetricMatrixCoefficient*>(MQ) == NULL)
-      {
-         MFEM_ABORT("VectorFEMassIntegrator transpose not implemented for asymmetric MatrixCoefficient");
-      }
-
       this->AddMultPA(x, y);
    }
 }
@@ -1092,8 +1156,19 @@ void MixedVectorGradientIntegrator::AssemblePA(const FiniteElementSpace
 
    pa_data.SetSize(symmDims * nq * ne, Device::GetMemoryType());
 
-   QuadratureSpace qs(*mesh, *ir);
-   CoefficientVector coeff(Q, qs, CoefficientStorage::FULL);
+   Vector coeff(ne * nq);
+   coeff = 1.0;
+   if (Q)
+   {
+      for (int e=0; e<ne; ++e)
+      {
+         ElementTransformation *tr = mesh->GetElementTransformation(e);
+         for (int p=0; p<nq; ++p)
+         {
+            coeff[p + (e * nq)] = Q->Eval(*tr, ir->IntPoint(p));
+         }
+      }
+   }
 
    // Use the same setup functions as VectorFEMassIntegrator.
    if (test_el->GetDerivType() == mfem::FiniteElement::CURL && dim == 3)

fem/ceed/integrators/convection/convection.cpp

@@ -62,20 +62,6 @@ PAConvectionIntegrator::PAConvectionIntegrator(
 #endif
 }
 
-MixedPAConvectionIntegrator::MixedPAConvectionIntegrator(
-   const ConvectionIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::VectorCoefficient *Q,
-   const double alpha)
-{
-#ifdef MFEM_USE_CEED
-   ConvectionOperatorInfo info(fes.GetMesh()->Dimension(), alpha);
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 MFConvectionIntegrator::MFConvectionIntegrator(
    const mfem::FiniteElementSpace &fes,
    const mfem::IntegrationRule &irm,
@@ -91,20 +77,6 @@ MFConvectionIntegrator::MFConvectionIntegrator(
 #endif
 }
 
-MixedMFConvectionIntegrator::MixedMFConvectionIntegrator(
-   const ConvectionIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::VectorCoefficient *Q,
-   const double alpha)
-{
-#ifdef MFEM_USE_CEED
-   ConvectionOperatorInfo info(fes.GetMesh()->Dimension(), alpha);
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 } // namespace ceed
 
 } // namespace mfem

fem/ceed/integrators/convection/convection.hpp

@@ -13,7 +13,6 @@
 #define MFEM_LIBCEED_CONV_HPP
 
 #include "../../interface/integrator.hpp"
-#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"
 
 namespace mfem
@@ -27,39 +26,21 @@ class PAConvectionIntegrator : public PAIntegrator
 {
 public:
    PAConvectionIntegrator(const mfem::FiniteElementSpace &fes,
-                          const mfem::IntegrationRule &ir,
+                          const mfem::IntegrationRule &irm,
                           mfem::VectorCoefficient *Q,
                           const double alpha);
 };
 
-class MixedPAConvectionIntegrator : public MixedIntegrator<PAIntegrator>
-{
-public:
-   MixedPAConvectionIntegrator(const ConvectionIntegrator &integ,
-                               const mfem::FiniteElementSpace &fes,
-                               mfem::VectorCoefficient *Q,
-                               const double alpha);
-};
-
 /// Represent a ConvectionIntegrator with AssemblyLevel::None using libCEED.
 class MFConvectionIntegrator : public MFIntegrator
 {
 public:
    MFConvectionIntegrator(const mfem::FiniteElementSpace &fes,
-                          const mfem::IntegrationRule &ir,
+                          const mfem::IntegrationRule &irm,
                           mfem::VectorCoefficient *Q,
                           const double alpha);
 };
 
-class MixedMFConvectionIntegrator : public MixedIntegrator<MFIntegrator>
-{
-public:
-   MixedMFConvectionIntegrator(const ConvectionIntegrator &integ,
-                               const mfem::FiniteElementSpace &fes,
-                               mfem::VectorCoefficient *Q,
-                               const double alpha);
-};
-
 }
 
 }

fem/ceed/integrators/diffusion/diffusion.cpp

@@ -60,32 +60,6 @@ PADiffusionIntegrator::PADiffusionIntegrator(
 #endif
 }
 
-MixedPADiffusionIntegrator::MixedPADiffusionIntegrator(
-   const DiffusionIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-MixedPADiffusionIntegrator::MixedPADiffusionIntegrator(
-   const VectorDiffusionIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 MFDiffusionIntegrator::MFDiffusionIntegrator(
    const mfem::FiniteElementSpace &fes,
    const mfem::IntegrationRule &irm,
@@ -100,32 +74,6 @@ MFDiffusionIntegrator::MFDiffusionIntegrator(
 #endif
 }
 
-MixedMFDiffusionIntegrator::MixedMFDiffusionIntegrator(
-   const DiffusionIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-MixedMFDiffusionIntegrator::MixedMFDiffusionIntegrator(
-   const VectorDiffusionIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   DiffusionOperatorInfo info(fes.GetMesh()->Dimension());
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 } // namespace ceed
 
 } // namespace mfem

fem/ceed/integrators/diffusion/diffusion.hpp

@@ -13,7 +13,6 @@
 #define MFEM_LIBCEED_DIFF_HPP
 
 #include "../../interface/integrator.hpp"
-#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"
 
 namespace mfem
@@ -27,43 +26,19 @@ class PADiffusionIntegrator : public PAIntegrator
 {
 public:
    PADiffusionIntegrator(const mfem::FiniteElementSpace &fes,
-                         const mfem::IntegrationRule &ir,
+                         const mfem::IntegrationRule &irm,
                          mfem::Coefficient *Q);
 };
 
-class MixedPADiffusionIntegrator : public MixedIntegrator<PAIntegrator>
-{
-public:
-   MixedPADiffusionIntegrator(const DiffusionIntegrator &integ,
-                              const mfem::FiniteElementSpace &fes,
-                              mfem::Coefficient *Q);
-
-   MixedPADiffusionIntegrator(const VectorDiffusionIntegrator &integ,
-                              const mfem::FiniteElementSpace &fes,
-                              mfem::Coefficient *Q);
-};
-
 /// Represent a DiffusionIntegrator with AssemblyLevel::None using libCEED.
 class MFDiffusionIntegrator : public MFIntegrator
 {
 public:
    MFDiffusionIntegrator(const mfem::FiniteElementSpace &fes,
-                         const mfem::IntegrationRule &ir,
+                         const mfem::IntegrationRule &irm,
                          mfem::Coefficient *Q);
 };
 
-class MixedMFDiffusionIntegrator : public MixedIntegrator<MFIntegrator>
-{
-public:
-   MixedMFDiffusionIntegrator(const DiffusionIntegrator &integ,
-                              const mfem::FiniteElementSpace &fes,
-                              mfem::Coefficient *Q);
-
-   MixedMFDiffusionIntegrator(const VectorDiffusionIntegrator &integ,
-                              const mfem::FiniteElementSpace &fes,
-                              mfem::Coefficient *Q);
-};
-
 }
 
 }

fem/ceed/integrators/mass/mass.cpp

@@ -59,30 +59,6 @@ PAMassIntegrator::PAMassIntegrator(const mfem::FiniteElementSpace &fes,
 #endif
 }
 
-MixedPAMassIntegrator::MixedPAMassIntegrator(const MassIntegrator &integ,
-                                             const mfem::FiniteElementSpace &fes,
-                                             mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   MassOperatorInfo info;
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-MixedPAMassIntegrator::MixedPAMassIntegrator(const VectorMassIntegrator &integ,
-                                             const mfem::FiniteElementSpace &fes,
-                                             mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   MassOperatorInfo info;
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
                                    const mfem::IntegrationRule &irm,
                                    mfem::Coefficient *Q)
@@ -96,30 +72,6 @@ MFMassIntegrator::MFMassIntegrator(const mfem::FiniteElementSpace &fes,
 #endif
 }
 
-MixedMFMassIntegrator::MixedMFMassIntegrator(const MassIntegrator &integ,
-                                             const mfem::FiniteElementSpace &fes,
-                                             mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   MassOperatorInfo info;
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
-MixedMFMassIntegrator::MixedMFMassIntegrator(const VectorMassIntegrator &integ,
-                                             const mfem::FiniteElementSpace &fes,
-                                             mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   MassOperatorInfo info;
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 } // namespace ceed
 
 } // namespace mfem

fem/ceed/integrators/mass/mass.hpp

@@ -13,7 +13,6 @@
 #define MFEM_LIBCEED_MASS_HPP
 
 #include "../../interface/integrator.hpp"
-#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"
 
 namespace mfem
@@ -27,43 +26,19 @@ class PAMassIntegrator : public PAIntegrator
 {
 public:
    PAMassIntegrator(const mfem::FiniteElementSpace &fes,
-                    const mfem::IntegrationRule &ir,
+                    const mfem::IntegrationRule &irm,
                     mfem::Coefficient *Q);
 };
 
-class MixedPAMassIntegrator : public MixedIntegrator<PAIntegrator>
-{
-public:
-   MixedPAMassIntegrator(const MassIntegrator &integ,
-                         const mfem::FiniteElementSpace &fes,
-                         mfem::Coefficient *Q);
-
-   MixedPAMassIntegrator(const VectorMassIntegrator &integ,
-                         const mfem::FiniteElementSpace &fes,
-                         mfem::Coefficient *Q);
-};
-
 /// Represent a MassIntegrator with AssemblyLevel::None using libCEED.
 class MFMassIntegrator : public MFIntegrator
 {
 public:
    MFMassIntegrator(const mfem::FiniteElementSpace &fes,
-                    const mfem::IntegrationRule &ir,
+                    const mfem::IntegrationRule &irm,
                     mfem::Coefficient *Q);
 };
 
-class MixedMFMassIntegrator : public MixedIntegrator<MFIntegrator>
-{
-public:
-   MixedMFMassIntegrator(const MassIntegrator &integ,
-                         const mfem::FiniteElementSpace &fes,
-                         mfem::Coefficient *Q);
-
-   MixedMFMassIntegrator(const VectorMassIntegrator &integ,
-                         const mfem::FiniteElementSpace &fes,
-                         mfem::Coefficient *Q);
-};
-
 }
 
 }

fem/ceed/integrators/nlconvection/nlconvection.cpp

@@ -60,19 +60,6 @@ PAVectorConvectionNLFIntegrator::PAVectorConvectionNLFIntegrator(
 #endif
 }
 
-MixedPAVectorConvectionNLIntegrator::MixedPAVectorConvectionNLIntegrator(
-   const VectorConvectionNLFIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   NLConvectionOperatorInfo info(fes.GetMesh()->Dimension());
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 MFVectorConvectionNLFIntegrator::MFVectorConvectionNLFIntegrator(
    const mfem::FiniteElementSpace &fes,
    const mfem::IntegrationRule &irm,
@@ -87,19 +74,6 @@ MFVectorConvectionNLFIntegrator::MFVectorConvectionNLFIntegrator(
 #endif
 }
 
-MixedMFVectorConvectionNLIntegrator::MixedMFVectorConvectionNLIntegrator(
-   const VectorConvectionNLFIntegrator &integ,
-   const mfem::FiniteElementSpace &fes,
-   mfem::Coefficient *Q)
-{
-#ifdef MFEM_USE_CEED
-   NLConvectionOperatorInfo info(fes.GetMesh()->Dimension());
-   Assemble(integ, info, fes, Q);
-#else
-   MFEM_ABORT("MFEM must be built with MFEM_USE_CEED=YES to use libCEED.");
-#endif
-}
-
 } // namespace ceed
 
 } // namespace mfem

fem/ceed/integrators/nlconvection/nlconvection.hpp

@@ -13,7 +13,6 @@
 #define MFEM_LIBCEED_NLCONV_HPP
 
 #include "../../interface/integrator.hpp"
-#include "../../interface/mixed_integrator.hpp"
 #include "../../../fespace.hpp"
 
 namespace mfem
@@ -32,15 +31,6 @@ public:
                                    mfem::Coefficient *coeff);
 };
 
-class MixedPAVectorConvectionNLIntegrator : public MixedIntegrator<PAIntegrator>
-{
-public:
-   MixedPAVectorConvectionNLIntegrator(
-      const VectorConvectionNLFIntegrator &integ,
-      const mfem::FiniteElementSpace &fes,
-      mfem::Coefficient *Q);
-};
-
 /** Represent a VectorConvectionNLFIntegrator with AssemblyLevel::None
     using libCEED. */
 class MFVectorConvectionNLFIntegrator : public MFIntegrator
@@ -51,15 +41,6 @@ public:
                                    mfem::Coefficient *coeff);
 };
 
-class MixedMFVectorConvectionNLIntegrator : public MixedIntegrator<MFIntegrator>
-{
-public:
-   MixedMFVectorConvectionNLIntegrator(
-      const VectorConvectionNLFIntegrator &integ,
-      const mfem::FiniteElementSpace &fes,
-      mfem::Coefficient *Q);
-};
-
 }
 
 }