Fix a small issue in miniapps/shifted/distance.cpp

in supported cases.

.codecov.yml

@@ -12,7 +12,7 @@ coverage:
         threshold: 0%
         base: auto
         branches:
-          - main
+          - master
         if_ci_failed: error
         informational: true
         only_pulls: true
@@ -22,7 +22,7 @@ coverage:
         threshold: 1%  # allows variations around the target
         base: auto
         branches:
-          - main
+          - master
         if_ci_failed: error
         only_pulls: true
 

.github/workflows/builds-and-tests.yml

@@ -132,14 +132,12 @@ jobs:
             hypre-target: int32
             precision: fp64
             enzyme: true
-            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme) LDFLAGS=-L$LLVM_PREFIX/lib/c++
+            config-opts: MFEM_USE_ENZYME=YES ENZYME_DIR=$(brew --prefix enzyme)
 
     name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}${{ matrix.enzyme && '-enzyme' || '' }}
 
     runs-on: ${{ matrix.os }}
 
-    continue-on-error: ${{ matrix.enzyme && true || false }}
-
     steps:
       # Fix 'No space left on device' errors for Ubuntu builds.
       - name: Run Actions Cleaner
@@ -170,13 +168,10 @@ jobs:
           env
         shell: bash
 
-      # For info on Xcode see:
-      #  - https://github.com/actions/runner-images/issues/12541
-      #  - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
       - name: Xcode version setup (MacOS)
         if: matrix.os == 'macos-latest'
         run: |
-          XCODE_PATH="/Applications/Xcode_16.4.app"
+          XCODE_PATH="/Applications/Xcode_15.3.app"
           echo "> sudo xcode-select -s ${XCODE_PATH}"
           sudo xcode-select -s ${XCODE_PATH}
           echo "> g++ -v"
@@ -294,12 +289,10 @@ jobs:
         run: |
           export HOMEBREW_NO_INSTALL_CLEANUP=1
           brew update
-          brew install enzyme
-          ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
-          LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
-          echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
-          echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
-          echo "OMPI_CXX=$LLVM_PREFIX/bin/clang++" >> $GITHUB_ENV
+          brew install llvm@20 enzyme
+          echo "LLVM_PREFIX=$(brew --prefix llvm@20)" >> $GITHUB_ENV
+          echo "OMPI_CC=$(brew --prefix llvm@20)/bin/clang" >> $GITHUB_ENV
+          echo "OMPI_CXX=$(brew --prefix llvm@20)/bin/clang++" >> $GITHUB_ENV
 
       # MFEM build and test
       - name: build

.github/workflows/stale.yml

@@ -29,47 +29,3 @@ jobs:
         operations-per-run: 500
         exempt-issue-labels: "bug,WIP,ready-for-review,in-review,in-next"
         exempt-pr-labels: "bug,WIP,ready-for-review,in-review,in-next"
-
-  # Stale action for PRs with "in-review" label.
-  stale-in-review-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 150 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "in-review"
-        days-before-pr-stale: 150
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500
-
-  # Stale action for PRs with "WIP" label.
-  stale-wip-pr:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-      actions: write
-
-    steps:
-    - uses: actions/stale@v9
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: ':warning: This PR has been automatically marked as stale because it has not had any activity in the last 300 days. *If no activity occurs in the next 30 days, it will be automatically closed.* Thank you for your contributions.'
-        only-pr-labels: "WIP"
-        days-before-pr-stale: 300
-        days-before-pr-close: 30
-        days-before-issue-stale: -1
-        days-before-issue-close: -1
-        stale-pr-label: 'stale'
-        operations-per-run: 500

.gitignore

@@ -208,13 +208,10 @@ miniapps/electromagnetics/volta
 miniapps/electromagnetics/tesla
 miniapps/electromagnetics/maxwell
 miniapps/electromagnetics/joule
-miniapps/electromagnetics/lorentz
 miniapps/electromagnetics/Volta-AMR*
 miniapps/electromagnetics/Tesla-AMR*
 miniapps/electromagnetics/Maxwell-Parallel*
 miniapps/electromagnetics/Joule_[0-9]*
-miniapps/electromagnetics/Lorentz_[0-9]*
-miniapps/electromagnetics/Lorentz.dat
 
 miniapps/gslib/field-diff
 miniapps/gslib/field-interp
@@ -414,6 +411,8 @@ miniapps/tribol/contact-patch-test
 miniapps/diag-smoothers/abs-l1-jacobi
 miniapps/diag-smoothers/mg-abs-l1-jacobi
 
+miniapps/benchmarks/ceed-solver-bps/solver-bp
+
 # Unit test binary and outputs
 tests/unit/output_meshes
 tests/unit/unit_tests

CHANGELOG

@@ -27,29 +27,9 @@ Discretization improvements
 - In the ParMoonolith integration, added support for variational resampling of
   H1 vector fields.
 
-- Added support for boundary integration to the hyperbolic framework. In this
-  regard, new classes `BdrHyperbolicDirichletIntegrator` and
-  `BoundaryHyperbolicFlowIntegrator` have been introduced for implementation
-  of weak Dirichlet boundary conditions with a general flux or for the linear
-  case respectively.
-
-- Added method to compute piecewise linear bounds on high-order functions on
-  tensor-product elements.
-
-- Parallel anisotropic refinement of hexahedral meshes is now supported,
-  provided that neighboring hexahedra are not refined in conflicting directions.
-  A new ParMesh method is added to check for such conflicts, before refinement.
-
-- Renamed the default GitHub branch from "master" to "main".
-
 Meshing improvements
 --------------------
 
-- Introduced NC-patch NURBS meshes, which are conforming element-wise but allow
-  for nonconforming patch topology. This new mesh format supports element
-  spacing formulas for refinement, as well as local refinement factors for a
-  subset of knot vectors.
-
 - Added support for higher order meshes in Mesh::MakeSimplicial and
   ParMesh::MakeSimplicial.
 
@@ -64,19 +44,13 @@ GPU computing
   set. This is most often used for setting constant essential boundary
   conditions. A new function Vector::SetSubVectorHost has been added in cases
   where host execution is always needed (e.g. when the DOFs array is small).
-
 - Introduced MFEM_FOREACH_THREAD_DIRECT, which directly maps loop tasks to GPU
   threads, assigning one task per thread.
-
 - Implemented a GPU-accelerated matrix-free AMR derefinement `GridFunction`
   update operator. This supports mixed geometry meshes and variable order
   spaces, and is the default derefinement operator constructed by
   `FiniteElementSpace::Update` and `ParFiniteElementSpace::Update`.
   The operator requires `FiniteElementSpace::Nonconforming() == true`.
-- Added new method: GridFunction::GetGradients, with GPU support, for computing
-  the gradients of a GridFunction on all elements.
-- Added GPU support in GradientGridFunctionCoefficient and
-  InnerProductCoefficient by implementing their Project methods.
 
 New and updated examples and miniapps
 -------------------------------------
@@ -86,22 +60,10 @@ New and updated examples and miniapps
   operators as smoothers.
   These miniapps can be found in `miniapps/diag-smoothers`.
 
-- Added a new miniapp (meshing/mesh-bounding-boxes) that computes the bounding
-  boxes for each element of a given mesh, and the bounds on the determinant of
-  the Jacobian of the transformation.
-
-- Added a new miniapp (tools/gridfunction-bounds) to compute piecewise linear
-  bounds on a given high-order grid function.
-
-- Added a new miniapp (electromagnetics/lorentz) which computes the trajectory
-  of a charged particle, subject to Lorentz forces, in electrostatic and/or
-  magnetostatic fields as computed by the volta or tesla miniapps.
-
-API changes:
+API changes
 -----------
 - mfem::internal::tensor and mfem::internal::dual have been moved to
   mfem::future::tensor and mfem::future::dual.
-
 - API addition: in class `Operator`, added virtual functions: `AbsMult`, and
   `AbsMultTranspose`; in class `Vector`, added `Abs` and `Pow`.
 
@@ -109,24 +71,22 @@ Miscellaneous
 -------------
 - Added the "gpu", "raja-gpu", and "ceed-gpu" backend aliases/shortcuts which
   automatically select between CUDA or HIP.
-
 - The CUDA-specific names used by some of the unit tests like 'cunit_tests' and
   'pcunit_tests' were replaced by names using 'gpu' instead of 'c' (short for
   CUDA) or 'cuda'. These tests automatically run the CUDA/HIP tests based on the
   MFEM build configuration.
-
 - Added the option to enable GPU-aware MPI in MFEM using the environment
   variable 'MFEM_GPU_AWARE_MPI' set to any value. Setting this environment
   variable is an alternative to calling 'Device::SetGPUAwareMPI(true)'.
-
 - Added parallel Address Sanitizer, serial and parallel Undefined Behavior
   Sanitizer and serial Memory Sanitizer GitHub actions tests on Ubuntu.
+- MFEM_PERF_* annotations: added options to enable GPU-stream- and
+  MPI-synchronizations at the start and at the end of annotation regions. These
+  synchronizations can be enabled or disabled (default) in code via the new
+  macros: MFEM_PERF_SYNC, MFEM_PERF_SYNC_STREAM, and MFEM_PERF_SYNC_MPI; the
+  environment variables with the same names can be set to 0/1 to control the
+  synchronization as well.
 
-- FindPointsGSLIB has a new constructor that accepts the mesh object and
-  internally calls the Setup() method so that the user does not have to.
-  The FreeData() method has also been moved to the destructor so the user does
-  not need to manually free-up the memory if the destructor is called before
-  MPI_Finalize().
 
 Version 4.8, released on Apr 9, 2025
 ====================================

CMakeLists.txt

@@ -278,11 +278,6 @@ if (MFEM_USE_OPENMP OR MFEM_USE_LEGACY_OPENMP)
   endif()
 endif()
 
-# Umpire (must be included before hypre, so hypre can use it if needed)
-if (MFEM_USE_UMPIRE)
-   find_package(UMPIRE REQUIRED)
-endif()
-
 # MPI -> hypre; PETSc (optional)
 if (MFEM_USE_MPI)
   find_package(MPI REQUIRED)
@@ -500,13 +495,14 @@ endif()
 
 # RAJA
 if (MFEM_USE_RAJA)
-   # RAJA uses FindCUDA, which needs CMP0146=OLD in CMake >= 3.27
-   if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.27.0)
-      cmake_policy(SET CMP0146 OLD)
-   endif()
    find_package(RAJA REQUIRED)
 endif()
 
+# UMPIRE
+if (MFEM_USE_UMPIRE)
+   find_package(UMPIRE REQUIRED)
+endif()
+
 # GOOGLE-BENCHMARK
 if (MFEM_USE_BENCHMARK)
    find_package(Benchmark REQUIRED)
@@ -600,7 +596,7 @@ set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
     NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
     ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
     BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
-    ALGOIM ENZYME CUDA::cudart)
+    ALGOIM ENZYME)
 
 # Add all created targets and *_FOUND libraries in the variables TPL_TARGETS and
 # TPL_LIBRARIES, respectively.
@@ -618,7 +614,6 @@ foreach(TPL IN LISTS MFEM_TPLS)
     endif()
   endif()
 endforeach(TPL)
-
 list(REVERSE TPL_LIBRARIES)
 list(REMOVE_DUPLICATES TPL_LIBRARIES)
 list(REVERSE TPL_LIBRARIES)
@@ -652,7 +647,7 @@ if (MFEM_USE_CUDA)
 endif()
 
 add_subdirectory(config)
-set(MAIN_HEADERS
+set(MASTER_HEADERS
   ${PROJECT_SOURCE_DIR}/mfem.hpp
   ${PROJECT_SOURCE_DIR}/mfem-performance.hpp)
 
@@ -689,7 +684,7 @@ set(MFEM_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(MFEM_INSTALL_DIR ${CMAKE_INSTALL_PREFIX})
 
 # Declaring the library
-mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MAIN_HEADERS})
+mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
 # message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
 target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
 if (TPL_TARGETS)
@@ -888,12 +883,12 @@ install(TARGETS ${PROJECT_NAME}
   LIBRARY DESTINATION ${INSTALL_LIB_DIR}
   ARCHIVE DESTINATION ${INSTALL_LIB_DIR})
 
-# Install the main headers
+# Install the master headers
 foreach(Header mfem.hpp mfem-performance.hpp)
   install(FILES ${PROJECT_BINARY_DIR}/InstallHeaders/${Header}
     DESTINATION ${INSTALL_INCLUDE_DIR})
 endforeach()
-install(FILES ${MAIN_HEADERS} DESTINATION ${INSTALL_INCLUDE_DIR}/mfem)
+install(FILES ${MASTER_HEADERS} DESTINATION ${INSTALL_INCLUDE_DIR}/mfem)
 
 # Install the headers (except common miniapp which is installed from its subdir)
 install(DIRECTORY ${MFEM_SOURCE_DIRS}

CONTRIBUTING.md

@@ -3,10 +3,10 @@
 </p>
 
 <p align="center">
-<a href="https://github.com/mfem/mfem/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amain"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=main"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amain"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=main"></a>
-<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amain"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=main"></a>
+<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
+<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
 <a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
 <a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
 </p>
@@ -27,7 +27,7 @@ in the MFEM community, you agree to abide by its rules.
 If you plan on contributing to MFEM, consider reviewing the
 [issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
 already exists for your desired feature or the bug you ran into. Use a pull
-request (PR) toward the `mfem:main` branch to propose your contribution. If
+request (PR) toward the `mfem:master` branch to propose your contribution. If
 you are planning significant code changes or have questions, you may want to
 open an [issue](https://github.com/mfem/mfem/issues) before issuing a PR. In
 addition to technical contributions, we are also interested in your results and
@@ -47,7 +47,7 @@ back to them before issuing pull requests:
   - [Pull Requests](#pull-requests)
   - [MFEM PR Rules](#mfem-pr-rules)
   - [Pull Request Checklist](#pull-request-checklist)
-  - [Main/Next Workflow](#mainnext-workflow)
+  - [Master/Next Workflow](#masternext-workflow)
   - [Releases](#releases)
   - [Release Checklist](#release-checklist)
 - [LLNL Workflow](#llnl-workflow)
@@ -66,12 +66,12 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 ## Quick Summary
 
 - We encourage you to [join the MFEM organization](#mfem-organization) and create
-  development branches off `mfem:main`.
+  development branches off `mfem:master`.
 - Please follow the [developer guidelines](#developer-guidelines), in particular
   with regards to documentation and code styling.
 - Please do not commit large/binary files to the central repository (use a fork
   instead).
-- Pull requests should be issued toward `mfem:main`. Make sure
+- Pull requests should be issued toward `mfem:master`. Make sure
   to check the items off the [Pull Request Checklist](#pull-request-checklist) and
   follow the [MFEM PR Rules](#mfem-pr-rules).
 - When your contribution is fully working and ready to be reviewed, add
@@ -81,8 +81,8 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
 - The reviewers have 3 weeks to evaluate the PR and work with the author to
   fix issues and implement improvements.
 - During review there should be no force pushes/rewriting history in the branch.
-- After approval, MFEM developers merge the PR manually in the [mfem:next branch](#mainnext-workflow).
-- After a week of testing in `mfem:next`, the original PR is merged in `mfem:main`.
+- After approval, MFEM developers merge the PR manually in the [mfem:next branch](#masternext-workflow).
+- After a week of testing in `mfem:next`, the original PR is merged in `mfem:master`.
 - We use [milestones](https://github.com/mfem/mfem/milestones) to coordinate the
   work on different PRs toward a release.
 - Don't hesitate to [contact us](#contact-information) if you have any questions.
@@ -289,7 +289,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   author, is willing to work on it and be its champion.
 
 - The author creates a branch for the new feature (with suffix `-dev`), off
-  the `main` branch, or another existing feature branch, for example:
+  the `master` branch, or another existing feature branch, for example:
 
   ```
   # Clone assuming you have setup your ssh keys on GitHub:
@@ -298,8 +298,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   # Alternatively, clone using the "https" protocol:
   git clone https://github.com/mfem/mfem.git
 
-  # Create a new feature branch starting from "main":
-  git checkout main
+  # Create a new feature branch starting from "master":
+  git checkout master
   git pull
   git checkout -b feature-dev
 
@@ -375,7 +375,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
 ### Pull Requests
 
 - When your branch is ready for other developers to review / comment on
-  the code, create a pull request towards `mfem:main`.
+  the code, create a pull request towards `mfem:master`.
 
 - Pull request typically have titles like:
 
@@ -411,7 +411,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
   assigned, the PR is considered under review. To help with the review process
   there should be no force pushes/rewriting history in the branch.
 
-- After approval, the PR is [tested](#mainnext-workflow) for a week with
+- After approval, the PR is [tested](#masternext-workflow) for a week with
   other approved PRs in the `mfem:next` branch.
 
 - Consider manually running the tests in `tests/scripts` before merging in
@@ -447,7 +447,7 @@ The Pull Request (PR) approval process in MFEM is similar to the approval of pap
 
 3. A PR can be (manually) merged in the *next* branch only if 2 of the assigned reviewers have approved it and it has passed internal testing. This merge can be performed by any of the assigned reviewers or by any of the editors.
 
-4. A PR can be merged in the *main* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
+4. A PR can be merged in the *master* branch only if it has been tested successfully for a week in *next* and an editor has (optionally) taken a final look. This merge can be performed only by one of the editors.
 
 #### Responsibilities of Editors
 
@@ -470,7 +470,7 @@ The current list of MFEM editors is:
 
 5. To remind the reviewers about timely completion of their review.
 
-6. To take a final look and complete the PR merge in *main*. The final look step is optional and shouldn't take more than 3 days.
+6. To take a final look and complete the PR merge in *master*. The final look step is optional and shouldn't take more than 3 days.
 
 7. The assignment of bugfixes should be expedited proportional to their importance, e.g. in some cases the editor can assign much shorter review window.
 
@@ -492,7 +492,7 @@ Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their a
 
 5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
 
-6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *main*.
+6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
 
 7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
 
@@ -582,11 +582,11 @@ Before a PR can be merged, it should satisfy the following:
    - [ ] Update internal tests to include the new features.
 
 
-### Main/Next Workflow
+### Master/Next Workflow
 
-MFEM uses a `main`/`next`-branch workflow as described below:
+MFEM uses a `master`/`next`-branch workflow as described below:
 
-- The `main` branch should always be of release quality and changes should not
+- The `master` branch should always be of release quality and changes should not
   be merged until they have been fully tested. This branch is protected, and
   changes can only be made through pull requests.
 
@@ -613,20 +613,20 @@ MFEM uses a `main`/`next`-branch workflow as described below:
 
 - After a week of testing in `next` (excluding bugfixes), both on GitHub, as
   well as [internally](#tests-at-llnl) at LLNL, the original PR is merged into
-  `main` (provided there are no issues).
+  `master` (provided there are no issues).
 
 - After the merge, the feature branch is deleted (unless it is a long-term
   project with periodic PRs).
 
 - The `next` branch is used just for integrated testing of all PRs approved for
-  merging into `main` to verify that each works individually and that all of
+  merging into `master` to verify that each works individually and that all of
   them work as a group. This branch can be discarded at any time, though we
   typically do that only at the end of a [release cycle](#releases).
 
 
 ### Releases
 
-- Releases are just tags in the `main` branch, e.g. https://github.com/mfem/mfem/releases/tag/v3.3.2,
+- Releases are just tags in the `master` branch, e.g. https://github.com/mfem/mfem/releases/tag/v3.3.2,
   and have a version that ends in an even "patch" number, e.g. `v3.2.2` or
   `v3.4` (by convention `v3.4` is the same as `v3.4.0`.)  Between releases, the
   version ends in an odd "patch" number, e.g. `v3.3.3`.
@@ -679,19 +679,19 @@ MFEM uses a `main`/`next`-branch workflow as described below:
 
 ### Mirroring on Bitbucket
 
-- The GitHub `main` and `next` branches are mirrored to the LLNL institutional
-  Bitbucket repository as `gh-main` and `gh-next`.
+- The GitHub `master` and `next` branches are mirrored to the LLNL institutional
+  Bitbucket repository as `gh-master` and `gh-next`.
 
-- `gh-main` is merged into LLNL's internal `main` through pull requests; write
-  permissions to `main` are restricted to ensure this is the only way in which it
+- `gh-master` is merged into LLNL's internal `master` through pull requests; write
+  permissions to `master` are restricted to ensure this is the only way in which it
   gets updated.
 
 - We never push directly from LLNL to GitHub.
 
 - Versions of the code on LLNL's internal server, from most to least stable:
   - MFEM official release on mfem.org -- Most stable, tested in many apps.
-  - `mfem:main` -- Recent development version, guaranteed to work.
-  - `mfem:gh-main` -- Stable development version, passed testing, you can use
+  - `mfem:master` -- Recent development version, guaranteed to work.
+  - `mfem:gh-master` -- Stable development version, passed testing, you can use
      it to build your code between releases.
   - `mfem:gh-next` -- Bleeding-edge development version, may be broken, use at
      your own risk.
@@ -721,14 +721,14 @@ directory.
 
 
 ### Linux and Mac smoke tests
-We use GitHub Actions to drive the default tests on the `main` and `next`
+We use GitHub Actions to drive the default tests on the `master` and `next`
 branches. See the `.github/workflows` files and the logs at
 [https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
 
 Testing using GitHub Actions should be kept lightweight, as there is a time
 constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
 
-- Tests on the `main` branch are triggered whenever a PR is issued on this branch.
+- Tests on the `master` branch are triggered whenever a PR is issued on this branch.
 - Tests on the `next` branch are currently scheduled to run each night.
 
 
@@ -744,7 +744,7 @@ and debug build is performed with a simple run of `ex1` to verify the executable
 
 ### Tests at LLNL
 
-- We mirror the `main` and `next` branches internally (to `gh-main` and
+- We mirror the `master` and `next` branches internally (to `gh-master` and
   `gh-next`) and run longer nightly tests via cron. On the weekends, a more
   extensive test is run which extracts and executes all the different sample
   runs from each example and most miniapps.

INSTALL

@@ -859,7 +859,7 @@ The specific libraries and their options are:
   URL: https://github.com/CEED/libCEED
        https://ceed.exascaleproject.org/libceed
   Options: CEED_DIR, CEED_OPT, CEED_LIB.
-  Versions: libCEED >= 0.12.0.
+  Versions: libCEED >= 0.12.
 
 - RAJA (optional), used when MFEM_USE_RAJA = YES.
   Beginning with MFEM v4.5.1, only RAJA v2022.10.3+ is supported.

config/cmake/MFEMConfig.cmake.in

@@ -84,31 +84,6 @@ set_and_check(MFEM_LIBRARY_DIR "@PACKAGE_LIB_INSTALL_DIR@")
 
 check_required_components(MFEM)
 
-include(CMakeFindDependencyMacro)
-
-if (MFEM_USE_CUDA)
-  # required for projects linking to MFEM+CUDA, even if they don't use CUDA directly
-  find_dependency(CUDAToolkit)
-endif (MFEM_USE_CUDA)
-
-if (MFEM_USE_HIP)
-  # hip/rocm uses the modern MFEM way of linking to targets, need to find dependencies
-  find_dependency(HIP)
-  find_dependency(HIPBLAS)
-  find_dependency(HIPSPARSE)
-  if (MFEM_USE_MPI)
-    # assume HYPRE uses HIP
-    # alternatively could check HYPRE_USING_HIP
-    find_dependency(rocsparse)
-    find_dependency(rocrand)
-    find_dependency(rocsolver)
-  endif (MFEM_USE_MPI)
-endif (MFEM_USE_HIP)
-
-if (MFEM_USE_RAJA)
-   find_dependency(RAJA)
-endif()
-
 if (NOT TARGET mfem)
   include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
 endif (NOT TARGET mfem)

config/cmake/modules/FindHYPRE.cmake

@@ -27,7 +27,6 @@ if (HYPRE_FOUND OR TARGET HYPRE)
   if (HYPRE_USING_HIP)
     find_package(rocsparse REQUIRED)
     find_package(rocrand REQUIRED)
-    find_package(rocsolver REQUIRED)
   endif()
   if (HYPRE_LIBRARIES AND HYPRE_INCLUDE_DIRS AND HYPRE_VERSION)
     find_package_handle_standard_args(HYPRE
@@ -38,91 +37,51 @@ if (HYPRE_FOUND OR TARGET HYPRE)
 endif()
 
 if (HYPRE_FETCH OR FETCH_TPLS)
-  # Collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
-  set(HYPRE_CMAKE_OPTIONS "")
-  get_cmake_property(all_vars VARIABLES)
-  foreach(var ${all_vars})
-    if(var MATCHES "^HYPRE_ENABLE")
-      list(APPEND HYPRE_CMAKE_OPTIONS "-D${var}:BOOL=${${var}}")
-    endif()
-  endforeach()
-
   set(HYPRE_FETCH_VERSION 2.33.0)
-  set(HYPRE_FETCH_TAG "v${HYPRE_FETCH_VERSION}" CACHE STRING "Tag, branch, or commit for HYPRE")
   add_library(HYPRE STATIC IMPORTED)
   # set options and associated dependencies
-  list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
+  set(CMAKE_OPTIONS)
+  list(APPEND CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
   if (MFEM_USE_CUDA)
-    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_CUDA:BOOL=ON -DCMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES})
+    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_CUDA:BOOL=ON)
     find_package(CUDAToolkit REQUIRED)
     target_link_libraries(HYPRE INTERFACE CUDA::cusparse CUDA::curand CUDA::cublas)
   elseif (MFEM_USE_HIP)
-    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_HIP:BOOL=ON)
+    list(APPEND CMAKE_OPTIONS -DHYPRE_WITH_HIP:BOOL=ON)
     find_package(rocsparse REQUIRED)
     find_package(rocrand REQUIRED)
     target_link_libraries(HYPRE INTERFACE rocsparse rocrand)
   endif()
-  if (MFEM_USE_CUDA OR MFEM_USE_HIP)
-    if (MFEM_USE_UMPIRE)
-      if (EXISTS ${umpire_DIR})
-        list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=ON -Dumpire_DIR:PATH=${umpire_DIR})
-      else()
-        message(FATAL_ERROR "MFEM_USE_UMPIRE=ON, however umpire_DIR isn't visible to HYPRE")
-      endif()
-    else()
-      list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_UMPIRE:BOOL=OFF)
-      message(WARNING
-"================================================================================
- Umpire is disabled while building HYPRE with GPU support.
- This is not recommended for performance reasons!
- Consider enabling Umpire with -DMFEM_USE_UMPIRE=ON and providing -DUMPIRE_DIR.
-================================================================================")
-    endif()
-  endif()
   if (MFEM_USE_SINGLE)
-    list(APPEND HYPRE_CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
+    list(APPEND CMAKE_OPTIONS -DHYPRE_ENABLE_SINGLE:BOOL=ON)
   endif()
   # define external project and create future include directory so it is present
   # to pass CMake checks at end of MFEM configuration step
-  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_TAG} to be built with ${HYPRE_CMAKE_OPTIONS}")
-  set(HYPRE_INSTALL ${CMAKE_BINARY_DIR}/fetch/hypre)
+  message(STATUS "Will fetch HYPRE ${HYPRE_FETCH_VERSION} to be built with ${CMAKE_OPTIONS}")
+  set(PREFIX ${CMAKE_BINARY_DIR}/fetch/hypre)
   include(ExternalProject)
   ExternalProject_Add(hypre
     GIT_REPOSITORY https://github.com/hypre-space/hypre.git
-    GIT_TAG ${HYPRE_FETCH_TAG}
+    GIT_TAG v${HYPRE_FETCH_VERSION}
     GIT_SHALLOW TRUE
-    GIT_PROGRESS TRUE
     UPDATE_DISCONNECTED TRUE
     SOURCE_SUBDIR src
-    PREFIX ${HYPRE_INSTALL}
-    BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
-    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
-  file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
+    PREFIX ${PREFIX}
+    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PREFIX} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${CMAKE_OPTIONS})
+  file(MAKE_DIRECTORY ${PREFIX}/include)
   # set imported library target properties
   add_dependencies(HYPRE hypre)
   set_target_properties(HYPRE PROPERTIES
-    IMPORTED_LOCATION ${HYPRE_INSTALL}/lib/libHYPRE.a
-    INTERFACE_INCLUDE_DIRECTORIES ${HYPRE_INSTALL}/include)
+    IMPORTED_LOCATION ${PREFIX}/lib/libHYPRE.a
+    INTERFACE_INCLUDE_DIRECTORIES ${PREFIX}/include)
   # convert HYPRE version to integer
-  if (HYPRE_FETCH_TAG MATCHES "^v?([0-9]+)\\.([0-9]+)\\.([0-9]+)$")
-    # Exact release tag X.Y.Z
-    string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_TAG}")
-  elseif (HYPRE_FETCH_VERSION MATCHES "([0-9]+)\\.([0-9]+)(\\.([0-9]+))?")
-    string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION "${HYPRE_FETCH_VERSION}")
-  else (NOT DEFINED HYPRE_VERSION)
-    message(FATAL_ERROR "Unable to find HYPRE release version. Please provide it via -DHYPRE_VERSION")
-  endif()
-  if (HYPRE_SPLIT_VERSION AND NOT DEFINED HYPRE_VERSION)
-    list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
-    list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
-    if (HYPRE_SPLIT_VERSION GREATER 2)
-      list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
-    else()
-      set(HYPRE_PATCH_VERSION 0)
-    endif()
-    math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
-    set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
-  endif()
+  string(REGEX MATCHALL "[0-9]+" HYPRE_SPLIT_VERSION ${HYPRE_FETCH_VERSION})
+  list(GET HYPRE_SPLIT_VERSION 0 HYPRE_MAJOR_VERSION)
+  list(GET HYPRE_SPLIT_VERSION 1 HYPRE_MINOR_VERSION)
+  list(GET HYPRE_SPLIT_VERSION 2 HYPRE_PATCH_VERSION)
+  math(EXPR HYPRE_VERSION "10000*${HYPRE_MAJOR_VERSION} + 100*${HYPRE_MINOR_VERSION} + ${HYPRE_PATCH_VERSION}")
+  # set cache variables that would otherwise be set after mfem_find_package call
+  set(HYPRE_VERSION ${HYPRE_VERSION} CACHE STRING "HYPRE version." FORCE)
   return()
 endif()
 
@@ -190,8 +149,7 @@ endif()
 if (HYPRE_FOUND AND HYPRE_USING_HIP)
   find_package(rocsparse REQUIRED)
   find_package(rocrand REQUIRED)
-  find_package(rocsolver REQUIRED)
-  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES} roc::rocsolver roc::rocblas)
+  list(APPEND HYPRE_LIBRARIES ${rocsparse_LIBRARIES} ${rocrand_LIBRARIES})
   set(HYPRE_LIBRARIES ${HYPRE_LIBRARIES} CACHE STRING
       "HYPRE libraries + dependencies." FORCE)
   message(STATUS "Updated HYPRE_LIBRARIES: ${HYPRE_LIBRARIES}")

config/cmake/modules/FindMETIS.cmake

@@ -32,7 +32,6 @@ if (METIS_FETCH OR FETCH_TPLS)
     UPDATE_DISCONNECTED TRUE
     PREFIX ${PREFIX}
     CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
-    BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
     INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
   # set imported library target properties
   add_dependencies(METIS metis)

config/cmake/modules/MfemCmakeUtilities.cmake

@@ -718,7 +718,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
   get_target_property(IsImported ${tgt} IMPORTED)
   # message(STATUS "${tgt}[IMPORTED]: ${IsImported}")
   # Generally, the possible target types are: STATIC_LIBRARY, MODULE_LIBRARY,
-  # SHARED_LIBRARY, INTERFACE_LIBRARY, UNKNOWN_LIBRARY, EXECUTABLE.
+  # SHARED_LIBRARY, INTERFACE_LIBRARY, EXECUTABLE.
   get_target_property(type ${tgt} TYPE)
   # message(STATUS "${tgt}[TYPE]: ${type}")
   unset(ImportConfig)
@@ -766,7 +766,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
     else()
       message(STATUS " *** Warning: [${tgt}] LOCATION not defined!")
     endif()
-  elseif ("${type}" STREQUAL "SHARED_LIBRARY" OR "${type}" STREQUAL "UNKNOWN_LIBRARY")
+  elseif ("${type}" STREQUAL "SHARED_LIBRARY")
     get_target_property(Location ${tgt} LOCATION)
     if (Location)
       get_filename_component(Dir ${Location} DIRECTORY)
@@ -932,14 +932,12 @@ function(mfem_export_mk_files)
   endif()
   set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
   set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
-  # For the next 4 variables, these are the values for the build-tree version of
+  # For the next 4 variable, these are the values for the build-tree version of
   # 'config.mk'
   set(MFEM_INC_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
   set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
   set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
-  # TODO: CUDA/HIP support:
-  set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
   set(MFEM_MPIEXEC ${MPIEXEC})
   if (NOT MFEM_MPIEXEC)
     set(MFEM_MPIEXEC "mpirun")

config/config.mk.in

@@ -88,7 +88,6 @@ MFEM_BUILD_TAG = @MFEM_BUILD_TAG@
 MFEM_PREFIX    = @MFEM_PREFIX@
 MFEM_INC_DIR   = @MFEM_INC_DIR@
 MFEM_LIB_DIR   = @MFEM_LIB_DIR@
-MFEM_XLINKER   = @MFEM_XLINKER@
 
 # Location of test.mk
 MFEM_TEST_MK = @MFEM_TEST_MK@

config/defaults.mk

@@ -57,7 +57,7 @@ CUDA_DIR = $(or $(CUDA_HOME),$(patsubst %/,%,$(dir \
 CLANG_CUDA_FLAGS = -xcuda --cuda-path=$(CUDA_DIR) --cuda-gpu-arch=$(CUDA_ARCH)
 # flags for nvcc
 NVCC_FLAGS = -x=cu --expt-extended-lambda --expt-relaxed-constexpr \
- -arch=$(CUDA_ARCH) -isystem "$(CUDA_DIR)/include"
+ -arch=$(CUDA_ARCH)
 # Prefixes for passing flags to the host compiler and linker when using
 # CUDA_CXX=nvcc
 CUDA_XCOMPILER = -Xcompiler=

config/docker/Dockerfile.base

@@ -33,8 +33,8 @@ RUN mkdir -p /opt/mfem-env \
 RUN cd /opt/mfem-env && \
     . /opt/spack/share/spack/setup-env.sh && \
     spack env activate . && \
-    spack develop --path /code mfem@main+examples+miniapps && \
-    spack add mfem@main+examples+miniapps # && \
+    spack develop --path /code mfem@master+examples+miniapps && \
+    spack add mfem@master+examples+miniapps && \
     spack install
 
 # ensure mfem always on various paths

config/docker/spack.yaml

@@ -1,11 +1,11 @@
 spack:
-  specs: [mfem@main+examples+miniapps]
-  view:
+  specs: [mfem@master+examples+miniapps]
+  view: 
     mfem:
-      root: /opt/mfem-view
+      root: /opt/mfem-view 
       link_type: copy
   concretization: together
   develop:
     mfem:
       path: /code
-      spec: mfem@main+examples+miniapps
+      spec: mfem@master+examples+miniapps

config/githooks/pre-push

@@ -115,7 +115,7 @@ fi
 
 # branch-history
 if [[ "${option}" == "--history" || "${option}" == "" ]]; then
-  git fetch origin main:main
+  git fetch origin master:master
   cd tests/scripts
   if ! ./runtest branch-history; then code=1; fi
   cd -

data/mfem.mesh

@@ -1,593 +0,0 @@
-MFEM mesh v1.0
-
-# Created by: Pointwise
-
-# MFEM Geometry Types:
-#
-# POINT       = 0
-# SEGMENT     = 1
-# TRIANGLE    = 2
-# SQUARE      = 3
-# TETRAHEDRON = 4
-# CUBE        = 5
-# PRISM       = 6
-
-dimension
-2
-
-elements
-160
-1 3 1 164 163 0
-1 3 164 165 162 163
-1 3 2 166 164 1
-1 3 166 132 165 164
-1 3 3 167 166 2
-1 3 167 131 132 166
-1 3 4 168 167 3
-1 3 168 130 131 167
-1 3 5 169 168 4
-1 3 169 129 130 168
-1 3 6 170 169 5
-1 3 170 128 129 169
-1 3 171 172 170 6
-1 3 172 127 128 170
-1 3 124 125 172 171
-1 3 125 126 127 172
-1 3 162 165 173 161
-1 3 165 132 133 173
-1 3 161 173 174 160
-1 3 173 133 134 174
-1 3 160 174 175 159
-1 3 174 134 135 175
-1 3 6 7 176 171
-1 3 7 8 177 176
-1 3 171 176 123 124
-1 3 176 177 122 123
-1 3 159 175 178 158
-1 3 175 135 136 178
-1 3 158 178 179 157
-1 3 178 136 137 179
-1 3 157 179 180 156
-1 3 179 137 138 180
-1 3 122 177 181 121
-1 3 177 8 182 181
-1 3 8 9 183 182
-1 3 9 10 184 183
-1 3 10 11 185 184
-1 3 11 12 186 185
-1 3 12 13 187 186
-1 3 13 14 15 187
-1 3 121 181 119 120
-1 3 181 182 118 119
-1 3 182 183 117 118
-1 3 183 184 188 117
-1 3 184 185 109 188
-1 3 185 186 108 109
-1 3 186 187 189 108
-1 3 187 15 16 189
-1 3 109 110 190 188
-1 3 110 111 191 190
-1 3 111 112 113 191
-1 3 188 190 116 117
-1 3 190 191 115 116
-1 3 191 113 114 115
-1 3 189 192 107 108
-1 3 192 193 106 107
-1 3 193 194 105 106
-1 3 194 195 104 105
-1 3 195 196 103 104
-1 3 16 17 192 189
-1 3 17 18 193 192
-1 3 18 19 194 193
-1 3 19 20 195 194
-1 3 20 21 196 195
-1 3 97 98 197 96
-1 3 98 99 198 197
-1 3 99 100 199 198
-1 3 100 101 200 199
-1 3 101 102 201 200
-1 3 102 103 202 201
-1 3 103 196 203 202
-1 3 196 21 22 203
-1 3 96 197 204 95
-1 3 197 198 39 204
-1 3 198 199 38 39
-1 3 199 200 205 38
-1 3 200 201 32 205
-1 3 201 202 31 32
-1 3 202 203 206 31
-1 3 203 22 23 206
-1 3 32 33 207 205
-1 3 33 34 35 207
-1 3 205 207 37 38
-1 3 207 35 36 37
-1 3 39 40 208 204
-1 3 40 41 209 208
-1 3 41 42 210 209
-1 3 42 43 211 210
-1 3 43 44 212 211
-1 3 204 208 94 95
-1 3 208 209 93 94
-1 3 209 210 92 93
-1 3 210 211 91 92
-1 3 211 212 90 91
-1 3 90 212 213 89
-1 3 212 44 214 213
-1 3 44 45 215 214
-1 3 45 46 216 215
-1 3 46 47 217 216
-1 3 47 48 218 217
-1 3 48 49 219 218
-1 3 49 50 51 219
-1 3 89 213 87 88
-1 3 213 214 86 87
-1 3 214 215 85 86
-1 3 215 216 84 85
-1 3 216 217 83 84
-1 3 217 218 82 83
-1 3 218 219 220 82
-1 3 219 51 52 220
-1 3 53 221 220 52
-1 3 221 81 82 220
-1 3 54 222 221 53
-1 3 222 80 81 221
-1 3 55 223 222 54
-1 3 223 79 80 222
-1 3 26 27 224 25
-1 3 27 28 29 224
-1 3 25 224 225 24
-1 3 224 29 30 225
-1 3 24 225 206 23
-1 3 225 30 31 206
-1 3 154 155 226 153
-1 3 155 156 180 226
-1 3 153 226 227 152
-1 3 226 180 138 227
-1 3 152 227 228 151
-1 3 227 138 139 228
-1 3 151 228 229 150
-1 3 228 139 140 229
-1 3 150 229 230 149
-1 3 229 140 141 230
-1 3 149 230 231 148
-1 3 230 141 142 231
-1 3 148 231 232 147
-1 3 231 142 143 232
-1 3 147 232 145 146
-1 3 232 143 144 145
-1 3 56 233 223 55
-1 3 233 78 79 223
-1 3 57 234 233 56
-1 3 234 77 78 233
-1 3 58 235 234 57
-1 3 235 76 77 234
-1 3 61 236 59 60
-1 3 236 235 58 59
-1 3 62 237 236 61
-1 3 237 76 235 236
-1 3 63 238 237 62
-1 3 238 75 76 237
-1 3 64 239 238 63
-1 3 239 74 75 238
-1 3 65 240 239 64
-1 3 240 73 74 239
-1 3 66 241 240 65
-1 3 241 72 73 240
-1 3 67 242 241 66
-1 3 242 71 72 241
-1 3 68 69 242 67
-1 3 69 70 71 242
-
-boundary
-164
-3 1 0 1
-3 1 1 2
-3 1 2 3
-3 1 3 4
-3 1 4 5
-3 1 5 6
-3 1 6 7
-3 1 7 8
-3 1 8 9
-3 1 9 10
-3 1 10 11
-3 1 11 12
-3 1 12 13
-3 1 13 14
-3 1 16 17
-3 1 17 18
-3 1 18 19
-3 1 19 20
-3 1 20 21
-3 1 21 22
-3 1 22 23
-3 1 23 24
-3 1 24 25
-3 1 25 26
-3 1 26 27
-3 1 27 28
-3 1 28 29
-3 1 29 30
-3 1 30 31
-3 1 31 32
-3 1 32 33
-3 1 33 34
-3 1 34 35
-3 1 35 36
-3 1 36 37
-3 1 37 38
-3 1 38 39
-3 1 39 40
-3 1 40 41
-3 1 41 42
-3 1 42 43
-3 1 43 44
-3 1 49 50
-3 1 48 49
-3 1 47 48
-3 1 46 47
-3 1 45 46
-3 1 44 45
-3 1 52 53
-3 1 53 54
-3 1 54 55
-3 1 57 58
-3 1 56 57
-3 1 55 56
-3 1 60 61
-3 1 61 62
-3 1 62 63
-3 1 63 64
-3 1 64 65
-3 1 65 66
-3 1 66 67
-3 1 67 68
-3 1 75 76
-3 1 74 75
-3 1 73 74
-3 1 72 73
-3 1 71 72
-3 1 70 71
-3 1 76 77
-3 1 77 78
-3 1 78 79
-3 1 81 82
-3 1 80 81
-3 1 79 80
-3 1 82 83
-3 1 83 84
-3 1 84 85
-3 1 85 86
-3 1 86 87
-3 1 87 88
-3 1 94 95
-3 1 93 94
-3 1 92 93
-3 1 91 92
-3 1 90 91
-3 1 96 97
-3 1 95 96
-3 1 97 98
-3 1 98 99
-3 1 99 100
-3 1 100 101
-3 1 101 102
-3 1 102 103
-3 1 107 108
-3 1 106 107
-3 1 105 106
-3 1 104 105
-3 1 103 104
-3 1 108 109
-3 1 109 110
-3 1 110 111
-3 1 111 112
-3 1 112 113
-3 1 113 114
-3 1 114 115
-3 1 115 116
-3 1 116 117
-3 1 119 120
-3 1 118 119
-3 1 117 118
-3 1 131 132
-3 1 130 131
-3 1 129 130
-3 1 128 129
-3 1 127 128
-3 1 126 127
-3 1 132 133
-3 1 133 134
-3 1 134 135
-3 1 137 138
-3 1 136 137
-3 1 135 136
-3 1 138 139
-3 1 139 140
-3 1 140 141
-3 1 141 142
-3 1 142 143
-3 1 143 144
-3 1 147 148
-3 1 146 147
-3 1 153 154
-3 1 152 153
-3 1 151 152
-3 1 150 151
-3 1 149 150
-3 1 148 149
-3 1 156 157
-3 1 157 158
-3 1 158 159
-3 1 161 162
-3 1 160 161
-3 1 159 160
-2 1 69 70
-2 1 68 69
-3 1 88 89
-3 1 89 90
-3 1 121 122
-3 1 120 121
-3 1 123 124
-3 1 122 123
-3 1 125 126
-3 1 124 125
-1 1 144 145
-1 1 145 146
-3 1 15 16
-3 1 14 15
-3 1 50 51
-3 1 51 52
-3 1 59 60
-3 1 58 59
-3 1 154 155
-3 1 155 156
-3 1 163 0
-3 1 162 163
-
-vertices
-243
-2
-4 4
-4 3.5
-4 3
-4 2.5
-4 2
-4 1.5
-4 1
-4.5 1
-5 1
-5 1.5
-5 2
-5 2.5
-5 3
-5 3.5
-5 4
-5.500 4
-6 4
-6.500 4
-7 4
-7.5 4
-8 4
-8.5 4
-9 4
-9.5 4
-10 4
-10.5 4
-11 4
-11 3.5
-11 3
-10.5 3
-10 3
-9.5 3
-9.5 2.5
-10 2.5
-10.5 2.5
-10.5 2
-10.5 1.5
-10 1.5
-9.5 1.5
-9.5 1
-10 1
-10.5 1
-11 1
-11.5 1
-12 1
-12 1.5
-12 2
-12 2.5
-12 3
-12 3.5
-12 4
-12.5 4
-13 4
-13.333 3.75
-13.666 3.5
-14.000 3.25
-14.333 3.5
-14.666 3.75
-15.000 4
-15.500 4
-16.000 4
-16.000 3.5
-16.000 3
-16.000 2.5
-16.000 2
-16.000 1.5
-16.000 1
-16.000 0.5
-16.000 0
-15.500 0
-15.000 0
-15.000 0.5000000000000002
-15.000 1
-15.000 1.5
-15.000 2
-15.000 2.5
-15.000 3
-14.666 2.75
-14.333 2.5
-14.000 2.25
-13.666 2.5
-13.333 2.75
-13 3
-13 2.5
-13 2
-13 1.5
-13 1
-13 0.500
-13 0
-12.5 0
-12 0
-11.5 0
-11 0
-10.5 0
-10 0
-9.5 0
-9 0
-8.5 0
-8.5 0.5
-8.5 1
-8.5 1.5
-8.5 2
-8.5 2.5
-8.5 3
-8 3
-7.5 3
-7 3
-6.500 3
-6 3
-6 2.5
-6.5 2.5
-7 2.5
-7.5 2.5
-7.5 2
-7.5 1.5
-7.000 1.5
-6.5 1.5
-6 1.5
-6 1
-6 0.5
-6 0
-5.5 0
-5 0
-4.5 0
-4 0
-3.5 0
-3 0
-3 0.500
-3 1
-3 1.5
-3 2
-3 2.5
-3 3
-2.666 2.75
-2.333 2.5
-2.000 2.25
-1.666 2.5
-1.333 2.75
-1.000 3
-1.000 2.5
-1.000 2
-1.000 1.5
-1.000 1
-1.000 0.5000
-1.000 0
-0.5000 0
-0.0000 0
-0.0000 0.5
-0.0000 1
-0.0000 1.5
-0.0000 2
-0.0000 2.5
-0.0000 3
-0.0000 3.5
-0.0000 4
-0.5000 4
-1.000 4
-1.333 3.75
-1.666 3.5
-2.000 3.25
-2.333 3.5
-2.666 3.75
-3 4
-3.5 4
-3.5 3.5
-3 3.5
-3.5 3
-3.5 2.5
-3.5 2
-3.5 1.5
-3.5 1
-4 0.5
-3.5 0.5
-2.666 3.25
-2.333 3
-2.000 2.75
-4.5 0.5
-5 0.5
-1.666 3
-1.333 3.25
-1.000 3.5
-5.5 0.5
-5.500 1
-5.500 1.5
-5.500 2
-5.500 2.5
-5.500 3
-5.500 3.5
-6 2
-6 3.5
-6.5 2
-7 2
-6.5 3.5
-7 3.5
-7.5 3.5
-8 3.5
-8.5 3.5
-9 0.5
-9 1
-9 1.5
-9 2
-9 2.5
-9 3
-9 3.5
-9.5 0.5
-9.5 2
-9.5 3.5
-10 2
-10 0.5
-10.5 0.5
-11 0.5
-11.5 0.5
-12 0.5
-12.5 0.500
-12.5 1
-12.5 1.5
-12.5 2
-12.5 2.5
-12.5 3
-12.5 3.5
-13 3.5
-13.333 3.250
-13.666 3
-14.000 2.75
-10.5 3.5
-10 3.5
-0.500 3.5
-0.500 3
-0.500 2.5
-0.500 2
-0.500 1.5
-0.500 1
-0.500 0.5
-14.333 3
-14.666 3.25
-15.000 3.5
-15.500 3.5
-15.500 3
-15.500 2.5
-15.500 2
-15.500 1.5
-15.500 1
-15.500 0.5

data/nc-nurbs3d.mesh

@@ -1,342 +0,0 @@
-MFEM NURBS NC-patch mesh v1.0
-dimension
-3
-
-elements
-13
-0 1 5 0 8 10 11 9 4 6 7 5
-0 1 5 0 18 8 24 32 30 23 36 38
-0 1 5 0 0 18 32 14 12 30 38 29
-0 1 5 0 32 24 10 20 38 36 26 35
-0 1 5 0 14 32 20 2 29 38 35 16
-0 1 5 0 30 23 36 38 31 22 37 39
-0 1 5 0 12 30 38 29 13 31 39 28
-0 1 5 0 38 36 26 35 39 37 27 34
-0 1 5 0 29 38 35 16 28 39 34 17
-0 1 5 0 31 22 37 39 19 9 25 33
-0 1 5 0 13 31 39 28 1 19 33 15
-0 1 5 0 39 37 27 34 33 25 11 21
-0 1 5 0 28 39 34 17 15 33 21 3
-
-boundary
-31
-9999 3 8 10 6 4
-9999 3 10 11 7 6
-9999 3 11 9 5 7
-9999 3 9 8 4 5
-9999 3 4 6 7 5
-9999 3 32 24 8 18
-9999 3 18 8 23 30
-9999 3 14 32 18 0
-9999 3 0 18 30 12
-9999 3 14 0 12 29
-9999 3 20 10 24 32
-9999 3 10 20 35 26
-9999 3 2 20 32 14
-9999 3 20 2 16 35
-9999 3 2 14 29 16
-9999 3 30 23 22 31
-9999 3 12 30 31 13
-9999 3 29 12 13 28
-9999 3 26 35 34 27
-9999 3 35 16 17 34
-9999 3 16 29 28 17
-9999 3 31 22 9 19
-9999 3 19 9 25 33
-9999 3 13 31 19 1
-9999 3 28 13 1 15
-9999 3 1 19 33 15
-9999 3 27 34 21 11
-9999 3 33 25 11 21
-9999 3 34 17 3 21
-9999 3 17 28 15 3
-9999 3 15 33 21 3
-
-vertex_to_knotspan
-8
-23 0 1 8 10 11 9
-22 0 2 8 10 11 9
-24 1 0 8 10 11 9
-36 1 1 8 10 11 9
-37 1 2 8 10 11 9
-25 1 3 8 10 11 9
-26 2 1 8 10 11 9
-27 2 2 8 10 11 9
-
-coordinates
-40
-3
-0 0 0
-0 1 0
-4 0 0
-4 1 0
-0 0 4
-0 1 4
-4 0 4
-4 1 4
-0 0 2
-0 1 2
-4 0 2
-4 1 2
-0 0.333333333333333 0
-0 0.666666666666667 0
-2 0 0
-2 1 0
-4 0.333333333333334 0
-4 0.666666666666667 0
-0 0 1
-0 1 1
-4 0 1
-4 1 1
-0 0.666666666666667 2
-0 0.333333333333333 2
-2 0 2
-2 1 2
-4 0.333333333333333 2
-4 0.666666666666667 2
-2 0.666666666666667 0
-2 0.333333333333333 0
-0 0.333333333333333 1
-0 0.666666666666667 1
-1.81325211007895 0 1
-1.81325211007895 1 1
-4 0.666666666666667 1
-4 0.333333333333333 1
-2 0.333333333333333 2
-2 0.666666666666667 2
-1.81325211007895 0.333333333333333 1
-1.81325211007895 0.666666666666667 1
-
-edges
-87
-0 8 10
-1 10 11
-0 9 11
-1 8 9
-0 4 6
-1 6 7
-0 5 7
-1 4 5
-2 4 8
-2 6 10
-2 7 11
-2 5 9
-9 18 8
-7 8 24
-9 32 24
-7 18 32
-9 30 23
-7 23 36
-9 38 36
-7 30 38
-3 18 30
-3 8 23
-3 24 36
-3 32 38
-8 0 18
-8 14 32
-7 0 14
-8 12 30
-8 29 38
-7 12 29
-3 0 12
-3 14 29
-6 24 10
-9 20 10
-6 32 20
-6 36 26
-9 35 26
-6 38 35
-3 10 26
-3 20 35
-8 2 20
-6 14 2
-8 16 35
-6 29 16
-3 2 16
-9 31 22
-7 22 37
-9 39 37
-7 31 39
-4 30 31
-4 23 22
-4 36 37
-4 38 39
-8 13 31
-8 28 39
-7 13 28
-4 12 13
-4 29 28
-6 37 27
-9 34 27
-6 39 34
-4 26 27
-4 35 34
-8 17 34
-6 28 17
-4 16 17
-9 19 9
-7 9 25
-9 33 25
-7 19 33
-5 31 19
-5 22 9
-5 37 25
-5 39 33
-8 1 19
-8 15 33
-7 1 15
-5 13 1
-5 28 15
-6 25 11
-9 21 11
-6 33 21
-5 27 11
-5 34 21
-8 3 21
-6 15 3
-5 17 3
-
-knotvectors
-10
-1 3 0 0 0.5 1 1
-1 4 0 0 0.333333333333333 0.666666666666667 1 1
-1 3 0 0 0.5 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-
-spacing
-0
-
-weights
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-
-FiniteElementSpace
-FiniteElementCollection: NURBS1
-VDim: 3
-Ordering: 1
-
-0 0 0
-0 1 0
-4 0 0
-4 1 0
-0 0 4
-0 1 4
-4 0 4
-4 1 4
-0 0 2
-0 1 2
-4 0 2
-4 1 2
-0 0.333333333333333 0
-0 0.666666666666667 0
-2 0 0
-2 1 0
-4 0.333333333333334 0
-4 0.666666666666667 0
-0 0 1
-0 1 1
-4 0 1
-4 1 1
-0 0.666666666666667 2
-0 0.333333333333333 2
-2 0 2
-2 1 2
-4 0.333333333333333 2
-4 0.666666666666667 2
-2 0.666666666666667 0
-2 0.333333333333333 0
-0 0.333333333333333 1
-0 0.666666666666667 1
-1.81325211007895 0 1
-1.81325211007895 1 1
-4 0.666666666666667 1
-4 0.333333333333333 1
-2 0.333333333333333 2
-2 0.666666666666667 2
-1.81325211007895 0.333333333333333 1
-1.81325211007895 0.666666666666667 1
-2 0 4
-4 0.333333333333333 4
-4 0.666666666666667 4
-2 1 4
-0 0.333333333333333 4
-0 0.666666666666667 4
-0 0 3
-4 0 3
-4 1 3
-0 1 3
-2 0 3
-4 0.333333333333333 3
-4 0.666666666666667 3
-2 1 3
-0 0.666666666666667 3
-0 0.333333333333333 3
-2 0.333333333333333 4
-2 0.666666666666667 4
-2 0.333333333333333 3
-2 0.666666666666667 3

data/nc3-nurbs.mesh

@@ -1,96 +0,0 @@
-MFEM NURBS NC-patch mesh v1.0
-dimension
-2
-
-# rank attr geom ref_type nodes/children
-elements
-3
-0 1 3 0 0 4 5 1
-0 1 3 0 6 7 4 2
-0 1 3 0 6 3 5 7
-
-# attr geom nodes
-boundary
-7
-1 1 0 4
-1 1 5 1
-1 1 1 0
-1 1 2 6
-1 1 6 3
-1 1 4 2
-1 1 5 3
-
-vertex_to_knotspan
-1
-7 1 4 5
-
-# top-level node coordinates
-coordinates
-8
-2
-0 0
-0 1
-2 0
-2 1
-1 0
-1 1
-2 0.5
-1 0.5
-
-edges
-11
-0 0 4
-1 4 5
-0 1 5
-1 0 1
-2 6 7
-4 7 4
-2 2 4
-4 6 2
-3 6 3
-2 3 5
-3 7 5
-
-knotvectors
-5
-1 3 0 0 0.5 1 1
-1 3 0 0 0.5 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-1 2 0 0 1 1
-
-spacing
-0
-
-weights
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-1.0
-
-FiniteElementSpace
-FiniteElementCollection: NURBS1
-VDim: 2
-Ordering: 1
-
-0 0
-0 1
-2 0
-2 1
-1 0
-1 1
-2 0.5
-1 0.5
-0.5 0
-0.5 1
-0 0.5
-0.5 0.5
-mfem_mesh_end

doc/CodeDocumentation.dox

@@ -202,7 +202,6 @@ namespace mfem {
  * - <a class="el" href="tesla_8cpp_source.html">Tesla</a>: simple magnetostatics simulation code
  * - <a class="el" href="maxwell_8cpp_source.html">Maxwell</a>: simple transient full-wave electromagnetics simulation code
  * - <a class="el" href="joule_8cpp_source.html">Joule</a>: transient magnetics and Joule heating miniapp
- * - <a class="el" href="lorentz_8cpp_source.html">Lorentz</a>: simple particle tracking code based on the Lorentz force
  * - <a class="el" href="classmfem_1_1navier_1_1NavierSolver.html">Navier</a>: solve the transient incompressible Navier-Stokes equations
  * - <a class="el" href="mobius-strip_8cpp_source.html">Mobius Strip</a>: generate various Mobius strip-like meshes
  * - <a class="el" href="klein-bottle_8cpp_source.html">Klein Bottle</a>: generate three types of Klein bottle surfaces

examples/CMakeLists.txt

@@ -205,15 +205,6 @@ if (MFEM_ENABLE_TESTING)
       $<TARGET_FILE:ex25p> "-no-vis" "--mumps-solver"
       ${MPIEXEC_POSTFLAGS})
   endif()
-
-  # Parallel libCEED example
-  if (MFEM_USE_CEED AND MFEM_USE_MPI)
-    add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
-      COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
-      ${MPIEXEC_PREFLAGS}
-      $<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
-      ${MPIEXEC_POSTFLAGS})
-  endif()
 endif()
 
 # Include the examples/amgx directory if AmgX is enabled

examples/ex1.cpp

@@ -27,7 +27,6 @@
 //               ex1 -m ../data/fichera-amr.mesh
 //               ex1 -m ../data/mobius-strip.mesh
 //               ex1 -m ../data/mobius-strip.mesh -o -1 -sc
-//               ex1 -m ../data/nc3-nurbs.mesh -o -1
 //
 // Device sample runs:
 //               ex1 -pa -d cuda

examples/ex26.cpp

@@ -78,6 +78,7 @@ private:
       opr.SetOperatorOwner(false);
 
       CGSolver* pcg = new CGSolver();
+      // pcg->iterative_mode = false; // the multigrid algorithm does this
       pcg->SetPrintLevel(-1);
       pcg->SetMaxIter(200);
       pcg->SetRelTol(sqrt(1e-4));
@@ -100,7 +101,8 @@ private:
       Vector diag(fespace.GetTrueVSize());
       bfs[level]->AssembleDiagonal(diag);
 
-      Solver* smoother = new OperatorChebyshevSmoother(*opr, diag, ess_tdof_list, 2);
+      Solver *smoother = new OperatorChebyshevSmoother(
+         *opr, diag, ess_tdof_list, 2);
       AddLevel(opr.Ptr(), smoother, true, true);
    }
 };

examples/ex26p.cpp

@@ -88,6 +88,7 @@ private:
       amg->SetPrintLevel(-1);
 
       CGSolver* pcg = new CGSolver(MPI_COMM_WORLD);
+      // pcg->iterative_mode = false; // the multigrid algorithm does this
       pcg->SetPrintLevel(-1);
       pcg->SetMaxIter(10);
       pcg->SetRelTol(sqrt(1e-4));

examples/jupyter/README.md

@@ -1,6 +1,6 @@
 # Jupyter Notebooks using xeus-cling
 
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mfem/mfem/main?filepath=examples%2Fjupyter%2Fex.ipynb)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mfem/mfem/master?filepath=examples%2Fjupyter%2Fex.ipynb)
 
 [xeus-cling](https://github.com/jupyter-xeus/xeus-cling) is a C++ Jupyter Kernel based on [cling](https://github.com/root-project/cling),
 which can be used to create interactive C++ MFEM and GLVis notebooks.

examples/makefile

@@ -173,12 +173,6 @@ ex11p-test-cpardiso: ex11p
 	@$(call mfem-test,$<, $(RUN_MPI), MKL_CPARDISO example,--cpardiso)
 test-par-YES: ex11p-test-cpardiso
 endif
-ifeq ($(MFEM_USE_CEED),YES)
-ex1p-test-ceed: ex1p
-	@$(call mfem-test,$<, $(RUN_MPI),\
-	   Parallel libCEED example,-d ceed-cpu -pa -a)
-test-par-YES: ex1p-test-ceed
-endif
 
 # Testing: "test" target and mfem-test* variables are defined in config/test.mk
 

fem/CMakeLists.txt

@@ -171,11 +171,6 @@ set(HDRS
   bilinearform.hpp
   bilinearform_ext.hpp
   bilininteg.hpp
-  integ/lininteg_domain_kernels.hpp
-  integ/bilininteg_dgdiffusion_kernels.hpp
-  integ/bilininteg_dgtrace_kernels.hpp
-  integ/bilininteg_vecdiffusion_kernels.hpp
-  integ/bilininteg_convection_kernels.hpp
   integ/bilininteg_diffusion_kernels.hpp
   integ/bilininteg_elasticity_kernels.hpp
   integ/bilininteg_hcurl_kernels.hpp
@@ -246,13 +241,8 @@ set(HDRS
   lor/lor_ams.hpp
   lor/lor_batched.hpp
   lor/lor_h1.hpp
-  lor/lor_dg.hpp
   lor/lor_nd.hpp
   lor/lor_rt.hpp
-  lor/lor_h1_impl.hpp
-  lor/lor_dg_impl.hpp
-  lor/lor_nd_impl.hpp
-  lor/lor_rt_impl.hpp
   lor/lor_util.hpp
   multigrid.hpp
   nonlinearform.hpp

fem/bilinearform_ext.cpp

@@ -255,6 +255,8 @@ PABilinearFormExtension::PABilinearFormExtension(BilinearForm *form)
 
 void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 {
+   MFEM_PERF_FUNCTION;
+
    if ( Device::Allows(Backend::CEED_MASK) ) { return; }
    ElementDofOrdering ordering = GetEVectorOrdering(*a->FESpace());
    elem_restrict = trial_fes->GetElementRestriction(ordering);
@@ -331,6 +333,8 @@ void PABilinearFormExtension::SetupRestrictionOperators(const L2FaceValues m)
 
 void PABilinearFormExtension::Assemble()
 {
+   MFEM_PERF_FUNCTION;
+
    SetupRestrictionOperators(L2FaceValues::DoubleValued);
 
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
@@ -487,6 +491,8 @@ void PABilinearFormExtension::FormLinearSystem(const Array<int> &ess_tdof_list,
 void PABilinearFormExtension::MultInternal(const Vector &x, Vector &y,
                                            const bool useAbs) const
 {
+   MFEM_PERF_FUNCTION;
+
    Array<BilinearFormIntegrator*> &integrators = *a->GetDBFI();
 
    const int iSz = integrators.Size();
@@ -859,6 +865,7 @@ EABilinearFormExtension::EABilinearFormExtension(BilinearForm *form)
 
 void EABilinearFormExtension::Assemble()
 {
+   MFEM_PERF_FUNCTION;
    SetupRestrictionOperators(L2FaceValues::SingleValued);
 
    ne = trial_fes->GetMesh()->GetNE();
@@ -1407,6 +1414,7 @@ FABilinearFormExtension::FABilinearFormExtension(BilinearForm *form)
 
 void FABilinearFormExtension::Assemble()
 {
+   MFEM_PERF_FUNCTION;
    EABilinearFormExtension::Assemble();
    FiniteElementSpace &fes = *a->FESpace();
    int width = fes.GetVSize();

fem/bilininteg.hpp

@@ -23,8 +23,6 @@
 
 namespace mfem
 {
-class QuadratureSpace;
-class FaceQuadratureSpace;
 
 /// Abstract base class BilinearFormIntegrator
 class BilinearFormIntegrator : public NonlinearFormIntegrator
@@ -2496,7 +2494,8 @@ private:
 #endif
 
 public:
-   ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0);
+   ConvectionIntegrator(VectorCoefficient &q, real_t a = 1.0)
+      : Q(&q) { alpha = a; }
 
    void AssembleElementMatrix(const FiniteElement &,
                               ElementTransformation &,
@@ -2529,28 +2528,6 @@ public:
 
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
 
-   /// arguments: NE, B, G, Bt, Gt, pa_data, x, y, D1D, Q1D
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &, const Vector &,
-                                    const Vector &, Vector &, const int,
-                                    const int);
-
-   /// arguments: DIMS, D1D, Q1D
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// arguments: DIMS, D1D, Q1D
-   MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels { Kernels(); };
-
 protected:
    const IntegrationRule* GetDefaultIntegrationRule(
       const FiniteElement& trial_fe,
@@ -2821,13 +2798,15 @@ protected:
    bool symmetric = true; ///< False if using a nonsymmetric matrix coefficient
 
 public:
-   CurlCurlIntegrator();
+   CurlCurlIntegrator() { Q = NULL; DQ = NULL; MQ = NULL; }
    /// Construct a bilinear form integrator for Nedelec elements
-   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = nullptr);
+   CurlCurlIntegrator(Coefficient &q, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(&q), DQ(NULL), MQ(NULL) { }
    CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
-                      const IntegrationRule *ir = nullptr);
-   CurlCurlIntegrator(MatrixCoefficient &mq,
-                      const IntegrationRule *ir = nullptr);
+                      const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(&dq), MQ(NULL) { }
+   CurlCurlIntegrator(MatrixCoefficient &mq, const IntegrationRule *ir = NULL) :
+      BilinearFormIntegrator(ir), Q(NULL), DQ(NULL), MQ(&mq) { }
 
    /* Given a particular Finite Element, compute the
       element curl-curl matrix elmat */
@@ -2857,34 +2836,6 @@ public:
    void AssembleDiagonalPA(Vector& diag) override;
 
    const Coefficient *GetCoefficient() const { return Q; }
-
-   /// arguments: d1d, q1d, symmetric, NE, bo, bc, bot, bct, gc, gct, pa_data,
-   /// x, y, useAbs
-   using ApplyKernelType = void (*)(
-                              const int, const int, const bool, const int, const Array<real_t> &,
-                              const Array<real_t> &, const Array<real_t> &, const Array<real_t> &,
-                              const Array<real_t> &, const Array<real_t> &, const Vector &,
-                              const Vector &, Vector &, const bool);
-
-   /// arguments: d1d, q1d, symmetric, ne, Bo, Bc, Go, Gc, pa_data, diag
-   using DiagonalKernelType = void (*)(const int, const int, const bool,
-                                       const int, const Array<real_t> &,
-                                       const Array<real_t> &,
-                                       const Array<real_t> &,
-                                       const Array<real_t> &, const Vector &,
-                                       Vector &);
-
-   /// parameters: dim, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// parameters: dim, d1d, q1d
-   MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
-   struct Kernels { Kernels(); };
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      DiagonalPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
 };
 
 /** Integrator for $(\mathrm{curl}(u), \mathrm{curl}(v))$ for FE spaces defined by 'dim' copies of a
@@ -3138,18 +3089,21 @@ private:
    Vector vcoeff;
 
 public:
-   VectorDiffusionIntegrator(const IntegrationRule *ir = nullptr);
+   VectorDiffusionIntegrator() { }
 
    /** \brief Integrator with unit coefficient for caller-specified vector
        dimension.
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(int vector_dimension);
+   VectorDiffusionIntegrator(int vector_dimension)
+      : vdim(vector_dimension) { }
 
-   VectorDiffusionIntegrator(Coefficient &q);
+   VectorDiffusionIntegrator(Coefficient &q)
+      : Q(&q) { }
 
-   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir);
+   VectorDiffusionIntegrator(Coefficient &q, const IntegrationRule *ir)
+      : BilinearFormIntegrator(ir), Q(&q) { }
 
    /** \brief Integrator with scalar coefficient for caller-specified vector
        dimension.
@@ -3159,7 +3113,8 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(Coefficient &q, int vector_dimension);
+   VectorDiffusionIntegrator(Coefficient &q, int vector_dimension)
+      : Q(&q), vdim(vector_dimension) { }
 
    /** \brief Integrator with \c VectorCoefficient. The vector dimension of the
        \c FiniteElementSpace is assumed to be the same as the dimension of the
@@ -3170,7 +3125,8 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(VectorCoefficient &vq);
+   VectorDiffusionIntegrator(VectorCoefficient &vq)
+      : VQ(&vq), vdim(vq.GetVDim()) { }
 
    /** \brief Integrator with \c MatrixCoefficient. The vector dimension of the
        \c FiniteElementSpace is assumed to be the same as the dimension of the
@@ -3181,7 +3137,8 @@ public:
 
        If the vector dimension does not match the true dimension of the space,
        the resulting element matrix will be mathematically invalid. */
-   VectorDiffusionIntegrator(MatrixCoefficient& mq);
+   VectorDiffusionIntegrator(MatrixCoefficient& mq)
+      : MQ(&mq), vdim(mq.GetVDim()) { }
 
    void AssembleElementMatrix(const FiniteElement &el,
                               ElementTransformation &Trans,
@@ -3197,28 +3154,6 @@ public:
    void AddMultPA(const Vector &x, Vector &y) const override;
    void AddMultMF(const Vector &x, Vector &y) const override;
    bool SupportsCeed() const override { return DeviceCanUseCeed(); }
-
-   /// arguments: ne, B, G, Bt, Gt, pa_data, x, y, d1d, q1d, vdim
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &, const Vector &,
-                                    const Vector &, Vector &, const int,
-                                    const int, const int);
-
-   /// arguments: dim, vdim, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int, int));
-
-   template <int DIM, int VDIM, int D1D, int Q1D>
-   static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, VDIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels
-   {
-      Kernels();
-   };
 };
 
 /** Integrator for the linear elasticity form:
@@ -3372,8 +3307,8 @@ public:
 class DGTraceIntegrator : public BilinearFormIntegrator
 {
 protected:
-   Coefficient *rho = nullptr;
-   VectorCoefficient *u = nullptr;
+   Coefficient *rho;
+   VectorCoefficient *u;
    real_t alpha, beta;
    // PA extension
    Vector pa_data;
@@ -3386,16 +3321,17 @@ private:
    Vector tr_shape1, te_shape1, tr_shape2, te_shape2;
 
 public:
-   DGTraceIntegrator(real_t a, real_t b);
-
    /// Construct integrator with $\rho = 1$, $\beta = \alpha/2$.
-   DGTraceIntegrator(VectorCoefficient &u_, real_t a);
+   DGTraceIntegrator(VectorCoefficient &u_, real_t a)
+   { rho = NULL; u = &u_; alpha = a; beta = 0.5*a; }
 
    /// Construct integrator with $\rho = 1$.
-   DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b);
+   DGTraceIntegrator(VectorCoefficient &u_, real_t a, real_t b)
+   { rho = NULL; u = &u_; alpha = a; beta = b; }
 
    DGTraceIntegrator(Coefficient &rho_, VectorCoefficient &u_,
-                     real_t a, real_t b);
+                     real_t a, real_t b)
+   { rho = &rho_; u = &u_; alpha = a; beta = b; }
 
    using BilinearFormIntegrator::AssembleFaceMatrix;
    void AssembleFaceMatrix(const FiniteElement &el1,
@@ -3434,26 +3370,6 @@ public:
    static const IntegrationRule &GetRule(Geometry::Type geom, int order,
                                          const ElementTransformation &T);
 
-   /// arguments: nf, B, Bt, pa_data, x, y, dofs1D, quad1D
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &, const Vector &,
-                                    const Vector &, Vector &, const int,
-                                    const int);
-
-   /// arguments: DIM, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-   /// arguments: DIM, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPATKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-      ApplyPATKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels { Kernels(); };
-
-
 private:
    void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };
@@ -3500,8 +3416,8 @@ public:
 class DGDiffusionIntegrator : public BilinearFormIntegrator
 {
 protected:
-   Coefficient *Q = nullptr;
-   MatrixCoefficient *MQ = nullptr;
+   Coefficient *Q;
+   MatrixCoefficient *MQ;
    real_t sigma, kappa;
 
    // these are not thread-safe!
@@ -3516,11 +3432,15 @@ protected:
    IntegrationRules irs{0, Quadrature1D::GaussLobatto};
 
 public:
-   DGDiffusionIntegrator(const real_t s, const real_t k);
-   DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k);
-   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k);
+   DGDiffusionIntegrator(const real_t s, const real_t k)
+      : Q(NULL), MQ(NULL), sigma(s), kappa(k) { }
+   DGDiffusionIntegrator(Coefficient &q, const real_t s, const real_t k)
+      : Q(&q), MQ(NULL), sigma(s), kappa(k) { }
+   DGDiffusionIntegrator(MatrixCoefficient &q, const real_t s, const real_t k)
+      : Q(NULL), MQ(&q), sigma(s), kappa(k) { }
    using BilinearFormIntegrator::AssembleFaceMatrix;
-   void AssembleFaceMatrix(const FiniteElement &el1, const FiniteElement &el2,
+   void AssembleFaceMatrix(const FiniteElement &el1,
+                           const FiniteElement &el2,
                            FaceElementTransformations &Trans,
                            DenseMatrix &elmat) override;
 
@@ -3539,28 +3459,6 @@ public:
 
    const IntegrationRule &GetRule(int order, Geometry::Type geom);
 
-   real_t GetPenaltyParameter() const { return kappa; }
-
-   /// arguments: nf, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D,
-   /// quad1D
-   using ApplyKernelType = void (*)(const int, const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &,
-                                    const Array<real_t> &, const real_t,
-                                    const Vector &, const Vector &_,
-                                    const Vector &, Vector &, Vector &,
-                                    const int, const int);
-
-   /// arguments: DIM, d1d, q1d
-   MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      ApplyPAKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
-
-   struct Kernels { Kernels(); };
-
 private:
    void SetupPA(const FiniteElementSpace &fes, FaceType type);
 };

fem/ceed/integrators/convection/convection_qf.h

@@ -8,7 +8,6 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_conv and f_apply_conv
 struct ConvectionContext {
@@ -92,7 +91,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a convection operator
@@ -168,7 +167,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -234,7 +233,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -382,7 +381,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
@@ -526,5 +525,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/integrators/diffusion/diffusion_qf.h

@@ -8,7 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
+
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct DiffusionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -85,7 +85,7 @@ CEED_QFUNCTION(f_build_diff_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a diffusion operator
@@ -161,7 +161,7 @@ CEED_QFUNCTION(f_build_diff_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -241,7 +241,7 @@ CEED_QFUNCTION(f_apply_diff)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -394,7 +394,7 @@ CEED_QFUNCTION(f_apply_diff_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
@@ -549,5 +549,5 @@ CEED_QFUNCTION(f_apply_diff_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/integrators/mass/mass_qf.h

@@ -8,7 +8,7 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
+
 
 /// A structure used to pass additional data to f_build_diff and f_apply_diff
 struct MassContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -53,7 +53,7 @@ CEED_QFUNCTION(f_build_mass_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a mass operator with a
@@ -95,7 +95,7 @@ CEED_QFUNCTION(f_build_mass_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a mass operator
@@ -135,7 +135,7 @@ CEED_QFUNCTION(f_apply_mass)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a diff operator
@@ -199,7 +199,7 @@ CEED_QFUNCTION(f_apply_mass_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
@@ -266,5 +266,5 @@ CEED_QFUNCTION(f_apply_mass_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/integrators/nlconvection/nlconvection_qf.h

@@ -8,7 +8,6 @@
 // MFEM is free software; you can redistribute it and/or modify it under the
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
-#include <ceed/types.h>
 
 /// A structure used to pass additional data to f_build_conv and f_apply_conv
 struct NLConvectionContext { CeedInt dim, space_dim, vdim; CeedScalar coeff; };
@@ -88,7 +87,7 @@ CEED_QFUNCTION(f_build_conv_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for building quadrature data for a convection operator
@@ -168,7 +167,7 @@ CEED_QFUNCTION(f_build_conv_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -248,7 +247,7 @@ CEED_QFUNCTION(f_apply_conv)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 /// libCEED Q-function for applying a conv operator
@@ -363,7 +362,7 @@ CEED_QFUNCTION(f_apply_conv_mf_const)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }
 
 CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
@@ -476,5 +475,5 @@ CEED_QFUNCTION(f_apply_conv_mf_quad)(void *ctx, CeedInt Q,
          }
          break;
    }
-   return CEED_ERROR_SUCCESS;
+   return 0;
 }

fem/ceed/interface/ceed.hpp

@@ -18,21 +18,10 @@
 
 #include <ceed.h>
 
-#if !CEED_VERSION_GE(0, 12, 0)
+#if !CEED_VERSION_GE(0,12,0)
 #error MFEM requires a libCEED version >= 0.12.0
 #endif
 
-#if !CEED_VERSION_GE(0, 13, 0)
-#define CeedOperatorCreateComposite(ceed, op)                                  \
-    CeedCompositeOperatorCreate((ceed), (op))
-#define CeedOperatorCompositeAddSub(op, sub)                                   \
-    CeedCompositeOperatorAddSub((op), (sub))
-#define CeedOperatorCompositeGetNumSub(op, num)                                \
-    CeedCompositeOperatorGetNumSub((op), (num))
-#define CeedOperatorCompositeGetSubList(op, list)                              \
-    CeedCompositeOperatorGetSubList((op), (list))
-#endif
-
 namespace mfem
 {
 

fem/ceed/interface/mixed_integrator.hpp

@@ -83,7 +83,7 @@ public:
       }
 
       // Create composite CeedOperator
-      CeedOperatorCreateComposite(internal::ceed, &oper);
+      CeedCompositeOperatorCreate(internal::ceed, &oper);
 
       // Create each sub-CeedOperator
       sub_ops.reserve(element_indices.size());
@@ -101,7 +101,7 @@ public:
          int nelem = *count[value.first];
          sub_op->Assemble(info, fes, ir, nelem, indices, Q);
          sub_ops.push_back(sub_op);
-         CeedOperatorCompositeAddSub(oper, sub_op->GetCeedOperator());
+         CeedCompositeOperatorAddSub(oper, sub_op->GetCeedOperator());
       }
 
       const int ndofs = fes.GetVDim() * fes.GetNDofs();

fem/ceed/interface/util.cpp

@@ -140,7 +140,11 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
    CeedOperator *subops;
    if (isComposite)
    {
-      ierr = CeedOperatorCompositeGetSubList(oper, &subops); PCeedChk(ierr);
+#if CEED_VERSION_GE(0, 10, 2)
+      ierr = CeedCompositeOperatorGetSubList(oper, &subops); PCeedChk(ierr);
+#else
+      ierr = CeedOperatorGetSubList(oper, &subops); PCeedChk(ierr);
+#endif
       ierr = CeedOperatorGetQFunction(subops[0], &qf); PCeedChk(ierr);
    }
    else
@@ -167,11 +171,7 @@ int CeedOperatorGetActiveField(CeedOperator oper, CeedOperatorField *field)
    for (int i = 0; i < numinputfields; ++i)
    {
       ierr = CeedOperatorFieldGetVector(inputfields[i], &if_vector); PCeedChk(ierr);
-      bool is_active = if_vector == CEED_VECTOR_ACTIVE;
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&if_vector); PCeedChk(ierr);
-#endif
-      if (is_active)
+      if (if_vector == CEED_VECTOR_ACTIVE)
       {
          if (found)
          {

fem/ceed/solvers/algebraic.cpp

@@ -228,7 +228,7 @@ void AddToCompositeOperator(BilinearFormIntegrator *integ, CeedOperator op)
 {
    if (integ->SupportsCeed())
    {
-      CeedOperatorCompositeAddSub(op, integ->GetCeedOp().GetCeedOperator());
+      CeedCompositeOperatorAddSub(op, integ->GetCeedOp().GetCeedOperator());
    }
    else
    {
@@ -240,7 +240,7 @@ CeedOperator CreateCeedCompositeOperatorFromBilinearForm(BilinearForm &form)
 {
    int ierr;
    CeedOperator op;
-   ierr = CeedOperatorCreateComposite(internal::ceed, &op); PCeedChk(ierr);
+   ierr = CeedCompositeOperatorCreate(internal::ceed, &op); PCeedChk(ierr);
 
    MFEM_VERIFY(form.GetBBFI()->Size() == 0,
                "Not implemented for this integrator!");
@@ -271,13 +271,18 @@ CeedOperator CoarsenCeedCompositeOperator(
    MFEM_ASSERT(isComposite, "");
 
    CeedOperator op_coarse;
-   ierr = CeedOperatorCreateComposite(internal::ceed,
+   ierr = CeedCompositeOperatorCreate(internal::ceed,
                                       &op_coarse); PCeedChk(ierr);
 
    int nsub;
    CeedOperator *subops;
-   ierr = CeedOperatorCompositeGetNumSub(op, &nsub); PCeedChk(ierr);
-   ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr);
+#if CEED_VERSION_GE(0, 10, 2)
+   ierr = CeedCompositeOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
+   ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#else
+   ierr = CeedOperatorGetNumSub(op, &nsub); PCeedChk(ierr);
+   ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#endif
    for (int isub=0; isub<nsub; ++isub)
    {
       CeedOperator subop = subops[isub];
@@ -289,7 +294,7 @@ CeedOperator CoarsenCeedCompositeOperator(
       // refcounted by existing objects
       ierr = CeedBasisDestroy(&basis_coarse); PCeedChk(ierr);
       ierr = CeedBasisDestroy(&basis_c2f); PCeedChk(ierr);
-      ierr = CeedOperatorCompositeAddSub(op_coarse, subop_coarse);
+      ierr = CeedCompositeOperatorAddSub(op_coarse, subop_coarse);
       PCeedChk(ierr);
       ierr = CeedOperatorDestroy(&subop_coarse); PCeedChk(ierr);
    }

fem/ceed/solvers/full-assembly.cpp

@@ -81,27 +81,12 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
       ierr = CeedOperatorFieldGetVector(input_fields[i], &vec); PCeedChk(ierr);
       if (vec == CEED_VECTOR_ACTIVE)
       {
-         CeedBasis basis;
-         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basis); PCeedChk(ierr);
-         if (!basisin)
-         {
-            ierr = CeedBasisReferenceCopy(basis, &basisin); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
-#endif
+         ierr = CeedOperatorFieldGetBasis(input_fields[i], &basisin);
+         PCeedChk(ierr);
          ierr = CeedBasisGetNumComponents(basisin, &ncomp); PCeedChk(ierr);
          ierr = CeedBasisGetDimension(basisin, &dim); PCeedChk(ierr);
-         CeedElemRestriction rstr;
-         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstr);
+         ierr = CeedOperatorFieldGetElemRestriction(input_fields[i], &rstrin);
          PCeedChk(ierr);
-         if (!rstrin)
-         {
-            ierr = CeedElemRestrictionReferenceCopy(rstr, &rstrin); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedElemRestrictionDestroy(&rstr); PCeedChk(ierr);
-#endif
          CeedEvalMode emode;
          ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
          PCeedChk(ierr);
@@ -127,9 +112,6 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
                break; // Caught by QF Assembly
          }
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&vec); PCeedChk(ierr);
-#endif
    }
 
    // Determine active output basis
@@ -145,25 +127,11 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
       ierr = CeedOperatorFieldGetVector(output_fields[i], &vec); PCeedChk(ierr);
       if (vec == CEED_VECTOR_ACTIVE)
       {
-         CeedBasis basis;
-         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basis); PCeedChk(ierr);
-         if (!basisout)
-         {
-            ierr = CeedBasisReferenceCopy(basis, &basisout); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
-#endif
-         CeedElemRestriction rstr;
-         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstr);
+         ierr = CeedOperatorFieldGetBasis(output_fields[i], &basisout);
+         PCeedChk(ierr);
+         ierr = CeedOperatorFieldGetElemRestriction(output_fields[i], &rstrout);
+         PCeedChk(ierr);
          PCeedChk(ierr);
-         if (!rstrout)
-         {
-            ierr = CeedElemRestrictionReferenceCopy(rstr, &rstrout); PCeedChk(ierr);
-         }
-#if CEED_VERSION_GE(0, 13, 0)
-         ierr = CeedElemRestrictionDestroy(&rstr); PCeedChk(ierr);
-#endif
          CeedEvalMode emode;
          ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode);
          PCeedChk(ierr);
@@ -189,9 +157,6 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
                break; // Caught by QF Assembly
          }
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&vec); PCeedChk(ierr);
-#endif
    }
 
    CeedInt nelem, elemsize, nqpts;
@@ -235,11 +200,7 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
    PCeedChk(ierr);
 
    CeedInt layout[3];
-#if CEED_VERSION_GE(0, 13, 0)
-   ierr = CeedElemRestrictionGetELayout(rstr_q, layout); PCeedChk(ierr);
-#else
    ierr = CeedElemRestrictionGetELayout(rstr_q, &layout); PCeedChk(ierr);
-#endif
    ierr = CeedElemRestrictionDestroy(&rstr_q); PCeedChk(ierr);
 
    // enforce structurally symmetric for later elimination
@@ -324,10 +285,6 @@ int CeedSingleOperatorFullAssemble(CeedOperator op, SparseMatrix *out)
    ierr = CeedVectorRestoreArrayRead(assembledqf, &assembledqfarray);
    PCeedChk(ierr);
    ierr = CeedVectorDestroy(&assembledqf); PCeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstrin); PCeedChk(ierr);
-   ierr = CeedElemRestrictionDestroy(&rstrout); PCeedChk(ierr);
-   ierr = CeedBasisDestroy(&basisin); PCeedChk(ierr);
-   ierr = CeedBasisDestroy(&basisout); PCeedChk(ierr);
    ierr = CeedHackFree(&emodein); PCeedChk(ierr);
    ierr = CeedHackFree(&emodeout); PCeedChk(ierr);
 
@@ -353,8 +310,13 @@ int CeedOperatorFullAssemble(CeedOperator op, SparseMatrix **mat)
    {
       CeedInt numsub;
       CeedOperator *subops;
-      ierr = CeedOperatorCompositeGetNumSub(op, &numsub); PCeedChk(ierr);
-      ierr = CeedOperatorCompositeGetSubList(op, &subops); PCeedChk(ierr);
+#if CEED_VERSION_GE(0, 10, 2)
+      CeedCompositeOperatorGetNumSub(op, &numsub);
+      ierr = CeedCompositeOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#else
+      CeedOperatorGetNumSub(op, &numsub);
+      ierr = CeedOperatorGetSubList(op, &subops); PCeedChk(ierr);
+#endif
       for (int i = 0; i < numsub; ++i)
       {
          ierr = CeedSingleOperatorFullAssemble(subops[i], out); PCeedChk(ierr);

fem/ceed/solvers/solvers-atpmg.cpp

@@ -120,11 +120,7 @@ int CeedATPMGElemRestriction(int order,
    }
    ierr = CeedVectorRestoreArray(in_lvec, &lvec_data); PCeedChk(ierr);
    CeedInt in_layout[3];
-#if CEED_VERSION_GE(0, 13, 0)
-   ierr = CeedElemRestrictionGetELayout(er_in, in_layout); PCeedChk(ierr);
-#else
    ierr = CeedElemRestrictionGetELayout(er_in, &in_layout); PCeedChk(ierr);
-#endif
    if (in_layout[0] == 0 && in_layout[1] == 0 && in_layout[2] == 0)
    {
       return CeedError(ceed, 1, "Cannot interpret e-vector ordering of given"
@@ -668,11 +664,7 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
 
    for (int i = 0; i < numinputfields; ++i)
    {
-#if CEED_VERSION_GE(0, 13, 0)
-      const char * fieldname;
-#else
       char * fieldname;
-#endif
       ierr = CeedQFunctionFieldGetName(inputqfields[i], &fieldname); PCeedChk(ierr);
       if (if_vector[i] == CEED_VECTOR_ACTIVE)
       {
@@ -684,19 +676,10 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
          ierr = CeedOperatorSetField(coper, fieldname, er_input[i], basis_input[i],
                                      if_vector[i]); PCeedChk(ierr);
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&if_vector[i]); PCeedChk(ierr);
-      ierr = CeedElemRestrictionDestroy(&er_input[i]); PCeedChk(ierr);
-      ierr = CeedBasisDestroy(&basis_input[i]); PCeedChk(ierr);
-#endif
    }
    for (int i = 0; i < numoutputfields; ++i)
    {
-#if CEED_VERSION_GE(0, 13, 0)
-      const char * fieldname;
-#else
       char * fieldname;
-#endif
       ierr = CeedQFunctionFieldGetName(outputqfields[i], &fieldname); PCeedChk(ierr);
       if (of_vector[i] == CEED_VECTOR_ACTIVE)
       {
@@ -708,11 +691,6 @@ int CeedATPMGOperator(CeedOperator oper, int order_reduction,
          ierr = CeedOperatorSetField(coper, fieldname, er_output[i], basis_output[i],
                                      of_vector[i]); PCeedChk(ierr);
       }
-#if CEED_VERSION_GE(0, 13, 0)
-      ierr = CeedVectorDestroy(&of_vector[i]); PCeedChk(ierr);
-      ierr = CeedElemRestrictionDestroy(&er_output[i]); PCeedChk(ierr);
-      ierr = CeedBasisDestroy(&basis_output[i]); PCeedChk(ierr);
-#endif
    }
    delete [] er_input;
    delete [] er_output;
@@ -763,9 +741,7 @@ int CeedOperatorGetOrder(CeedOperator oper, CeedInt * order)
    int P1d;
    ierr = CeedBasisGetNumNodes1D(basis, &P1d); PCeedChk(ierr);
    *order = P1d - 1;
-#if CEED_VERSION_GE(0, 13, 0)
-   ierr = CeedBasisDestroy(&basis); PCeedChk(ierr);
-#endif
+
    return 0;
 }
 

fem/coefficient.cpp

@@ -12,7 +12,6 @@
 // Implementation of Coefficient class
 
 #include "fem.hpp"
-#include "../general/forall.hpp"
 
 #include <cmath>
 #include <limits>
@@ -51,6 +50,7 @@ ElementTransformation *RefinedToCoarse(
 
 void Coefficient::Project(QuadratureFunction &qf)
 {
+   MFEM_PERF_FUNCTION;
    QuadratureSpaceBase &qspace = *qf.GetSpace();
    const int ne = qspace.GetNE();
    Vector values;
@@ -81,49 +81,6 @@ real_t PWConstCoefficient::Eval(ElementTransformation & T,
    return (constants(att-1));
 }
 
-void PWConstCoefficient::Project(QuadratureFunction &qf)
-{
-   auto &qs = *qf.GetSpace();
-
-   const bool compressed =
-      qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Size() == 1;
-   const int *offsets = qs.Offsets(QSpaceOffsetStorage::COMPRESSED).Read();
-   const int ne = qs.GetNE();
-
-   const int *attributes = [&]()
-   {
-      if (dynamic_cast<QuadratureSpace*>(&qs) != nullptr)
-      {
-         return qs.GetMesh()->GetElementAttributes().Read();
-      }
-      else if (auto *qs_f = dynamic_cast<FaceQuadratureSpace*>(&qs))
-      {
-         MFEM_VERIFY(qs_f->GetFaceType() == FaceType::Boundary,
-                     "Interior faces do not have attributes.");
-         return qs.GetMesh()->GetBdrFaceAttributes().Read();
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported case.");
-      }
-   }();
-
-   const real_t *d_c = constants.Read();
-   real_t *d_qf = qf.Write();
-
-   mfem::forall(ne, [=] MFEM_HOST_DEVICE (int e)
-   {
-      const int a = attributes[e];
-      const real_t elementConstant = d_c[a - 1];
-      const int begin = compressed ? e*offsets[0] : offsets[e];
-      const int end = compressed ? (e+1)*offsets[0] : offsets[e+1];
-      for (int i = begin; i < end; ++i)
-      {
-         d_qf[i] = elementConstant;
-      }
-   });
-}
-
 void PWCoefficient::InitMap(const Array<int> & attr,
                             const Array<Coefficient*> & coefs)
 {
@@ -563,26 +520,6 @@ void GradientGridFunctionCoefficient::Eval(
    }
 }
 
-void GradientGridFunctionCoefficient::Project(QuadratureFunction &qf)
-{
-   const FiniteElementSpace &fes = *GridFunc->FESpace();
-   const Mesh &mesh = *fes.GetMesh();
-   const int sdim = mesh.SpaceDimension();
-   const int gf_vdim = fes.GetVDim(); // assumed to be 1 in this class
-   qf.SetVDim(sdim*gf_vdim);
-   if (mesh.GetNE() == 0) { return; }
-   // All mesh element must be the same type:
-   MFEM_VERIFY(mesh.GetNumGeometries(mesh.Dimension()) == 1,
-               "All mesh elements must be the same type!");
-   const IntegrationRule &ir = qf.GetIntRule(0);
-   // All elements must use the same quadrature rule:
-   MFEM_VERIFY(qf.Size() == sdim*gf_vdim*ir.GetNPoints()*mesh.GetNE(),
-               "All mesh elements must use the same quadrature rule!");
-   // QuadratureFunction uses the layout qf_vdim x nq x ne, i.e.
-   // gf_vdim x sdim x nq x nq, so we need to request QVectorLayout::byVDIM:
-   GridFunc->GetGradients(ir, qf, QVectorLayout::byVDIM);
-}
-
 CurlGridFunctionCoefficient::CurlGridFunctionCoefficient(
    const GridFunction *gf)
    : VectorCoefficient(0)
@@ -1129,41 +1066,6 @@ real_t InnerProductCoefficient::Eval(ElementTransformation &T,
    return va * vb;
 }
 
-void InnerProductCoefficient::Project(QuadratureFunction &qf)
-{
-   MFEM_VERIFY(a->GetVDim() == b->GetVDim(),
-               "Incompatible vector coefficients: a->GetVDim(): "
-               << a->GetVDim() << ", b->GetVDim(): " << b->GetVDim());
-
-   const int vdim = a->GetVDim();
-   MFEM_VERIFY(vdim >= 1, "invalid vdim: " << vdim);
-
-   // When running on device, make sure the output data is allocated before any
-   // local temporary data to reduce potential heap fragmentation:
-   auto dot_d = qf.Write();
-
-   QuadratureFunction qf_a(qf.GetSpace(), vdim);
-   QuadratureFunction qf_b(qf.GetSpace(), vdim);
-
-   a->Project(qf_a);
-   b->Project(qf_b);
-
-   auto a_d = qf_a.Read();
-   auto b_d = qf_b.Read();
-
-   mfem::forall(qf.GetSpace()->GetSize(), [=] MFEM_HOST_DEVICE (int i)
-   {
-      const real_t *ai = a_d + i*vdim;
-      const real_t *bi = b_d + i*vdim;
-      real_t dot = ai[0]*bi[0];
-      for (int d = 1; d < vdim; d++)
-      {
-         dot += ai[d]*bi[d];
-      }
-      dot_d[i] = dot;
-   });
-}
-
 VectorRotProductCoefficient::VectorRotProductCoefficient(VectorCoefficient &A,
                                                          VectorCoefficient &B)
    : a(&A), b(&B), va(A.GetVDim()), vb(B.GetVDim())

fem/coefficient.hpp

@@ -132,9 +132,6 @@ public:
    /// Evaluate the coefficient.
    real_t Eval(ElementTransformation &T,
                const IntegrationPoint &ip) override;
-
-   /// Fill the QuadratureFunction @a qf with the piecewise constant values.
-   void Project(QuadratureFunction &qf) override;
 };
 
 /** @brief A piecewise coefficient with the pieces keyed off the element
@@ -897,9 +894,6 @@ public:
    void Eval(DenseMatrix &M, ElementTransformation &T,
              const IntegrationRule &ir) override;
 
-   /// @copydoc VectorCoefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
-
    virtual ~GradientGridFunctionCoefficient() { }
 };
 
@@ -1777,9 +1771,6 @@ public:
    /// Evaluate the coefficient at @a ip.
    real_t Eval(ElementTransformation &T,
                const IntegrationPoint &ip) override;
-
-   /// @copydoc Coefficient::Project(QuadratureFunction &)
-   void Project(QuadratureFunction &qf) override;
 };
 
 /// Scalar coefficient defined as a cross product of two vectors in the xy-plane.

fem/dfem/doperator.hpp

@@ -231,53 +231,22 @@ public:
       const std::vector<FieldDescriptor> &parameters,
       const ParMesh &mesh);
 
-   /// MultLevel enum to indicate if the T->L Operators are used in the
-   /// Mult method.
-   enum MultLevel
-   {
-      TVECTOR,
-      LVECTOR
-   };
-
-   /// @brief Set the MultLevel mode for the DifferentiableOperator.
-   /// The default is TVECTOR, which means that the Operator will use
-   /// T->L before Mult and L->T Operators after.
-   void SetMultLevel(MultLevel level)
-   {
-      mult_level = level;
-   }
-
    /// @brief Compute the action of the operator on a given vector.
    ///
-   /// @param solutions_in The solution vector in which to compute the action.
-   /// This has to be a T-dof vector if MultLevel is set to TVECTOR, or L-dof
-   /// Vector if MultLevel is set to LVECTOR.
-   /// @param result_in Result vector of the action of the operator on
-   /// solutions. The result is a T-dof vector or L-dof vector depending on
-   /// the MultLevel.
-   void Mult(const Vector &solutions_in, Vector &result_in) const override
+   /// @param solutions_t The solution vector in which to compute the action.
+   /// This has to be a T-dof vector.
+   /// @param result_t Result vector of the action of the operator on
+   /// solutions_t. The result is a T-dof vector.
+   void Mult(const Vector &solutions_t, Vector &result_t) const override
    {
       MFEM_ASSERT(!action_callbacks.empty(), "no integrators have been set");
-
-      if (mult_level == MultLevel::LVECTOR)
+      prolongation(solutions, solutions_t, solutions_l);
+      residual_l = 0.0;
+      for (auto &action : action_callbacks)
       {
-         get_lvectors(solutions, solutions_in, solutions_l);
-         result_in = 0.0;
-         for (auto &action : action_callbacks)
-         {
-            action(solutions_l, parameters_l, result_in);
-         }
-      }
-      else
-      {
-         prolongation(solutions, solutions_in, solutions_l);
-         residual_l = 0.0;
-         for (auto &action : action_callbacks)
-         {
-            action(solutions_l, parameters_l, residual_l);
-         }
-         prolongation_transpose(residual_l, result_in);
+         action(solutions_l, parameters_l, residual_l);
       }
+      prolongation_transpose(residual_l, result_t);
    }
 
    /// @brief Add a domain integrator to the operator.
@@ -376,8 +345,6 @@ public:
 private:
    const ParMesh &mesh;
 
-   MultLevel mult_level = TVECTOR;
-
    std::vector<action_t> action_callbacks;
    std::map<size_t,
        std::vector<derivative_action_t>> derivative_action_callbacks;
@@ -387,6 +354,7 @@ private:
        std::vector<assemble_derivative_hypreparmatrix_callback_t>>
        assemble_derivative_hypreparmatrix_callbacks;
 
+
    std::vector<FieldDescriptor> solutions;
    std::vector<FieldDescriptor> parameters;
    // solutions and parameters

fem/dfem/util.hpp

@@ -1076,24 +1076,6 @@ void prolongation(const std::vector<FieldDescriptor> fields,
    }
 }
 
-inline
-void get_lvectors(const std::vector<FieldDescriptor> fields,
-                  const Vector &x,
-                  std::vector<Vector> &fields_l)
-{
-   int data_offset = 0;
-   for (std::size_t i = 0; i < fields.size(); i++)
-   {
-      const int sz = GetVSize(fields[i]);
-      fields_l[i].SetSize(sz);
-
-      const Vector x_i(const_cast<Vector&>(x), data_offset, sz);
-      fields_l[i] = x_i;
-
-      data_offset += sz;
-   }
-}
-
 /// @brief Get a transpose prolongation callback for a field descriptor.
 ///
 /// In the special case of a one field operator, the transpose prolongation

fem/fe/face_map_utils.hpp

@@ -259,30 +259,6 @@ inline void FaceIdxToVolIdx3D(const int index, const int size1d,
    i = yz_plane ? level : _i;
 }
 
-MFEM_HOST_DEVICE
-inline int FaceIdxToVolIdx(int dim, int i, int size1d, int face0, int face1,
-                           int side, int orientation)
-{
-   if (dim == 2)
-   {
-      int ix, iy;
-      internal::FaceIdxToVolIdx2D(i, size1d, face0, face1, side, ix, iy);
-      return ix + iy*size1d;
-   }
-   else if (dim == 3)
-   {
-      int ix, iy, iz;
-      internal::FaceIdxToVolIdx3D(i, size1d, face0, face1, side, orientation,
-                                  ix, iy, iz);
-      return ix + size1d*iy + size1d*size1d*iz;
-   }
-   else
-   {
-      MFEM_ABORT_KERNEL("Invalid dimension");
-      return -1;
-   }
-};
-
 } // namespace internal
 
 } // namespace mfem

fem/fe_coll.cpp

@@ -401,6 +401,9 @@ FiniteElementCollection *FiniteElementCollection::New(const char *name)
    {
       MFEM_ABORT("unknown FiniteElementCollection: " << name);
    }
+   MFEM_VERIFY(!strcmp(fec->Name(), name), "input name: \"" << name
+               << "\" does not match the created collection name: \""
+               << fec->Name() << '"');
 
    return fec;
 }
@@ -2456,7 +2459,8 @@ RT_FECollection::RT_FECollection(const int order, const int dim,
       const char *cb_name = BasisType::Name(cb_type); // this may abort
       MFEM_ABORT("unknown closed BasisType: " << cb_name);
    }
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
+       ob_type != BasisType::IntegratedGLL)
    {
       const char *ob_name = BasisType::Name(ob_type); // this may abort
       MFEM_ABORT("unknown open BasisType: " << ob_name);
@@ -2514,7 +2518,6 @@ RT_FECollection::RT_FECollection(const int p, const int dim,
                                  const int map_type, const bool signs,
                                  const int ob_type)
    : FiniteElementCollection(p + 1)
-   , dim(dim)
    , ob_type(ob_type)
 {
    if (Quadrature1D::CheckOpen(BasisType::GetQuadrature1D(ob_type)) ==
@@ -2783,7 +2786,8 @@ ND_FECollection::ND_FECollection(const int p, const int dim,
    int cp_type = BasisType::GetQuadrature1D(cb_type);
 
    // Error checking
-   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid)
+   if (Quadrature1D::CheckOpen(op_type) == Quadrature1D::Invalid &&
+       ob_type != BasisType::IntegratedGLL)
    {
       const char *ob_name = BasisType::Name(ob_type);
       MFEM_ABORT("Invalid open basis point type: " << ob_name);

fem/fe_coll.hpp

@@ -464,13 +464,6 @@ public:
    RT_Trace_FECollection(const int p, const int dim,
                          const int map_type = FiniteElement::INTEGRAL,
                          const int ob_type = BasisType::GaussLegendre);
-
-   FiniteElementCollection *Clone(int p) const override
-   {
-      const int map_type = (strncmp(rt_name, "RT_Trace", 8) == 0)?
-                           (FiniteElement::INTEGRAL):(FiniteElement::VALUE);
-      return new RT_Trace_FECollection(p, dim, map_type, ob_type);
-   }
 };
 
 /** Arbitrary order discontinuous finite elements defined on the interface
@@ -482,13 +475,6 @@ public:
    DG_Interface_FECollection(const int p, const int dim,
                              const int map_type = FiniteElement::VALUE,
                              const int ob_type = BasisType::GaussLegendre);
-
-   FiniteElementCollection *Clone(int p) const override
-   {
-      const int map_type = (strncmp(rt_name, "DG_Iface", 8) == 0)?
-                           (FiniteElement::VALUE):(FiniteElement::INTEGRAL);
-      return new DG_Interface_FECollection(p, dim, map_type, ob_type);
-   }
 };
 
 /// Arbitrary order H(curl)-conforming Nedelec finite elements.

fem/fes_kernels.hpp

@@ -224,7 +224,7 @@ struct DerefineMatrixOpMultFunctor
                sum += sign * bsptr[boptr[k] + i + j * block_height] *
                       xptr[this->IndexX(col, vdim, k)];
             }
-#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)
+#if defined(__CUDA_ARCH__) or defined(__HIP_DEVICE_COMPILE__)
             if (Atomic)
             {
                atomicAdd(yptr + this->IndexY(row, vdim), sum);

fem/fespace.cpp

@@ -101,7 +101,10 @@ FiniteElementSpace::FiniteElementSpace(const FiniteElementSpace &orig,
 FiniteElementSpace::FiniteElementSpace(Mesh *mesh,
                                        const FiniteElementCollection *fec,
                                        int vdim, int ordering)
-{ Constructor(mesh, NULL, fec, vdim, ordering); }
+{
+   MFEM_PERF_FUNCTION;
+   Constructor(mesh, NULL, fec, vdim, ordering);
+}
 
 FiniteElementSpace::FiniteElementSpace(Mesh *mesh, NURBSExtension *ext,
                                        const FiniteElementCollection *fec,
@@ -393,6 +396,8 @@ void FiniteElementSpace::BuildElementToDofTable() const
 {
    if (elem_dof) { return; }
 
+   MFEM_PERF_FUNCTION;
+
    // TODO: can we call GetElementDofs only once per element?
    Table *el_dof = new Table;
    Table *el_fos = (mesh->Dimension() > 2) ? (new Table) : NULL;
@@ -2748,6 +2753,8 @@ void FiniteElementSpace::BuildNURBSFaceToDofTable() const
 
 void FiniteElementSpace::Construct()
 {
+   MFEM_PERF_FUNCTION;
+
    // This method should be used only for non-NURBS spaces.
    MFEM_VERIFY(!NURBSext, "internal error");
 

fem/fespace.hpp

@@ -683,12 +683,8 @@ public:
    NURBSExtension *GetNURBSext() { return NURBSext; }
    NURBSExtension *StealNURBSext();
 
-   bool Conforming() const
-   {
-      return NURBSext != NULL ||
-             (mesh->Conforming() && cP == NULL);
-   }
-   bool Nonconforming() const { return !Conforming(); }
+   bool Conforming() const { return mesh->Conforming() && cP == NULL; }
+   bool Nonconforming() const { return mesh->Nonconforming() || cP != NULL; }
 
    /** Set the prolongation operator of the space to an arbitrary sparse matrix,
        creating a copy of the argument. */
@@ -926,9 +922,6 @@ public:
    { return mesh->GetBdrElementType(i); }
 
    /// Returns ElementTransformation for the @a i-th element.
-   /// @note The returned pointer references an object owned by the associated
-   /// @a Mesh that will be modified by other calls to `GetElementTransformation`.
-   /// As such, this pointer should @b not be deleted by the caller.
    ElementTransformation *GetElementTransformation(int i) const
    { return mesh->GetElementTransformation(i); }
 

fem/gridfunc.cpp

@@ -19,6 +19,7 @@
 #include "../mesh/nurbs.hpp"
 #include "../mesh/vtkhdf.hpp"
 #include "../general/text.hpp"
+#include "../general/reducers.hpp"
 
 #ifdef MFEM_USE_MPI
 #include "pfespace.hpp"
@@ -68,7 +69,7 @@ GridFunction::GridFunction(Mesh *m, std::istream &input)
       Vector::Load(input, fes->GetVSize());
 
       // if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
-      if (fes->Nonconforming() && fes->GetMesh()->ncmesh &&
+      if (fes->Nonconforming() &&
           fes->GetMesh()->ncmesh->IsLegacyLoaded())
       {
          LegacyNCReorder();
@@ -1374,50 +1375,6 @@ void GridFunction::GetVectorGradientHat(
    MultAtB(loc_data_mat, dshape, gh);
 }
 
-void GridFunction::GetGradients(const IntegrationRule &ir, Vector &grad,
-                                QVectorLayout ql, MemoryType d_mt) const
-{
-   const FiniteElement &fe = *fes->GetTypicalFE();
-   const int dim  = fe.GetDim();
-   const int vdim = fes->GetVDim();
-   const int NE   = fes->GetNE();
-   const int ND   = fe.GetDof();
-   const int NQ   = ir.GetNPoints();
-
-   MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt :
-                        Device::GetDeviceMemoryType();
-
-   // ql == QVectorLayout::byNODES :   NQ x VDIM x  DIM x NE
-   // ql == QVectorLayout::byVDIM  : VDIM x  DIM x NQPT x NE
-   grad.SetSize(dim*vdim*NQ*NE, my_d_mt);
-
-   const QuadratureInterpolator &qi = *fes->GetQuadratureInterpolator(ir);
-   qi.SetOutputLayout(ql);
-
-   const bool use_tensor_products = UsesTensorBasis(*fes);
-   qi.DisableTensorProducts(!use_tensor_products);
-   const ElementDofOrdering e_ordering = use_tensor_products ?
-                                         ElementDofOrdering::LEXICOGRAPHIC :
-                                         ElementDofOrdering::NATIVE;
-   const Operator *elem_restr = fes->GetElementRestriction(e_ordering);
-
-   // Pre-compute the geometric factors in order to set the desired MemoryType
-   // they use:
-   fes->GetMesh()->GetGeometricFactors(
-      ir, GeometricFactors::JACOBIANS, my_d_mt);
-
-   if (elem_restr) // currently, always true
-   {
-      Vector f_e(vdim*ND*NE, my_d_mt);
-      elem_restr->Mult(*this, f_e);
-      qi.PhysDerivatives(f_e, grad);
-   }
-   else
-   {
-      qi.PhysDerivatives(*this, grad);
-   }
-}
-
 real_t GridFunction::GetDivergence(ElementTransformation &T) const
 {
    DofTransformation doftrans;
@@ -2668,7 +2625,7 @@ void GridFunction::ProjectBdrCoefficient(Coefficient *coeff[],
    }
    for (int i = 0; i < values_counter.Size(); i++)
    {
-      MFEM_ASSERT(bool(values_counter[i]) == bool(ess_vdofs_marker[i]),
+      MFEM_ASSERT(bool(values_counter[i]) == ess_vdofs_marker[i],
                   "internal error");
    }
 #endif
@@ -3370,8 +3327,126 @@ real_t GridFunction::ComputeLpError(const real_t p, Coefficient &exsol,
                                     const IntegrationRule *irs[],
                                     const Array<int> *elems) const
 {
+   MFEM_PERF_FUNCTION;
+
+   MFEM_VERIFY(fes->GetVDim() == 1, "invalid vector dimension!");
+
    real_t error = 0.0;
-   const FiniteElement *fe;
+
+   bool device_eval = true;
+   // TODO: check for cases that are not supported on device:
+   //       * mixed meshes
+   //       * meshes with non-tensor-product elements can have negative weights
+   //       * variable orders
+   //       * weight is not NULL
+   //       * elems is not NULL
+   //       * map type is not VALUE
+   //       * ...
+   Mesh *mesh = fes->GetMesh();
+   const FiniteElement *fe = fes->GetTypicalFE();
+   if (mesh->GetNumGeometries(mesh->Dimension()) > 1 ||
+       (mesh->Dimension() > 1 && mesh->MeshGenerator() != 2) ||
+       fes->IsVariableOrder() ||
+       weight != nullptr ||
+       elems != nullptr ||
+       fe->GetMapType() != FiniteElement::MapType::VALUE)
+   {
+      device_eval = false;
+   }
+   if (device_eval)
+   {
+      Geometry::Type geom = mesh->GetTypicalElementGeometry();
+      const IntegrationRule *ir_p;
+      if (irs)
+      {
+         ir_p = irs[geom];
+      }
+      else
+      {
+         int intorder = 2*fe->GetOrder() + 3; // <----------
+         ir_p = &(IntRules.Get(geom, intorder));
+      }
+      const IntegrationRule &ir = *ir_p;
+      QuadratureSpace qs(*mesh, ir);
+      CoefficientVector coeff(exsol, qs, CoefficientStorage::FULL);
+
+      const QVectorLayout ql = QVectorLayout::byNODES;
+      const MemoryType d_mt = MemoryType::DEFAULT;
+      Vector q_vals;
+      // TODO: make this a method
+      {
+         // const FiniteElement *fe = fes->GetTypicalFE();
+         const int vdim = fes->GetVDim();
+         const int NE   = fes->GetNE();
+         const int ND   = fe->GetDof();
+         const int NQ   = ir.GetNPoints();
+         MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt :
+                              Device::GetDeviceMemoryType();
+         // byNODES : NQPT x VDIM x NE
+         // byVDIM  : VDIM x NQPT x NE
+         q_vals.SetSize(vdim*NQ*NE, my_d_mt);
+         const QuadratureInterpolator &qi = *fes->GetQuadratureInterpolator(ir);
+         qi.SetOutputLayout(ql);
+         const bool use_tensor_products = UsesTensorBasis(*fes);
+         qi.DisableTensorProducts(!use_tensor_products);
+         const ElementDofOrdering e_ordering =
+            use_tensor_products ?
+            ElementDofOrdering::LEXICOGRAPHIC :
+            ElementDofOrdering::NATIVE;
+         const Operator *elem_restr = fes->GetElementRestriction(e_ordering);
+         if (fe->GetMapType() == FiniteElement::MapType::INTEGRAL)
+         {
+            // Pre-compute the geometric factors in order to set the desired
+            // MemoryType they use:
+            fes->GetMesh()->GetGeometricFactors(
+               ir, GeometricFactors::DETERMINANTS, my_d_mt);
+         }
+         if (elem_restr)
+         {
+            Vector f_e(vdim*ND*NE, my_d_mt);
+            elem_restr->Mult(*this, f_e);
+            qi.PhysValues(f_e, q_vals);
+         }
+         else
+         {
+            qi.PhysValues(*this, q_vals);
+         }
+      }
+
+      const real_t *exact_d = coeff.Read();
+      const real_t *gridf_d = q_vals.Read();
+      // FIXME: reuse the workspace vector from vector.cpp?
+      static Array<real_t> workspace;
+      if (p < infinity())
+      {
+         MemoryType my_d_mt = (d_mt != MemoryType::DEFAULT) ? d_mt :
+                              Device::GetDeviceMemoryType();
+         const GeometricFactors *geom_factors =
+            fes->GetMesh()->GetGeometricFactors(
+               ir, GeometricFactors::DETERMINANTS, my_d_mt);
+         const real_t *detJ_d = geom_factors->detJ.Read();
+         const real_t *w_d = ir.GetWeights().Read();
+         const int NQ = ir.GetNPoints();
+         mfem::reduce(q_vals.Size(), error,
+                      [=] MFEM_HOST_DEVICE(int i, real_t &r)
+         {
+            const real_t diff = fabs(exact_d[i] - gridf_d[i]);
+            r += w_d[i%NQ] * detJ_d[i] * pow(diff, p);
+         }, SumReducer<real_t> {}, true, workspace);
+         error = pow(error, 1./p);
+      }
+      else
+      {
+         mfem::reduce(q_vals.Size(), error,
+                      [=] MFEM_HOST_DEVICE(int i, real_t &r)
+         {
+            const real_t diff = fabs(exact_d[i] - gridf_d[i]);
+            r = fmax(r, diff);
+         }, MaxReducer<real_t> {}, true, workspace);
+      }
+      return error;
+   }
+
    ElementTransformation *T;
    Vector vals;
 

fem/gridfunc.hpp

@@ -153,8 +153,7 @@ public:
    /// Shortcut for calling SetFromTrueDofs() with GetTrueVector() as argument.
    void SetFromTrueVector() { SetFromTrueDofs(GetTrueVector()); }
 
-   /** @brief Returns the values at the vertices of element @a i for the 1-based
-       dimension vdim. */
+   /// Returns the values in the vertices of i'th element for dimension vdim.
    void GetNodalValues(int i, Array<real_t> &nval, int vdim = 1) const;
 
    /** @name Element index Get Value Methods
@@ -309,8 +308,7 @@ public:
    /// For a vector grid function, makes sure that the ordering is byNODES.
    void ReorderByNodes();
 
-   /** @brief Returns the values as a vector at mesh vertices, for the 1-based
-       dimension vdim. */
+   /// Return the values as a vector on mesh vertices for dimension vdim.
    void GetNodalValues(Vector &nval, int vdim = 1) const;
 
    void GetVectorFieldNodalValues(Vector &val, int comp) const;
@@ -361,33 +359,6 @@ public:
        variable. */
    void GetVectorGradientHat(ElementTransformation &T, DenseMatrix &gh) const;
 
-   /** @brief Evaluate the gradients of the GridFunction at the given quadrature
-       points, @a ir, in all mesh elements. */
-   /** This method assumes that all mesh elements are the same type and that the
-       IntegrationRule @a ir is consistent with that type of element.
-
-       @param[in] ir     Quadrature points at which the gradients are to be
-                         evaluated.
-       @param[out] grad  Output vector of size `SDIM*VDIM*NQ*NE` where `SDIM` is
-                         the spatial dimention of the mesh, `VDIM` is the vector
-                         dimension of the GridFunction, `NQ` is the number of
-                         quadrature points in @a ir, and `NE` is the number of
-                         elements in the mesh. The layout of @a grad is
-                         determined by the parameter @a ql: when @a ql is
-                         QVectorLayout::byNODES, the layout is
-                         `NQ x VDIM x SDIM x NE`; when @a ql is
-                         QVectorLayout::byVDIM, the layout is
-                         `VDIM x SDIM x NQ x NE`.
-       @param[in] ql     Determines the layout of the output vector @a grad; see
-                         the description of @a grad for details.
-       @param[in] d_mt   MemoryType to use for allocating the output vector
-                         @a grad, as well the GeometricFactors and temporary
-                         vector used by the method. By default, the current
-                         device memory type is used. */
-   void GetGradients(const IntegrationRule &ir, Vector &grad,
-                     QVectorLayout ql = QVectorLayout::byNODES,
-                     MemoryType d_mt = MemoryType::DEFAULT) const;
-
    /** Compute $ (\int_{\Omega} (*this) \psi_i)/(\int_{\Omega} \psi_i) $,
        where $ \psi_i $ are the basis functions for the FE space of avgs.
        Both FE spaces should be scalar and on the same mesh. */

fem/gslib.cpp

@@ -85,9 +85,9 @@ namespace mfem
 {
 
 FindPointsGSLIB::FindPointsGSLIB()
-   : mesh(nullptr),
-     fec_map_lin(nullptr),
-     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
+   : mesh(NULL),
+     fec_map_lin(NULL),
+     fdataD(NULL), cr(NULL), gsl_comm(NULL),
      dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -97,10 +97,10 @@ FindPointsGSLIB::FindPointsGSLIB()
    gf_rst_map.SetSize(4);
    for (int i = 0; i < mesh_split.Size(); i++)
    {
-      mesh_split[i] = nullptr;
-      ir_split[i] = nullptr;
-      fes_rst_map[i] = nullptr;
-      gf_rst_map[i] = nullptr;
+      mesh_split[i] = NULL;
+      ir_split[i] = NULL;
+      fes_rst_map[i] = NULL;
+      gf_rst_map[i] = NULL;
    }
 
    gsl_comm = new gslib::comm;
@@ -117,40 +117,27 @@ FindPointsGSLIB::FindPointsGSLIB()
    crystal_init(cr, gsl_comm);
 }
 
-FindPointsGSLIB::FindPointsGSLIB(Mesh &mesh_in, const double bb_t,
-                                 const double newt_tol, const int npt_max)
-   : FindPointsGSLIB()
-{
-   Setup(mesh_in, bb_t, newt_tol, npt_max);
-}
-
 FindPointsGSLIB::~FindPointsGSLIB()
 {
-   FreeData();
-#ifdef MFEM_USE_MPI
-   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
-#endif
+   crystal_free(cr);
+   comm_free(gsl_comm);
+   delete gsl_comm;
+   delete cr;
+   for (int i = 0; i < 4; i++)
    {
-      crystal_free(cr);
-      comm_free(gsl_comm);
-      delete gsl_comm;
-      delete cr;
+      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = NULL; }
+      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = NULL; }
+      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = NULL; }
+      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = NULL; }
    }
-   for (int i = 0; i < mesh_split.Size(); i++)
-   {
-      if (mesh_split[i]) { delete mesh_split[i]; mesh_split[i] = nullptr; }
-      if (ir_split[i]) { delete ir_split[i]; ir_split[i] = nullptr; }
-      if (fes_rst_map[i]) { delete fes_rst_map[i]; fes_rst_map[i] = nullptr; }
-      if (gf_rst_map[i]) { delete gf_rst_map[i]; gf_rst_map[i] = nullptr; }
-   }
-   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = nullptr; }
+   if (fec_map_lin) { delete fec_map_lin; fec_map_lin = NULL; }
 }
 
 #ifdef MFEM_USE_MPI
 FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
-   : mesh(nullptr),
-     fec_map_lin(nullptr),
-     fdataD(nullptr), cr(nullptr), gsl_comm(nullptr),
+   : mesh(NULL),
+     fec_map_lin(NULL),
+     fdataD(NULL), cr(NULL), gsl_comm(NULL),
      dim(-1), points_cnt(-1), setupflag(false), default_interp_value(0),
      avgtype(AvgType::ARITHMETIC), bdr_tol(1e-8)
 {
@@ -160,10 +147,10 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    gf_rst_map.SetSize(4);
    for (int i = 0; i < mesh_split.Size(); i++)
    {
-      mesh_split[i] = nullptr;
-      ir_split[i] = nullptr;
-      fes_rst_map[i] = nullptr;
-      gf_rst_map[i] = nullptr;
+      mesh_split[i] = NULL;
+      ir_split[i] = NULL;
+      fes_rst_map[i] = NULL;
+      gf_rst_map[i] = NULL;
    }
 
    gsl_comm = new gslib::comm;
@@ -171,21 +158,12 @@ FindPointsGSLIB::FindPointsGSLIB(MPI_Comm comm_)
    comm_init(gsl_comm, comm_);
    crystal_init(cr, gsl_comm);
 }
-
-FindPointsGSLIB::FindPointsGSLIB(ParMesh &mesh_in, const double bb_t,
-                                 const double newt_tol, const int npt_max)
-   : FindPointsGSLIB(mesh_in.GetComm())
-{
-   Setup(mesh_in, bb_t, newt_tol, npt_max);
-}
 #endif
 
 void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
                             const int npt_max)
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
-   MFEM_VERIFY(m.SpaceDimension() == m.Dimension(),
-               "Mesh spatial dimension and reference element dimension must be the same");
    const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
 
    // call FreeData if FindPointsGSLIB::Setup has been called already
@@ -193,9 +171,37 @@ void FindPointsGSLIB::Setup(Mesh &m, const double bb_t, const double newt_tol,
 
    mesh = &m;
    dim  = mesh->Dimension();
-   const unsigned int dof1D = meshOrder+1;
+   unsigned dof1D = meshOrder + 1;
 
-   SetupSplitMeshesAndIntegrationRules(meshOrder);
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
+   }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
 
@@ -1122,18 +1128,13 @@ void FindPointsGSLIB::Interpolate(Mesh &m, const Vector &point_pos,
 void FindPointsGSLIB::FreeData()
 {
    if (!setupflag) { return; }
-#ifdef MFEM_USE_MPI
-   if (!Mpi::IsFinalized())  // currently segfaults inside gslib otherwise
-#endif
+   if (dim == 2)
    {
-      if (dim == 2)
-      {
-         findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
-      }
-      else
-      {
-         findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
-      }
+      findpts_free_2((gslib::findpts_data_2 *)this->fdataD);
+   }
+   else
+   {
+      findpts_free_3((gslib::findpts_data_3 *)this->fdataD);
    }
    gsl_code.DeleteAll();
    gsl_proc.DeleteAll();
@@ -1157,8 +1158,8 @@ void FindPointsGSLIB::FreeData()
 
 void FindPointsGSLIB::SetupSplitMeshes()
 {
-   if (fec_map_lin == nullptr) { fec_map_lin = new H1_FECollection(1, dim); }
-   if (dim == 2)
+   fec_map_lin = new H1_FECollection(1, dim);
+   if (mesh->Dimension() == 2)
    {
       int Nvert = 7;
       int NEsplit = 3;
@@ -1200,7 +1201,7 @@ void FindPointsGSLIB::SetupSplitMeshes()
       mesh_split[1] = new Mesh(Mesh::MakeCartesian2D(1, 1,
                                                      Element::QUADRILATERAL));
    }
-   else if (dim == 3)
+   else if (mesh->Dimension() == 3)
    {
       mesh_split[0] = new Mesh(Mesh::MakeCartesian3D(1, 1, 1,
                                                      Element::HEXAHEDRON));
@@ -1345,6 +1346,41 @@ void FindPointsGSLIB::SetupSplitMeshes()
          }
       }
    }
+
+   NE_split_total = 0;
+   split_element_map.SetSize(0);
+   split_element_index.SetSize(0);
+   int NEsplit = 0;
+   for (int e = 0; e < mesh->GetNE(); e++)
+   {
+      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
+      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
+      {
+         NEsplit = 3;
+      }
+      else if (gt == Geometry::TETRAHEDRON)
+      {
+         NEsplit = 4;
+      }
+      else if (gt == Geometry::PYRAMID)
+      {
+         NEsplit = 8;
+      }
+      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
+      {
+         NEsplit = 1;
+      }
+      else
+      {
+         MFEM_ABORT("Unsupported geometry type.");
+      }
+      NE_split_total += NEsplit;
+      for (int i = 0; i < NEsplit; i++)
+      {
+         split_element_map.Append(e);
+         split_element_index.Append(i);
+      }
+   }
 }
 
 void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
@@ -1395,79 +1431,6 @@ void FindPointsGSLIB::SetupIntegrationRuleForSplitMesh(Mesh *meshin,
    }
 }
 
-void FindPointsGSLIB::SetupSplitMeshesAndIntegrationRules(const int order)
-{
-   MFEM_VERIFY(mesh, "Setup FindPointsGSLIB with mesh first.");
-   const int dof1D = order+1;
-   const int dim = mesh->Dimension();
-
-   SetupSplitMeshes();
-   if (dim == 2)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
-   }
-   else if (dim == 3)
-   {
-      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
-      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], order);
-
-      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
-      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], order);
-
-      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
-      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], order);
-
-      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
-      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
-      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], order);
-   }
-
-   // Setup map for non tensor-product elements
-   NE_split_total = 0;
-   split_element_map.SetSize(0);
-   split_element_index.SetSize(0);
-   int NEsplit = 0;
-   for (int e = 0; e < mesh->GetNE(); e++)
-   {
-      const Geometry::Type gt   = mesh->GetElement(e)->GetGeometryType();
-      if (gt == Geometry::TRIANGLE || gt == Geometry::PRISM)
-      {
-         NEsplit = 3;
-      }
-      else if (gt == Geometry::TETRAHEDRON)
-      {
-         NEsplit = 4;
-      }
-      else if (gt == Geometry::PYRAMID)
-      {
-         NEsplit = 8;
-      }
-      else if (gt == Geometry::SQUARE || gt == Geometry::CUBE)
-      {
-         NEsplit = 1;
-      }
-      else
-      {
-         MFEM_ABORT("Unsupported geometry type.");
-      }
-      NE_split_total += NEsplit;
-      for (int i = 0; i < NEsplit; i++)
-      {
-         split_element_map.Append(e);
-         split_element_index.Append(i);
-      }
-   }
-}
-
 void FindPointsGSLIB::GetNodalValues(const GridFunction *gf_in,
                                      Vector &node_vals)
 {
@@ -2118,19 +2081,6 @@ void FindPointsGSLIB::InterpolateGeneral(const GridFunction &field_in,
    } // parallel
 }
 
-Array<unsigned int> FindPointsGSLIB::GetPointsNotFoundIndices() const
-{
-   Array<unsigned int> nf_idxs;
-   for (int i = 0; i < gsl_code.Size(); i++)
-   {
-      if (gsl_code[i] == 2)
-      {
-         nf_idxs.Append(i);
-      }
-   }
-   return nf_idxs;
-}
-
 void FindPointsGSLIB::DistributePointInfoToOwningMPIRanks(
    Array<unsigned int> &recv_elem, Vector &recv_ref,
    Array<unsigned int> &recv_code)
@@ -2436,10 +2386,6 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
 {
    MFEM_VERIFY(m.GetNodes() != NULL, "Mesh nodes are required.");
    const int meshOrder = m.GetNodes()->FESpace()->GetMaxElementOrder();
-   const int gfOrder = gfmax ? gfmax->FESpace()->GetMaxElementOrder() :
-                       meshOrder;
-   MFEM_VERIFY(meshOrder == gfOrder,
-               "Mesh order must match gfmax order in OversetFindPointsGSLIB.");
 
    // FreeData if OversetFindPointsGSLIB::Setup has been called already
    if (setupflag) { FreeData(); }
@@ -2449,7 +2395,35 @@ void OversetFindPointsGSLIB::Setup(Mesh &m, const int meshid,
    const FiniteElement *fe = mesh->GetNodalFESpace()->GetTypicalFE();
    unsigned dof1D = fe->GetOrder() + 1;
 
-   SetupSplitMeshesAndIntegrationRules(meshOrder);
+   SetupSplitMeshes();
+   if (dim == 2)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+   }
+   else if (dim == 3)
+   {
+      if (ir_split[0]) { delete ir_split[0]; ir_split[0] = NULL; }
+      ir_split[0] = new IntegrationRule(pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[0], ir_split[0], meshOrder);
+
+      if (ir_split[1]) { delete ir_split[1]; ir_split[1] = NULL; }
+      ir_split[1] = new IntegrationRule(4*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[1], ir_split[1], meshOrder);
+
+      if (ir_split[2]) { delete ir_split[2]; ir_split[2] = NULL; }
+      ir_split[2] = new IntegrationRule(3*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[2], ir_split[2], meshOrder);
+
+      if (ir_split[3]) { delete ir_split[3]; ir_split[3] = NULL; }
+      ir_split[3] = new IntegrationRule(8*pow(dof1D, dim));
+      SetupIntegrationRuleForSplitMesh(mesh_split[3], ir_split[3], meshOrder);
+   }
 
    GetNodalValues(mesh->GetNodes(), gsl_mesh);
 
@@ -2506,7 +2480,7 @@ void OversetFindPointsGSLIB::FindPoints(const Vector &point_pos,
 {
    MFEM_VERIFY(setupflag, "Use OversetFindPointsGSLIB::Setup before "
                "finding points.");
-   MFEM_VERIFY(overset, "Please use OversetFindPoints for overlapping grids.");
+   MFEM_VERIFY(overset, "Please setup FindPoints for overlapping grids.");
    points_cnt = point_pos.Size() / dim;
    unsigned int match = 0; // Don't find points in the mesh if point_id=mesh_id
 

fem/gslib.hpp

@@ -13,11 +13,7 @@
 #define MFEM_GSLIB
 
 #include "../config/config.hpp"
-#ifdef MFEM_USE_MPI
-#include "pgridfunc.hpp"
-#else
 #include "gridfunc.hpp"
-#endif
 
 #ifdef MFEM_USE_GSLIB
 
@@ -135,10 +131,6 @@ protected:
                                                  IntegrationRule *irule,
                                                  int order);
 
-   /// Helper function that calls \ref SetupSplitMeshes and
-   /// \ref SetupIntegrationRuleForSplitMesh.
-   virtual void SetupSplitMeshesAndIntegrationRules(const int order);
-
    /// Get GridFunction value at the points expected by GSLIB.
    virtual void GetNodalValues(const GridFunction *gf_in, Vector &node_vals);
 
@@ -198,23 +190,14 @@ protected:
    void InterpolateOnDevice(const Vector &field_in_evec, Vector &field_out,
                             const int nel, const int ncomp,
                             const int dof1dsol, const int ordering);
-
 public:
    FindPointsGSLIB();
-   FindPointsGSLIB(Mesh &mesh_in, const double bb_t = 0.1,
-                   const double newt_tol = 1.0e-12,
-                   const int npt_max = 256);
 
 #ifdef MFEM_USE_MPI
    FindPointsGSLIB(MPI_Comm comm_);
-   FindPointsGSLIB(ParMesh &mesh_in, const double bb_t = 0.1,
-                   const double newt_tol = 1.0e-12,
-                   const int npt_max = 256);
 #endif
 
    virtual ~FindPointsGSLIB();
-   FindPointsGSLIB(const FindPointsGSLIB&) = delete;
-   FindPointsGSLIB& operator=(const FindPointsGSLIB&) = delete;
 
    /** Initializes the internal mesh in gslib, by sending the positions of the
        Gauss-Lobatto nodes of the input Mesh object \p m.
@@ -229,8 +212,8 @@ public:
        @param[in] npt_max   (Optional) Number of points for simultaneous
                             iteration. This alters performance and
                             memory footprint.*/
-
-   void Setup(Mesh &m, const double bb_t = 0.1, const double newt_tol = 1.0e-12,
+   void Setup(Mesh &m, const double bb_t = 0.1,
+              const double newt_tol = 1.0e-12,
               const int npt_max = 256);
    /** Searches positions given in physical space by \p point_pos.
        These positions can be ordered byNodes: (XXX...,YYY...,ZZZ) or
@@ -306,12 +289,7 @@ public:
 
    /** Cleans up memory allocated internally by gslib.
        Note that in parallel, this must be called before MPI_Finalize(), as it
-       calls MPI_Comm_free() for internal gslib communicators. FreeData is
-       also called by the class destructor and there are no memory leaks if the
-       destructor is called before MPI_Finalize(). If the destructor is called
-       after MPI_Finalize(), there will be an error because gslib will try to
-       invoke some MPI functions.
-   */
+       calls MPI_Comm_free() for internal gslib communicators. */
    virtual void FreeData();
 
    /// Return code for each point searched by FindPoints: inside element (0), on
@@ -334,9 +312,6 @@ public:
    /// point found by FindPoints.
    virtual const Vector &GetGSLIBReferencePosition() const { return gsl_ref; }
 
-   /// Get array of indices of not-found points.
-   Array<unsigned int> GetPointsNotFoundIndices() const;
-
    /** @name Methods to support a custom interpolation procedure.
        \brief The physical-space point that the user seeks to interpolate at
        could be located inside an element on another mpi rank.

fem/hyperbolic.cpp

@@ -181,7 +181,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    // current elements' the number of degrees of freedom
    // does not consider the number of equations
    const int dof1 = el1.GetDof();
-   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
+   const int dof2 = el2.GetDof();
 
 #ifdef MFEM_THREAD_SAFE
    // Local storage for element integration
@@ -219,9 +219,7 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
    const IntegrationRule *ir = IntRule;
    if (!ir)
    {
-      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
-                                               el2.GetOrder()) : el1.GetOrder();
-      const int order = 2*max_el_order + IntOrderOffset;
+      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
    // loop over integration points
@@ -233,22 +231,18 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
 
       // Calculate basis functions on both elements at the face
       el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, state1);
-
-      if (dof2)
-      {
-         // Calculate basis functions on both elements at the face
-         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-         // Interpolate elfun at the point
-         elfun2_mat.MultTranspose(shape2, state2);
-      }
+      elfun2_mat.MultTranspose(shape2, state2);
 
       // Get the normal vector and the flux on the face
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+         // This assume the 1D integration point is in (0,1). This may not work
+         // if this changes.
+         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
       }
       else
       {
@@ -256,18 +250,14 @@ void HyperbolicFormIntegrator::AssembleFaceVector(
       }
       // Compute F(u+, x) and F(u-, x) with maximum characteristic speed
       // Compute hat(F) using evaluated quantities
-      const real_t speed = (dof2) ? numFlux.Eval(state1, state2, nor, Tr, fluxN):
-                           fluxFunction.ComputeFluxDotN(state1, nor, Tr, fluxN);
+      const real_t speed = numFlux.Eval(state1, state2, nor, Tr, fluxN);
 
       // Update the global max char speed
       max_char_speed = std::max(speed, max_char_speed);
 
       // pre-multiply integration weight to flux
       AddMult_a_VWt(-ip.weight*sign, shape1, fluxN, elvect1_mat);
-      if (dof2)
-      {
-         AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
-      }
+      AddMult_a_VWt(+ip.weight*sign, shape2, fluxN, elvect2_mat);
    }
 }
 
@@ -278,7 +268,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
    // current elements' the number of degrees of freedom
    // does not consider the number of equations
    const int dof1 = el1.GetDof();
-   const int dof2 = (Tr.Elem2No >= 0)?(el2.GetDof()):(0);
+   const int dof2 = el2.GetDof();
 
 #ifdef MFEM_THREAD_SAFE
    // Local storage for element integration
@@ -312,9 +302,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
    const IntegrationRule *ir = IntRule;
    if (!ir)
    {
-      const int max_el_order = dof2 ? std::max(el1.GetOrder(),
-                                               el2.GetOrder()) : el1.GetOrder();
-      const int order = 2*max_el_order + IntOrderOffset;
+      const int order = 2*std::max(el1.GetOrder(), el2.GetOrder()) + IntOrderOffset;
       ir = &IntRules.Get(Tr.GetGeometryType(), order);
    }
    // loop over integration points
@@ -324,25 +312,20 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
 
       Tr.SetAllIntPoints(&ip); // set face and element int. points
 
-      // Calculate basis functions of the first element at the face
+      // Calculate basis functions on both elements at the face
       el1.CalcShape(Tr.GetElement1IntPoint(), shape1);
+      el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
 
       // Interpolate elfun at the point
       elfun1_mat.MultTranspose(shape1, state1);
-
-      if (dof2)
-      {
-         // Calculate basis function of the second element at the face
-         el2.CalcShape(Tr.GetElement2IntPoint(), shape2);
-
-         // Interpolate elfun at the point
-         elfun2_mat.MultTranspose(shape2, state2);
-      }
+      elfun2_mat.MultTranspose(shape2, state2);
 
       // Get the normal vector and the flux on the face
       if (nor.Size() == 1)  // if 1D, use 1 or -1.
       {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
+         // This assume the 1D integration point is in (0,1). This may not work
+         // if this changes.
+         nor(0) = (Tr.GetElement1IntPoint().x - 0.5) * 2.0;
       }
       else
       {
@@ -352,14 +335,7 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
       // Trial side 1
 
       // Compute hat(J) using evaluated quantities
-      if (dof2)
-      {
-         numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
-      }
-      else
-      {
-         fluxFunction.ComputeFluxJacobianDotN(state1, nor, Tr, JDotN);
-      }
+      numFlux.Grad(1, state1, state2, nor, Tr, JDotN);
 
       const int ioff = fluxFunction.num_equations * dof1;
 
@@ -384,325 +360,36 @@ void HyperbolicFormIntegrator::AssembleFaceGrad(
             }
          }
 
-      if (dof2)
-      {
-         // Trial side 2
-
-         // Compute hat(J) using evaluated quantities
-         numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
-
-         const int joff = ioff;
-
-         for (int di = 0; di < fluxFunction.num_equations; di++)
-            for (int dj = 0; dj < fluxFunction.num_equations; dj++)
-            {
-               // pre-multiply integration weight to Jacobian
-               const real_t w = +ip.weight * sign * JDotN(di,dj);
-               for (int j = 0; j < dof2; j++)
-               {
-                  // Test side 1
-                  for (int i = 0; i < dof1; i++)
-                  {
-                     elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
-                  }
-
-                  // Test side 2
-                  for (int i = 0; i < dof2; i++)
-                  {
-                     elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
-                  }
-               }
-            }
-      }
-   }
-}
-
-BdrHyperbolicDirichletIntegrator::BdrHyperbolicDirichletIntegrator(
-   const NumericalFlux &numFlux,
-   VectorCoefficient &bdrState,
-   const int IntOrderOffset,
-   real_t sign)
-   : NonlinearFormIntegrator(),
-     numFlux(numFlux),
-     fluxFunction(numFlux.GetFluxFunction()),
-     u_vcoeff(bdrState),
-     IntOrderOffset(IntOrderOffset),
-     sign(sign),
-     num_equations(fluxFunction.num_equations)
-{
-   MFEM_VERIFY(fluxFunction.num_equations == bdrState.GetVDim(),
-               "Flux function does not match the vector dimension of the coefficient!");
-#ifndef MFEM_THREAD_SAFE
-   state_in.SetSize(num_equations);
-   state_out.SetSize(num_equations);
-   fluxN.SetSize(num_equations);
-   JDotN.SetSize(num_equations);
-   nor.SetSize(fluxFunction.dim);
-#endif
-   ResetMaxCharSpeed();
-}
-
-void BdrHyperbolicDirichletIntegrator::AssembleFaceVector(
-   const FiniteElement &el, const FiniteElement &,
-   FaceElementTransformations &Tr, const Vector &elfun, Vector &elvect)
-{
-   MFEM_ASSERT(Tr.Elem2No < 0, "Not a boundary face!");
-
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // state value at an integration point - interior
-   Vector state_in(num_equations);
-   // state value at an integration point - boundary
-   Vector state_out(num_equations);
-   // hat(F)(u,x)
-   Vector fluxN(num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elvect.SetSize(dof * num_equations);
-   elvect = 0.0;
-
-   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
-
-   DenseMatrix elvect_mat(elvect.GetData(), dof, num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Interpolate elfun at the point
-      elfun_mat.MultTranspose(shape, state_in);
-
-      // Evaluate boundary state at the point
-      u_vcoeff.Eval(state_out, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
-      // Compute F(u+, x) and F(u_b, x) with maximum characteristic speed
-      // Compute hat(F) using evaluated quantities
-      const real_t speed = numFlux.Eval(state_in, state_out, nor, Tr, fluxN);
-
-      // Update the global max char speed
-      max_char_speed = std::max(speed, max_char_speed);
-
-      // pre-multiply integration weight to flux
-      AddMult_a_VWt(-ip.weight*sign, shape, fluxN, elvect_mat);
-   }
-}
-
-void BdrHyperbolicDirichletIntegrator::AssembleFaceGrad(
-   const FiniteElement &el, const FiniteElement &,
-   FaceElementTransformations &Tr, const Vector &elfun, DenseMatrix &elmat)
-{
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // state value at an integration point - interior
-   Vector state_in(num_equations);
-   // state value at an integration point - boundary
-   Vector state_out(num_equations);
-   // hat(J)(u,x)
-   DenseMatrix JDotN(num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elmat.SetSize(dof * num_equations);
-   elmat = 0.0;
-
-   const DenseMatrix elfun_mat(elfun.GetData(), dof, num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int q = 0; q < ir->GetNPoints(); q++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(q);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Interpolate elfun at the point
-      elfun_mat.MultTranspose(shape, state_in);
-
-      // Evaluate boundary state at the point
-      u_vcoeff.Eval(state_out, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
+      // Trial side 2
 
       // Compute hat(J) using evaluated quantities
-      numFlux.Grad(1, state_in, state_out, nor, Tr, JDotN);
+      numFlux.Grad(2, state1, state2, nor, Tr, JDotN);
+
+      const int joff = ioff;
 
       for (int di = 0; di < fluxFunction.num_equations; di++)
          for (int dj = 0; dj < fluxFunction.num_equations; dj++)
          {
             // pre-multiply integration weight to Jacobian
-            const real_t w = -ip.weight * sign * JDotN(di,dj);
-            for (int j = 0; j < dof; j++)
-               for (int i = 0; i < dof; i++)
+            const real_t w = +ip.weight * sign * JDotN(di,dj);
+            for (int j = 0; j < dof2; j++)
+            {
+               // Test side 1
+               for (int i = 0; i < dof1; i++)
                {
-                  elmat(i+dof*di, j+dof*dj) += w * shape(i) * shape(j);
+                  elmat(i+dof1*di, joff+j+dof2*dj) += w * shape1(i) * shape2(j);
                }
+
+               // Test side 2
+               for (int i = 0; i < dof2; i++)
+               {
+                  elmat(ioff+i+dof2*di, joff+j+dof2*dj) -= w * shape2(i) * shape2(j);
+               }
+            }
          }
    }
 }
 
-BoundaryHyperbolicFlowIntegrator::BoundaryHyperbolicFlowIntegrator(
-   const FluxFunction &flux, VectorCoefficient &u, real_t alpha_, real_t beta_,
-   const int IntOrderOffset_)
-   : fluxFunction(flux), u_vcoeff(u), alpha(alpha_), beta(beta_),
-     IntOrderOffset(IntOrderOffset_)
-{
-   MFEM_VERIFY(fluxFunction.num_equations == u_vcoeff.GetVDim(),
-               "Flux function does not match the vector dimension of the coefficient!");
-#ifndef MFEM_THREAD_SAFE
-   state.SetSize(fluxFunction.num_equations);
-   nor.SetSize(fluxFunction.dim);
-   fluxN.SetSize(fluxFunction.num_equations);
-#endif
-   ResetMaxCharSpeed();
-}
-
-void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
-{
-   mfem_error("BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect\n"
-              "  is not implemented as boundary integrator!\n"
-              "  Use LinearForm::AddBdrFaceIntegrator instead of\n"
-              "  LinearForm::AddBoundaryIntegrator.");
-}
-
-void BoundaryHyperbolicFlowIntegrator::AssembleRHSElementVect(
-   const FiniteElement &el, FaceElementTransformations &Tr, Vector &elvect)
-{
-   // current elements' the number of degrees of freedom
-   // does not consider the number of equations
-   const int dof = el.GetDof();
-
-#ifdef MFEM_THREAD_SAFE
-   // Local storage for element integration
-
-   // shape function value at an integration point
-   Vector shape(dof);
-   // state value at an integration point
-   Vector state(fluxFunction.num_equations);
-   // normal vector (usually not a unit vector)
-   Vector nor(Tr.GetSpaceDim());
-   // hat(F)(u,x)
-   Vector fluxN(fluxFunction.num_equations);
-#else
-   shape.SetSize(dof);
-#endif
-
-   elvect.SetSize(dof * fluxFunction.num_equations);
-   elvect = 0.0;
-
-   DenseMatrix elvect_mat(elvect.GetData(), dof, fluxFunction.num_equations);
-
-   // Obtain integration rule. If integration is rule is given, then use it.
-   // Otherwise, get (2*p + IntOrderOffset) order integration rule
-   const IntegrationRule *ir = IntRule;
-   if (!ir)
-   {
-      const int order = 2*el.GetOrder() + IntOrderOffset;
-      ir = &IntRules.Get(Tr.GetGeometryType(), order);
-   }
-   // loop over integration points
-   for (int i = 0; i < ir->GetNPoints(); i++)
-   {
-      const IntegrationPoint &ip = ir->IntPoint(i);
-
-      Tr.SetAllIntPoints(&ip); // set face and element int. points
-
-      // Calculate basis functions on both elements at the face
-      el.CalcShape(Tr.GetElement1IntPoint(), shape);
-
-      // Evaluate the coefficient at the point
-      u_vcoeff.Eval(state, Tr, ip);
-
-      // Get the normal vector and the flux on the face
-      if (nor.Size() == 1)  // if 1D, use 1 or -1.
-      {
-         nor(0) = 2*Tr.GetElement1IntPoint().x - 1.;
-      }
-      else
-      {
-         CalcOrtho(Tr.Jacobian(), nor);
-      }
-      // Compute F(u, x) with maximum characteristic speed
-      const real_t speed = fluxFunction.ComputeFluxDotN(state, nor, Tr, fluxN);
-
-      // Update the global max char speed
-      max_char_speed = std::max(speed, max_char_speed);
-
-      // pre-multiply integration weight to flux
-      const real_t a = 0.5 * alpha  * ip.weight;
-      const real_t b = beta * ip.weight;
-
-      for (int n = 0; n < fluxFunction.num_equations; n++)
-      {
-         fluxN(n) = a * fluxN(n) - b * fabs(fluxN(n));
-      }
-
-      AddMultVWt(shape, fluxN, elvect_mat);
-   }
-}
-
 real_t FluxFunction::ComputeFluxDotN(const Vector &U,
                                      const Vector &normal,
                                      FaceElementTransformations &Tr,

fem/hyperbolic.hpp

@@ -306,14 +306,12 @@ MFEM_DEPRECATED typedef NumericalFlux RiemannSolver;
 class HyperbolicFormIntegrator : public NonlinearFormIntegrator
 {
 private:
+   // The maximum characteristic speed, updated during element/face vector assembly
+   real_t max_char_speed;
    const NumericalFlux &numFlux;   // Numerical flux that maps F(u±,x) to F̂
    const FluxFunction &fluxFunction;
    const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
    const real_t sign;
-
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
-
 #ifndef MFEM_THREAD_SAFE
    // Local storage for element integration
    Vector shape;              // shape function value at an integration point
@@ -333,9 +331,8 @@ private:
 
 public:
    const int num_equations;  // the number of equations
-
    /**
-    * @brief Construct a new HyperbolicFormIntegrator object
+    * @brief Construct a new Hyperbolic Form Integrator object
     *
     * @param[in] numFlux numerical flux
     * @param[in] IntOrderOffset integration order offset
@@ -346,14 +343,21 @@ public:
       const int IntOrderOffset = 0,
       const real_t sign = 1.);
 
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
+   /**
+    * @brief Reset the Max Char Speed 0
+    *
+    */
+   void ResetMaxCharSpeed()
+   {
+      max_char_speed = 0.0;
+   }
 
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
+   real_t GetMaxCharSpeed()
+   {
+      return max_char_speed;
+   }
 
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
+   const FluxFunction &GetFluxFunction() { return fluxFunction; }
 
    /**
     * @brief Implements (F(u), ∇v) with abstract F computed by
@@ -412,182 +416,6 @@ public:
                          const Vector &elfun, DenseMatrix &elmat) override;
 };
 
-/**
- * @brief Abstract boundary hyperbolic form integrator, assembling
- * <F̂(u⁻,u_b,x) n, [v]> term for scalar finite elements at the boundary.
- *
- * This form integrator is coupled with a NumericalFlux that implements the
- * numerical flux F̂ at the boundary faces. The flux F is obtained from the
- * FluxFunction assigned to the aforementioned NumericalFlux with the given
- * boundary coefficient for the state u_b.
- *
- * Note the class can be used for imposing conditions on interior interfaces.
- */
-class BdrHyperbolicDirichletIntegrator : public NonlinearFormIntegrator
-{
-private:
-   const NumericalFlux &numFlux;    // Numerical flux that maps F to F̂
-   const FluxFunction &fluxFunction;
-   VectorCoefficient &u_vcoeff;     // Boundary state vector coefficient
-   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
-   const real_t sign;
-
-   // The maximum characteristic speed, updated during element/face vector assembly
-   real_t max_char_speed;
-
-#ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
-   Vector shape;  // shape function value at an integration point
-   Vector state_in;  // state value at an integration point - interior
-   Vector state_out;  // state value at an integration point - boundary
-   Vector nor;     // normal vector, see mfem::CalcOrtho()
-   Vector fluxN;   // F̂(u⁻,u_b,x) n
-   DenseMatrix JDotN;   // Ĵ(u⁻,u_b,x) n
-#endif
-
-public:
-   const int num_equations;  // the number of equations
-
-   /**
-    * @brief Construct a new BdrHyperbolicDirichletIntegrator object
-    *
-    * @param[in] numFlux numerical flux
-    * @param[in] bdrState boundary state coefficient
-    * @param[in] IntOrderOffset integration order offset
-    * @param[in] sign sign of the convection term
-    */
-   BdrHyperbolicDirichletIntegrator(
-      const NumericalFlux &numFlux,
-      VectorCoefficient &bdrState,
-      const int IntOrderOffset = 0,
-      const real_t sign = 1.);
-
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
-
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
-
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
-
-   /**
-    * @brief Implements <-F̂(u⁻,u_b,x) n, [v]> with abstract F̂ computed by
-    * NumericalFlux::Eval() of the numerical flux object
-    *
-    * @param[in] el1 finite element of the interior element
-    * @param[in] el2 not used
-    * @param[in] Tr face element transformations
-    * @param[in] elfun local coefficient of basis for the interior element
-    * @param[out] elvect evaluated dual vector <-F̂(u⁻,u_b,x) n, [v]>
-    */
-   void AssembleFaceVector(const FiniteElement &el1,
-                           const FiniteElement &el2,
-                           FaceElementTransformations &Tr,
-                           const Vector &elfun, Vector &elvect) override;
-
-   /**
-    * @brief Implements <-Ĵ(u⁻,u_b,x) n, [v]> with abstract Ĵ computed by
-    * NumericalFlux::Grad() of the numerical flux object
-    *
-    * @param[in] el1 finite element of the interior element
-    * @param[in] el2 not used
-    * @param[in] Tr face element transformations
-    * @param[in] elfun local coefficient of basis for the interior element
-    * @param[out] elmat evaluated Jacobian matrix <-Ĵ(u⁻,u_b,x) n, [v]>
-    */
-   void AssembleFaceGrad(const FiniteElement &el1,
-                         const FiniteElement &el2,
-                         FaceElementTransformations &Tr,
-                         const Vector &elfun, DenseMatrix &elmat) override;
-};
-
-/**
- * @brief Abstract boundary hyperbolic linear form integrator, assembling
- * <ɑ/2 F(u,x) n - β |F(u,x) n|, v> terms for scalar finite elements.
- *
- * This form integrator is coupled with a FluxFunction that evaluates the
- * flux F at the boundary.
- *
- * Note the upwinding is performed component-wise. For general boundary
- * integration with a numerical flux, see BdrHyperbolicDirichletIntegrator.
- */
-class BoundaryHyperbolicFlowIntegrator : public LinearFormIntegrator
-{
-   const FluxFunction &fluxFunction;
-   VectorCoefficient &u_vcoeff;
-   const real_t alpha, beta;
-   const int IntOrderOffset; // integration order offset, 2*p + IntOrderOffset.
-
-   // The maximum characteristic speed, updated during face vector assembly
-   real_t max_char_speed;
-
-#ifndef MFEM_THREAD_SAFE
-   // Local storage for element integration
-   Vector shape;   // shape function value at an integration point
-   Vector state;   // state value at an integration point
-   Vector nor;     // normal vector, see mfem::CalcOrtho()
-   Vector fluxN;   // F(u,x) n
-#endif
-
-public:
-   /**
-    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
-    *
-    * @param[in] flux flux function
-    * @param[in] u vector state coefficient
-    * @param[in] alpha ɑ coefficient (β = ɑ/2)
-    * @param[in] IntOrderOffset integration order offset
-    */
-   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
-                                    real_t alpha = -1., int IntOrderOffset = 0)
-      : BoundaryHyperbolicFlowIntegrator(flux, u, alpha, alpha/2., IntOrderOffset) { }
-
-   /**
-    * @brief Construct a new BoundaryHyperbolicFlowIntegrator object
-    *
-    * @param[in] flux flux function
-    * @param[in] u vector state coefficient
-    * @param[in] alpha ɑ coefficient
-    * @param[in] beta β coefficient
-    * @param[in] IntOrderOffset integration order offset
-    */
-   BoundaryHyperbolicFlowIntegrator(const FluxFunction &flux, VectorCoefficient &u,
-                                    real_t alpha, real_t beta, int IntOrderOffset = 0);
-
-   /// Reset the maximum characteristic speed to zero
-   void ResetMaxCharSpeed() { max_char_speed = 0.0; }
-
-   /// Get the maximum characteristic speed
-   real_t GetMaxCharSpeed() const { return max_char_speed; }
-
-   /// Get the associated flux function
-   const FluxFunction &GetFluxFunction() const { return fluxFunction; }
-
-   using LinearFormIntegrator::AssembleRHSElementVect;
-
-   /**
-    * @warning Boundary element integration not implemented, use
-    * AssembleRHSElementVect(const FiniteElement&,
-    * FaceElementTransformations &, Vector &) instead
-    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               ElementTransformation &Tr,
-                               Vector &elvect) override;
-
-   /**
-    * @brief Implements <-F(u,x) n, v> with abstract F computed by
-    * FluxFunction::ComputeFluxDotN() of the flux function object
-    *
-    * @param[in] el finite element
-    * @param[in] Tr face element transformations
-    * @param[out] elvect evaluated dual vector <F(u,x) n, v>
-    */
-   void AssembleRHSElementVect(const FiniteElement &el,
-                               FaceElementTransformations &Tr,
-                               Vector &elvect) override;
-};
-
 
 /**
  * @brief Rusanov flux, also known as local Lax-Friedrichs,

fem/integ/bilininteg_curlcurl_pa.cpp

@@ -15,86 +15,6 @@
 namespace mfem
 {
 
-CurlCurlIntegrator::CurlCurlIntegrator() : Q(nullptr), DQ(nullptr), MQ(nullptr)
-{
-   static Kernels kernels;
-}
-
-CurlCurlIntegrator::CurlCurlIntegrator(Coefficient &q,
-                                       const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(&q), DQ(nullptr), MQ(nullptr)
-{
-   static Kernels kernels;
-}
-
-CurlCurlIntegrator::CurlCurlIntegrator(DiagonalMatrixCoefficient &dq,
-                                       const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(nullptr), DQ(&dq), MQ(nullptr)
-{
-   static Kernels kernels;
-}
-
-CurlCurlIntegrator::CurlCurlIntegrator(MatrixCoefficient &mq,
-                                       const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir), Q(nullptr), DQ(nullptr), MQ(&mq)
-{
-   static Kernels kernels;
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-CurlCurlIntegrator::Kernels::Kernels()
-{
-   CurlCurlIntegrator::AddSpecialization<3, 2, 3>();
-   CurlCurlIntegrator::AddSpecialization<3, 3, 4>();
-   CurlCurlIntegrator::AddSpecialization<3, 4, 5>();
-   CurlCurlIntegrator::AddSpecialization<3, 5, 6>();
-}
-
-CurlCurlIntegrator::ApplyKernelType
-CurlCurlIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 2) { return internal::PACurlCurlApply2D; }
-   else if (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlApply3D;
-      }
-      else
-      {
-         return internal::PACurlCurlApply3D;
-      }
-   }
-   else { MFEM_ABORT(""); }
-}
-
-CurlCurlIntegrator::DiagonalKernelType
-CurlCurlIntegrator::DiagonalPAKernels::Fallback(int DIM, int, int)
-{
-   if (DIM == 2)
-   {
-      return internal::PACurlCurlAssembleDiagonal2D;
-   }
-   else if (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlAssembleDiagonal3D;
-      }
-      else
-      {
-         return internal::PACurlCurlAssembleDiagonal3D;
-      }
-   }
-   else
-   {
-      MFEM_ABORT("");
-   }
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
    // Assumes tensor-product elements
@@ -157,16 +77,129 @@ void CurlCurlIntegrator::AssemblePA(const FiniteElementSpace &fes)
 
 void CurlCurlIntegrator::AssembleDiagonalPA(Vector& diag)
 {
-   DiagonalPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
-                          mapsO->B, mapsC->B, mapsO->G, mapsC->G, pa_data,
-                          diag);
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<2,3>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x34:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<3,4>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x45:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<4,5>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            case 0x56:
+               return internal::SmemPACurlCurlAssembleDiagonal3D<5,6>(
+                         dofs1D,
+                         quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+            default:
+               return internal::SmemPACurlCurlAssembleDiagonal3D(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B,
+                         mapsO->G, mapsC->G,
+                         pa_data, diag);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlAssembleDiagonal3D(dofs1D, quad1D, symmetric, ne,
+                                                mapsO->B, mapsC->B,
+                                                mapsO->G, mapsC->G,
+                                                pa_data, diag);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlAssembleDiagonal2D(dofs1D, quad1D, ne,
+                                             mapsO->B, mapsC->G, pa_data, diag);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
 }
 
 void CurlCurlIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
-                       mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt, mapsC->G,
-                       mapsC->Gt, pa_data, x, y, false);
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlApply3D<2,3>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x34:
+               return internal::SmemPACurlCurlApply3D<3,4>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x45:
+               return internal::SmemPACurlCurlApply3D<4,5>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            case 0x56:
+               return internal::SmemPACurlCurlApply3D<5,6>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+            default:
+               return internal::SmemPACurlCurlApply3D(
+                         dofs1D, quad1D, symmetric, ne,
+                         mapsO->B, mapsC->B, mapsO->Bt, mapsC->Bt,
+                         mapsC->G, mapsC->Gt, pa_data, x, y);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlApply3D(dofs1D, quad1D, symmetric, ne, mapsO->B, mapsC->B,
+                                     mapsO->Bt, mapsC->Bt, mapsC->G, mapsC->Gt,
+                                     pa_data, x, y);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, mapsO->B, mapsO->Bt,
+                                  mapsC->G, mapsC->Gt, pa_data, x, y);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
 }
 
 void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
@@ -176,9 +209,61 @@ void CurlCurlIntegrator::AddAbsMultPA(const Vector &x, Vector &y) const
    auto absO = mapsO->Abs();
    auto absC = mapsC->Abs();
 
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, dofs1D, quad1D, symmetric, ne,
-                       absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt,
-                       abs_pa_data, x, y, true);
+   if (dim == 3)
+   {
+      if (Device::Allows(Backend::DEVICE_MASK))
+      {
+         const int ID = (dofs1D << 4) | quad1D;
+         switch (ID)
+         {
+            case 0x23:
+               return internal::SmemPACurlCurlApply3D<2,3>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            case 0x34:
+               return internal::SmemPACurlCurlApply3D<3,4>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            case 0x45:
+               return internal::SmemPACurlCurlApply3D<4,5>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            case 0x56:
+               return internal::SmemPACurlCurlApply3D<5,6>(
+                         dofs1D, quad1D,
+                         symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+            default:
+               return internal::SmemPACurlCurlApply3D<0,0>(
+                         dofs1D, quad1D, symmetric, ne,
+                         absO.B, absC.B, absO.Bt, absC.Bt,
+                         absC.G, absC.Gt, abs_pa_data, x, y, true);
+         }
+      }
+      else
+      {
+         internal::PACurlCurlApply3D<0,0>(
+            dofs1D, quad1D, symmetric, ne,
+            absO.B, absC.B, absO.Bt, absC.Bt, absC.G, absC.Gt,
+            abs_pa_data, x, y, true);
+      }
+   }
+   else if (dim == 2)
+   {
+      internal::PACurlCurlApply2D(dofs1D, quad1D, ne, absO.B, absO.Bt,
+                                  absC.G, absC.Gt, abs_pa_data, x, y, true);
+   }
+   else
+   {
+      MFEM_ABORT("Unsupported dimension!");
+   }
 }
 
 } // namespace mfem

fem/integ/bilininteg_dgdiffusion_kernels.hpp

@@ -1,500 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP
-#define MFEM_BILININTEG_DGDIFFUSION_KERNELS_HPP
-
-#include "../../general/forall.hpp"
-#include "../../mesh/face_nbr_geom.hpp"
-#include "../fe/face_map_utils.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGDiffusionApply2D(const int NF, const Array<real_t> &b,
-                                 const Array<real_t> &bt,
-                                 const Array<real_t> &g,
-                                 const Array<real_t> &gt, const real_t sigma,
-                                 const Vector &pa_data, const Vector &x_,
-                                 const Vector &dxdn_, Vector &y_, Vector &dydn_,
-                                 const int d1d = 0, const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B_ = Reshape(b.Read(), Q1D, D1D);
-   auto G_ = Reshape(g.Read(), Q1D, D1D);
-
-   auto pa =
-      Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11)
-
-   auto x = Reshape(x_.Read(), D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, 2, NF);
-   auto dxdn = Reshape(dxdn_.Read(), D1D, 2, NF);
-   auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF);
-
-   const int NBX = std::max(D1D, Q1D);
-
-   mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void
-   {
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t u0[max_D1D];
-      MFEM_SHARED real_t u1[max_D1D];
-      MFEM_SHARED real_t du0[max_D1D];
-      MFEM_SHARED real_t du1[max_D1D];
-
-      MFEM_SHARED real_t Bu0[max_Q1D];
-      MFEM_SHARED real_t Bu1[max_Q1D];
-      MFEM_SHARED real_t Bdu0[max_Q1D];
-      MFEM_SHARED real_t Bdu1[max_Q1D];
-
-      MFEM_SHARED real_t r[max_Q1D];
-
-      MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D];
-      DeviceMatrix B(BG, Q1D, D1D);
-      DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D);
-
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(p, x, Q1D)
-         {
-            for (int d = 0; d < D1D; ++d)
-            {
-               B(p, d) = B_(p, d);
-               G(p, d) = G_(p, d);
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // copy edge values to u0, u1 and copy edge normals to du0, du1
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d, x, D1D)
-         {
-            u[d] = x(d, side, f);
-            du[d] = dxdn(d, side, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // eval @ quad points
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         real_t *Bu = (side == 0) ? Bu0 : Bu1;
-         real_t *Bdu = (side == 0) ? Bdu0 : Bdu1;
-
-         MFEM_FOREACH_THREAD(p, x, Q1D)
-         {
-            const real_t Je_side[] = {pa(2 + 2 * side, p, f),
-                                      pa(2 + 2 * side + 1, p, f)
-                                     };
-
-            Bu[p] = 0.0;
-            Bdu[p] = 0.0;
-
-            for (int d = 0; d < D1D; ++d)
-            {
-               const real_t b = B(p, d);
-               const real_t g = G(p, d);
-
-               Bu[p] += b * u[d];
-               Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d];
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // term - < {Q du/dn}, [v] > +  kappa * < {Q/h} [u], [v] >:
-      if (MFEM_THREAD_ID(y) == 0)
-      {
-         MFEM_FOREACH_THREAD(p, x, Q1D)
-         {
-            const real_t q = pa(0, p, f);
-            const real_t hi = pa(1, p, f);
-            const real_t jump = Bu0[p] - Bu1[p];
-            const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J)
-            r[p] = -avg + hi * q * jump;
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(d, x, D1D)
-      {
-         real_t Br = 0.0;
-
-         for (int p = 0; p < Q1D; ++p)
-         {
-            Br += B(p, d) * r[p];
-         }
-
-         u0[d] = Br; // overwrite u0, u1
-         u1[d] = -Br;
-      } // for d
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d, x, D1D) { du[d] = 0.0; }
-      }
-      MFEM_SYNC_THREAD;
-
-      // term sigma * < [u], {Q dv/dn} >
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *const du = (side == 0) ? du0 : du1;
-         real_t *const u = (side == 0) ? u0 : u1;
-
-         MFEM_FOREACH_THREAD(d, x, D1D)
-         {
-            for (int p = 0; p < Q1D; ++p)
-            {
-               const real_t Je[] = {pa(2 + 2 * side, p, f),
-                                    pa(2 + 2 * side + 1, p, f)
-                                   };
-               const real_t jump = Bu0[p] - Bu1[p];
-               const real_t r_p = Je[0] * jump; // normal
-               const real_t w_p = Je[1] * jump; // tangential
-               du[d] += sigma * B(p, d) * r_p;
-               u[d] += sigma * G(p, d) * w_p;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, y, 2)
-      {
-         real_t *u = (side == 0) ? u0 : u1;
-         real_t *du = (side == 0) ? du0 : du1;
-         MFEM_FOREACH_THREAD(d, x, D1D)
-         {
-            y(d, side, f) += u[d];
-            dydn(d, side, f) += du[d];
-         }
-      }
-   }); // mfem::forall
-}
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGDiffusionApply3D(const int NF, const Array<real_t> &b,
-                                 const Array<real_t> &bt,
-                                 const Array<real_t> &g,
-                                 const Array<real_t> &gt, const real_t sigma,
-                                 const Vector &pa_data, const Vector &x_,
-                                 const Vector &dxdn_, Vector &y_, Vector &dydn_,
-                                 const int d1d = 0, const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-
-   auto B_ = Reshape(b.Read(), Q1D, D1D);
-   auto G_ = Reshape(g.Read(), Q1D, D1D);
-
-   // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h)
-   auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF);
-
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
-   auto dxdn = Reshape(dxdn_.Read(), D1D, D1D, 2, NF);
-   auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF);
-
-   const int NBX = std::max(D1D, Q1D);
-
-   mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE(int f) -> void
-   {
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      MFEM_SHARED real_t u0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t u1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t du0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t du1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D];
-      MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D];
-
-      MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3];
-      MFEM_SHARED real_t BG[2 * max_D1D * max_Q1D];
-
-      // some buffers are reused multiple times, but for clarity have new names:
-      real_t(*Bj0)[max_Q1D] = Bu0;
-      real_t(*Bj1)[max_Q1D] = Bu1;
-      real_t(*Bjn0)[max_Q1D] = Bdu0;
-      real_t(*Bjn1)[max_Q1D] = Bdu1;
-      real_t(*Gj0)[max_Q1D] = Gu0;
-      real_t(*Gj1)[max_Q1D] = Gu1;
-
-      DeviceMatrix B(BG, Q1D, D1D);
-      DeviceMatrix G(BG + D1D * Q1D, Q1D, D1D);
-
-      // copy face values to u0, u1 and copy normals to du0, du1
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               u[d2][d1] = x(d1, d2, side,
-                             f); // copy transposed for better memory access
-               du[d2][d1] = dxdn(d1, d2, side, f);
-            }
-         }
-
-         MFEM_FOREACH_THREAD(p1, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(p2, y, Q1D)
-            {
-               for (int l = 0; l < 3; ++l)
-               {
-                  nJe[side][p2][p1][l] = pa(3 * side + l, p1, p2, f);
-               }
-
-               if (side == 0)
-               {
-                  kappa_Qh[p2][p1] = pa(6, p1, p2, f);
-               }
-            }
-         }
-
-         if (side == 0)
-         {
-            MFEM_FOREACH_THREAD(p, x, Q1D)
-            {
-               MFEM_FOREACH_THREAD(d, y, D1D)
-               {
-                  B(p, d) = B_(p, d);
-                  G(p, d) = G_(p, d);
-               }
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // eval u and normal derivative @ quad points
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
-         real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
-         real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
-
-         MFEM_FOREACH_THREAD(p1, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(d2, y, D1D)
-            {
-               real_t bu = 0.0;
-               real_t bdu = 0.0;
-               real_t gu = 0.0;
-
-               for (int d1 = 0; d1 < D1D; ++d1)
-               {
-                  const real_t b = B(p1, d1);
-                  const real_t g = G(p1, d1);
-
-                  bu += b * u[d2][d1];
-                  bdu += b * du[d2][d1];
-                  gu += g * u[d2][d1];
-               }
-
-               Bu[p1][d2] = bu;
-               Bdu[p1][d2] = bdu;
-               Gu[p1][d2] = gu;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t(*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
-         real_t(*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
-         real_t(*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
-
-         MFEM_FOREACH_THREAD(p2, x, Q1D)
-         {
-            MFEM_FOREACH_THREAD(p1, y, Q1D)
-            {
-               const real_t *Je = nJe[side][p2][p1];
-
-               real_t bbu = 0.0;
-               real_t bgu = 0.0;
-               real_t gbu = 0.0;
-               real_t bbdu = 0.0;
-
-               for (int d2 = 0; d2 < D1D; ++d2)
-               {
-                  const real_t b = B(p2, d2);
-                  const real_t g = G(p2, d2);
-                  bbu += b * Bu[p1][d2];
-                  gbu += g * Bu[p1][d2];
-                  bgu += b * Gu[p1][d2];
-                  bbdu += b * Bdu[p1][d2];
-               }
-
-               u[p2][p1] = bbu;
-               // du <- Q du/dn * w * det(J)
-               du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
-         real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
-         real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
-
-         MFEM_FOREACH_THREAD(d1, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(p2, y, Q1D)
-            {
-               real_t bj = 0.0;
-               real_t bjn = 0.0;
-               real_t gj = 0.0;
-               real_t br = 0.0;
-
-               for (int p1 = 0; p1 < Q1D; ++p1)
-               {
-                  const real_t b = B(p1, d1);
-                  const real_t g = G(p1, d1);
-
-                  const real_t *Je = nJe[side][p2][p1];
-
-                  const real_t jump = u0[p2][p1] - u1[p2][p1];
-                  const real_t avg = du0[p2][p1] + du1[p2][p1];
-
-                  // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >
-                  const real_t r = -avg + kappa_Qh[p2][p1] * jump;
-
-                  // bj, gj, bjn contribute to sigma term
-                  bj += b * Je[0] * jump;
-                  gj += g * Je[1] * jump;
-                  bjn += b * Je[2] * jump;
-
-                  br += b * r;
-               }
-
-               Bj[d1][p2] = sigma * bj;
-               Bjn[d1][p2] = sigma * bjn;
-
-               // group br and gj together since we will multiply them both by B
-               // and then sum
-               const real_t sgn = (side == 0) ? 1.0 : -1.0;
-               Gj[d1][p2] = sgn * br + sigma * gj;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-         real_t(*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
-         real_t(*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
-         real_t(*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               real_t bbj = 0.0;
-               real_t gbj = 0.0;
-               real_t bgj = 0.0;
-
-               for (int p2 = 0; p2 < Q1D; ++p2)
-               {
-                  const real_t b = B(p2, d2);
-                  const real_t g = G(p2, d2);
-
-                  bbj += b * Bj[d1][p2];
-                  bgj += b * Gj[d1][p2];
-                  gbj += g * Bjn[d1][p2];
-               }
-
-               du[d2][d1] = bbj;
-               u[d2][d1] = bgj + gbj;
-            }
-         }
-      }
-      MFEM_SYNC_THREAD;
-
-      // map back to y and dydn
-      MFEM_FOREACH_THREAD(side, z, 2)
-      {
-         const real_t(*u)[max_Q1D] = (side == 0) ? u0 : u1;
-         const real_t(*du)[max_Q1D] = (side == 0) ? du0 : du1;
-
-         MFEM_FOREACH_THREAD(d2, x, D1D)
-         {
-            MFEM_FOREACH_THREAD(d1, y, D1D)
-            {
-               y(d1, d2, side, f) += u[d2][d1];
-               dydn(d1, d2, side, f) += du[d2][d1];
-            }
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template <int DIM, int D1D, int Q1D>
-DGDiffusionIntegrator::ApplyKernelType
-DGDiffusionIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PADGDiffusionApply2D<D1D, Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::PADGDiffusionApply3D<D1D, Q1D>;
-   }
-   MFEM_ABORT("");
-}
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT
-#endif

fem/integ/bilininteg_dgdiffusion_pa.cpp

@@ -11,39 +11,42 @@
 
 #include "../../general/forall.hpp"
 #include "../../mesh/face_nbr_geom.hpp"
-#include "../fe/face_map_utils.hpp"
 #include "../gridfunc.hpp"
 #include "../qfunction.hpp"
+#include "../fe/face_map_utils.hpp"
 
-#include "bilininteg_dgdiffusion_kernels.hpp"
+using namespace std;
 
 namespace mfem
 {
 
-static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
+static void PADGDiffusionSetup2D(const int Q1D,
+                                 const int NE,
+                                 const int NF,
                                  const Array<real_t> &w,
                                  const GeometricFactors &el_geom,
                                  const FaceGeometricFactors &face_geom,
                                  const FaceNeighborGeometricFactors *nbr_geom,
-                                 const Vector &q, const real_t sigma,
-                                 const real_t kappa, Vector &pa_data,
+                                 const Vector &q,
+                                 const real_t sigma,
+                                 const real_t kappa,
+                                 Vector &pa_data,
                                  const Array<int> &face_info_)
 {
    const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, 2, 2, NE);
    const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, NE);
 
    const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0;
-   const auto J_shared =
-      Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D, Q1D, 2, 2, n_nbr);
-   const auto detJ_shared =
-      Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, n_nbr);
+   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr,
+                                 Q1D, Q1D, 2, 2, n_nbr);
+   const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr,
+                                    Q1D, Q1D, n_nbr);
 
    const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, NF);
    const auto n = Reshape(face_geom.normal.Read(), Q1D, 2, NF);
 
    const bool const_q = (q.Size() == 1);
-   const auto Q =
-      const_q ? Reshape(q.Read(), 1, 1) : Reshape(q.Read(), Q1D, NF);
+   const auto Q = const_q ? Reshape(q.Read(), 1,1) : Reshape(q.Read(), Q1D,NF);
 
    const auto W = w.Read();
 
@@ -53,7 +56,7 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
    // (q, 1/h, J0_0, J0_1, J1_0, J1_1)
    auto pa = Reshape(pa_data.Write(), 6, Q1D, NF);
 
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f) -> void
+   mfem::forall(NF, [=] MFEM_HOST_DEVICE (int f) -> void
    {
       const int normal_dir[] = {face_info(0, f), face_info(1, f)};
       const int fid[] = {face_info(4, f), face_info(5, f)};
@@ -71,7 +74,7 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
 
       for (int p = 0; p < Q1D; ++p)
       {
-         const real_t Qp = const_q ? Q(0, 0) : Q(p, f);
+         const real_t Qp = const_q ? Q(0,0) : Q(p, f);
          pa(0, p, f) = kappa * Qp * W[p] * detJf(p, f);
 
          real_t hi = 0.0;
@@ -82,19 +85,17 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
 
             // Always opposite direction in "native" ordering
             // Need to multiply the native=>lex0 with native=>lex1 and negate
-            const int sgn = (side == 1) ? -1 * sgn0 * sgn1 : 1;
+            const int sgn = (side == 1) ? -1*sgn0*sgn1 : 1;
 
             const int e = el[side];
             const auto &J = (side == 1 && shared) ? J_shared : J_loc;
             const auto &detJ = (side == 1 && shared) ? detJ_shared : detJe_loc;
 
             real_t nJi[2];
-            nJi[0] =
-               n(p, 0, f) * J(i, j, 1, 1, e) - n(p, 1, f) * J(i, j, 0, 1, e);
-            nJi[1] =
-               -n(p, 0, f) * J(i, j, 1, 0, e) + n(p, 1, f) * J(i, j, 0, 0, e);
+            nJi[0] = n(p,0,f)*J(i,j, 1,1, e) - n(p,1,f)*J(i,j,0,1,e);
+            nJi[1] = -n(p,0,f)*J(i,j,1,0, e) + n(p,1,f)*J(i,j,0,0,e);
 
-            const real_t dJe = detJ(i, j, e);
+            const real_t dJe = detJ(i,j,e);
             const real_t dJf = detJf(p, f);
 
             const real_t w = factor * Qp * W[p] * dJf / dJe;
@@ -103,9 +104,9 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
             const int ti = 1 - ni;
 
             // Normal
-            pa(2 + 2 * side + 0, p, f) = w * nJi[ni];
+            pa(2 + 2*side + 0, p, f) = w * nJi[ni];
             // Tangential
-            pa(2 + 2 * side + 1, p, f) = sgn * w * nJi[ti];
+            pa(2 + 2*side + 1, p, f) = sgn * w * nJi[ti];
 
             hi += factor * dJf / dJe;
          }
@@ -121,43 +122,47 @@ static void PADGDiffusionSetup2D(const int Q1D, const int NE, const int NF,
    });
 }
 
-static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
+static void PADGDiffusionSetup3D(const int Q1D,
+                                 const int NE,
+                                 const int NF,
                                  const Array<real_t> &w,
                                  const GeometricFactors &el_geom,
                                  const FaceGeometricFactors &face_geom,
                                  const FaceNeighborGeometricFactors *nbr_geom,
-                                 const Vector &q, const real_t sigma,
-                                 const real_t kappa, Vector &pa_data,
+                                 const Vector &q,
+                                 const real_t sigma,
+                                 const real_t kappa,
+                                 Vector &pa_data,
                                  const Array<int> &face_info_)
 {
    const auto J_loc = Reshape(el_geom.J.Read(), Q1D, Q1D, Q1D, 3, 3, NE);
    const auto detJe_loc = Reshape(el_geom.detJ.Read(), Q1D, Q1D, Q1D, NE);
 
    const int n_nbr = nbr_geom ? nbr_geom->num_neighbor_elems : 0;
-   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr, Q1D,
-                                 Q1D, Q1D, 3, 3, n_nbr);
-   const auto detJ_shared =
-      Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr, Q1D, Q1D, Q1D, n_nbr);
+   const auto J_shared = Reshape(nbr_geom ? nbr_geom->J.Read() : nullptr,
+                                 Q1D, Q1D, Q1D, 3, 3, n_nbr);
+   const auto detJ_shared = Reshape(nbr_geom ? nbr_geom->detJ.Read() : nullptr,
+                                    Q1D, Q1D, Q1D, n_nbr);
 
    const auto detJf = Reshape(face_geom.detJ.Read(), Q1D, Q1D, NF);
    const auto n = Reshape(face_geom.normal.Read(), Q1D, Q1D, 3, NF);
 
    const bool const_q = (q.Size() == 1);
-   const auto Q =
-      const_q ? Reshape(q.Read(), 1, 1, 1) : Reshape(q.Read(), Q1D, Q1D, NF);
+   const auto Q = const_q ? Reshape(q.Read(), 1, 1, 1)
+                  : Reshape(q.Read(), Q1D, Q1D, NF);
 
    const auto W = Reshape(w.Read(), Q1D, Q1D);
 
    // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation)
    const auto face_info = Reshape(face_info_.Read(), 6, 2, NF);
-   constexpr int _el_ = 3;  // offset in face_info for element index
+   constexpr int _el_ = 3; // offset in face_info for element index
    constexpr int _fid_ = 4; // offset in face_info for local face id
-   constexpr int _or_ = 5;  // offset in face_info for orientation
+   constexpr int _or_ = 5; // offset in face_info for orientation
 
    // (J00, J01, J02, J10, J11, J12, q/h)
    const auto pa = Reshape(pa_data.Write(), 7, Q1D, Q1D, NF);
 
-   mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE(int f) -> void
+   mfem::forall_2D(NF, Q1D, Q1D, [=] MFEM_HOST_DEVICE (int f) -> void
    {
       MFEM_SHARED int perm[2][3];
       MFEM_SHARED int el[2];
@@ -167,7 +172,10 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
 
       MFEM_FOREACH_THREAD(side, x, 2)
       {
-         MFEM_FOREACH_THREAD(i, y, 3) { perm[side][i] = face_info(i, side, f); }
+         MFEM_FOREACH_THREAD(i, y, 3)
+         {
+            perm[side][i] = face_info(i, side, f);
+         }
 
          if (MFEM_THREAD_ID(y) == 0)
          {
@@ -192,16 +200,16 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
       {
          MFEM_FOREACH_THREAD(p2, y, Q1D)
          {
-            const real_t Qp = const_q ? Q(0, 0, 0) : Q(p1, p2, f);
-            const real_t dJf = detJf(p1, p2, f);
+            const real_t Qp = const_q ? Q(0,0,0) : Q(p1, p2, f);
+            const real_t dJf = detJf(p1,p2,f);
 
             real_t hi = 0.0;
 
             for (int side = 0; side < nsides; ++side)
             {
                int i, j, k;
-               internal::FaceIdxToVolIdx3D(p1 + Q1D * p2, Q1D, fid[0], fid[1],
-                                           side, ortn[1], i, j, k);
+               internal::FaceIdxToVolIdx3D(
+                  p1 + Q1D*p2, Q1D, fid[0], fid[1], side, ortn[1], i, j, k);
 
                const int e = el[side];
                const auto &J = shared[side] ? J_shared : J_loc;
@@ -209,45 +217,27 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
 
                // *INDENT-OFF*
                real_t nJi[3];
-               nJi[0] = (-J(i, j, k, 1, 2, e) * J(i, j, k, 2, 1, e) +
-                         J(i, j, k, 1, 1, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 0, f) +
-                        (J(i, j, k, 0, 2, e) * J(i, j, k, 2, 1, e) -
-                         J(i, j, k, 0, 1, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 1, f) +
-                        (-J(i, j, k, 0, 2, e) * J(i, j, k, 1, 1, e) +
-                         J(i, j, k, 0, 1, e) * J(i, j, k, 1, 2, e)) *
-                           n(p1, p2, 2, f);
+               nJi[0] = (  -J(i,j,k, 1,2, e)*J(i,j,k, 2,1, e) + J(i,j,k, 1,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f)
+                        + ( J(i,j,k, 0,2, e)*J(i,j,k, 2,1, e) - J(i,j,k, 0,1, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f)
+                        + (-J(i,j,k, 0,2, e)*J(i,j,k, 1,1, e) + J(i,j,k, 0,1, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f);
 
-               nJi[1] = (J(i, j, k, 1, 2, e) * J(i, j, k, 2, 0, e) -
-                         J(i, j, k, 1, 0, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 0, f) +
-                        (-J(i, j, k, 0, 2, e) * J(i, j, k, 2, 0, e) +
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 2, 2, e)) *
-                           n(p1, p2, 1, f) +
-                        (J(i, j, k, 0, 2, e) * J(i, j, k, 1, 0, e) -
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 1, 2, e)) *
-                           n(p1, p2, 2, f);
+               nJi[1] = (   J(i,j,k, 1,2, e)*J(i,j,k, 2,0, e) - J(i,j,k, 1,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 0, f)
+                        + (-J(i,j,k, 0,2, e)*J(i,j,k, 2,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 2,2, e)) * n(p1,p2, 1, f)
+                        + ( J(i,j,k, 0,2, e)*J(i,j,k, 1,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 1,2, e)) * n(p1,p2, 2, f);
 
-               nJi[2] = (-J(i, j, k, 1, 1, e) * J(i, j, k, 2, 0, e) +
-                         J(i, j, k, 1, 0, e) * J(i, j, k, 2, 1, e)) *
-                           n(p1, p2, 0, f) +
-                        (J(i, j, k, 0, 1, e) * J(i, j, k, 2, 0, e) -
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 2, 1, e)) *
-                           n(p1, p2, 1, f) +
-                        (-J(i, j, k, 0, 1, e) * J(i, j, k, 1, 0, e) +
-                         J(i, j, k, 0, 0, e) * J(i, j, k, 1, 1, e)) *
-                           n(p1, p2, 2, f);
+               nJi[2] = (  -J(i,j,k, 1,1, e)*J(i,j,k, 2,0, e) + J(i,j,k, 1,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 0, f)
+                        + ( J(i,j,k, 0,1, e)*J(i,j,k, 2,0, e) - J(i,j,k, 0,0, e)*J(i,j,k, 2,1, e)) * n(p1,p2, 1, f)
+                        + (-J(i,j,k, 0,1, e)*J(i,j,k, 1,0, e) + J(i,j,k, 0,0, e)*J(i,j,k, 1,1, e)) * n(p1,p2, 2, f);
                // *INDENT-ON*
 
-               const real_t dJe = detJe(i, j, k, e);
+               const real_t dJe = detJe(i,j,k,e);
                const real_t val = factor * Qp * W(p1, p2) * dJf / dJe;
 
                for (int d = 0; d < 3; ++d)
                {
                   const int idx = std::abs(perm[side][d]) - 1;
                   const int sgn = (perm[side][d] < 0) ? -1 : 1;
-                  pa(3 * side + d, p1, p2, f) = sgn * val * nJi[idx];
+                  pa(3*side + d, p1, p2, f) = sgn * val * nJi[idx];
                }
 
                hi += factor * dJf / dJe;
@@ -267,8 +257,7 @@ static void PADGDiffusionSetup3D(const int Q1D, const int NE, const int NF,
 }
 
 static void PADGDiffusionSetupFaceInfo2D(const int nf, const Mesh &mesh,
-                                         const FaceType type,
-                                         Array<int> &face_info_)
+                                         const FaceType type, Array<int> &face_info_)
 {
    const int ne = mesh.GetNE();
 
@@ -337,7 +326,8 @@ inline void FaceNormalPermutation(int perm[3], const int face_id)
 
 // Assigns to perm the permutation as in FaceNormalPermutation for the second
 // element on the face but signed to indicate the sign of the normal derivative.
-inline void SignedFaceNormalPermutation(int perm[3], const int face_id1,
+inline void SignedFaceNormalPermutation(int perm[3],
+                                        const int face_id1,
                                         const int face_id2,
                                         const int orientation)
 {
@@ -396,19 +386,17 @@ inline void SignedFaceNormalPermutation(int perm[3], const int face_id1,
 }
 
 static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
-                                         const FaceType type,
-                                         Array<int> &face_info_)
+                                         const FaceType type, Array<int> &face_info_)
 {
    const int ne = mesh.GetNE();
 
    int fidx = 0;
-   // face_info array has 12 entries per face, 6 for each of the adjacent
-   // elements: (perm[0], perm[1], perm[2], element_index, local_face_id,
-   // orientation)
+   // face_info array has 12 entries per face, 6 for each of the adjacent elements:
+   // (perm[0], perm[1], perm[2], element_index, local_face_id, orientation)
    face_info_.SetSize(nf * 12);
-   constexpr int _e_ = 3;   // offset for element index
+   constexpr int _e_ = 3; // offset for element index
    constexpr int _fid_ = 4; // offset for local face id
-   constexpr int _or_ = 5;  // offset for orientation
+   constexpr int _or_ = 5; // offset for orientation
 
    auto face_info = Reshape(face_info_.HostWrite(), 6, 2, nf);
    for (int f = 0; f < mesh.GetNumFaces(); ++f)
@@ -420,9 +408,9 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
          const int fid0 = f_info.element[0].local_face_id;
          const int or0 = f_info.element[0].orientation;
 
-         face_info(_e_, 0, fidx) = f_info.element[0].index;
+         face_info(  _e_, 0, fidx) = f_info.element[0].index;
          face_info(_fid_, 0, fidx) = fid0;
-         face_info(_or_, 0, fidx) = or0;
+         face_info( _or_, 0, fidx) = or0;
 
          FaceNormalPermutation(&face_info(0, 0, fidx), fid0);
 
@@ -433,17 +421,16 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
 
             if (f_info.IsShared())
             {
-               face_info(_e_, 1, fidx) = ne + f_info.element[1].index;
+               face_info(  _e_, 1, fidx) = ne + f_info.element[1].index;
             }
             else
             {
-               face_info(_e_, 1, fidx) = f_info.element[1].index;
+               face_info(  _e_, 1, fidx) = f_info.element[1].index;
             }
             face_info(_fid_, 1, fidx) = fid1;
-            face_info(_or_, 1, fidx) = or1;
+            face_info( _or_, 1, fidx) = or1;
 
-            SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1,
-                                        or1);
+            SignedFaceNormalPermutation(&face_info(0, 1, fidx), fid0, fid1, or1);
          }
          else
          {
@@ -461,8 +448,8 @@ static void PADGDiffusionSetupFaceInfo3D(const int nf, const Mesh &mesh,
 void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
                                     FaceType type)
 {
-   const MemoryType mt =
-      (pa_mt == MemoryType::DEFAULT) ? Device::GetDeviceMemoryType() : pa_mt;
+   const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
+                         Device::GetDeviceMemoryType() : pa_mt;
 
    const int ne = fes.GetNE();
    nf = fes.GetNFbyType(type);
@@ -471,17 +458,16 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    Mesh &mesh = *fes.GetMesh();
    const Geometry::Type face_geom_type = mesh.GetTypicalFaceGeometry();
    const FiniteElement &el = *fes.GetTypicalTraceElement();
-   const int ir_order = IntRule
-                        ? IntRule->GetOrder()
+   const int ir_order = IntRule ? IntRule->GetOrder()
                         : GetRule(el.GetOrder(), face_geom_type).GetOrder();
    const IntegrationRule &ir = irs.Get(face_geom_type, ir_order);
    dim = mesh.Dimension();
-   const int q1d = (ir.GetOrder() + 3) / 2;
-   MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0 / (dim - 1)), "");
+   const int q1d = (ir.GetOrder() + 3)/2;
+   MFEM_ASSERT(q1d == pow(real_t(ir.Size()), 1.0/(dim - 1)), "");
 
    const auto vol_ir = irs.Get(mesh.GetTypicalElementGeometry(), ir_order);
-   const auto geom_flags =
-      GeometricFactors::JACOBIANS | GeometricFactors::DETERMINANTS;
+   const auto geom_flags = GeometricFactors::JACOBIANS |
+                           GeometricFactors::DETERMINANTS;
    const auto el_geom = mesh.GetGeometricFactors(vol_ir, geom_flags, mt);
 
    std::unique_ptr<FaceNeighborGeometricFactors> nbr_geom;
@@ -490,8 +476,8 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
       nbr_geom.reset(new FaceNeighborGeometricFactors(*el_geom));
    }
 
-   const auto face_geom_flags =
-      FaceGeometricFactors::DETERMINANTS | FaceGeometricFactors::NORMALS;
+   const auto face_geom_flags = FaceGeometricFactors::DETERMINANTS |
+                                FaceGeometricFactors::NORMALS;
    auto face_geom = mesh.GetFaceGeometricFactors(ir, face_geom_flags, type, mt);
    maps = &el.GetDofToQuad(ir, DofToQuad::TENSOR);
    dofs1D = maps->ndof;
@@ -503,18 +489,9 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    // Evaluate the coefficient at the face quadrature points.
    FaceQuadratureSpace fqs(mesh, ir, type);
    CoefficientVector q(fqs, CoefficientStorage::COMPRESSED);
-   if (Q)
-   {
-      q.Project(*Q);
-   }
-   else if (MQ)
-   {
-      MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */
-   }
-   else
-   {
-      q.SetConstant(1.0);
-   }
+   if (Q) { q.Project(*Q); }
+   else if (MQ) { MFEM_ABORT("Not yet implemented"); /* q.Project(*MQ); */ }
+   else { q.SetConstant(1.0); }
 
    Array<int> face_info;
    if (dim == 1)
@@ -524,16 +501,14 @@ void DGDiffusionIntegrator::SetupPA(const FiniteElementSpace &fes,
    else if (dim == 2)
    {
       PADGDiffusionSetupFaceInfo2D(nf, mesh, type, face_info);
-      PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom,
-                           *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data,
-                           face_info);
+      PADGDiffusionSetup2D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom,
+                           nbr_geom.get(), q, sigma, kappa, pa_data, face_info);
    }
    else if (dim == 3)
    {
       PADGDiffusionSetupFaceInfo3D(nf, mesh, type, face_info);
-      PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom,
-                           *face_geom, nbr_geom.get(), q, sigma, kappa, pa_data,
-                           face_info);
+      PADGDiffusionSetup3D(quad1D, ne, nf, ir.GetWeights(), *el_geom, *face_geom,
+                           nbr_geom.get(), q, sigma, kappa, pa_data, face_info);
    }
 }
 
@@ -549,76 +524,529 @@ void DGDiffusionIntegrator::AssemblePABoundaryFaces(
    SetupPA(fes, FaceType::Boundary);
 }
 
-void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(const Vector &x,
-                                                           const Vector &dxdn,
-                                                           Vector &y,
-                                                           Vector &dydn) const
+template<int T_D1D = 0, int T_Q1D = 0> static
+void PADGDiffusionApply2D(const int NF,
+                          const Array<real_t> &b,
+                          const Array<real_t> &bt,
+                          const Array<real_t>& g,
+                          const Array<real_t>& gt,
+                          const real_t sigma,
+                          const Vector &pa_data,
+                          const Vector &x_,
+                          const Vector &dxdn_,
+                          Vector &y_,
+                          Vector &dydn_,
+                          const int d1d = 0,
+                          const int q1d = 0)
 {
-   ApplyPAKernels::Run(dim, dofs1D, quad1D, nf, maps->B, maps->Bt, maps->G,
-                       maps->Gt, sigma, pa_data, x, dxdn, y, dydn, dofs1D,
-                       quad1D);
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B_ = Reshape(b.Read(), Q1D, D1D);
+   auto G_ = Reshape(g.Read(), Q1D, D1D);
+
+   auto pa = Reshape(pa_data.Read(), 6, Q1D, NF); // (q, 1/h, J00, J01, J10, J11)
+
+   auto x =    Reshape(x_.Read(),         D1D, 2, NF);
+   auto y =    Reshape(y_.ReadWrite(),    D1D, 2, NF);
+   auto dxdn = Reshape(dxdn_.Read(),      D1D, 2, NF);
+   auto dydn = Reshape(dydn_.ReadWrite(), D1D, 2, NF);
+
+   const int NBX = std::max(D1D, Q1D);
+
+   mfem::forall_2D(NF, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t u0[max_D1D];
+      MFEM_SHARED real_t u1[max_D1D];
+      MFEM_SHARED real_t du0[max_D1D];
+      MFEM_SHARED real_t du1[max_D1D];
+
+      MFEM_SHARED real_t Bu0[max_Q1D];
+      MFEM_SHARED real_t Bu1[max_Q1D];
+      MFEM_SHARED real_t Bdu0[max_Q1D];
+      MFEM_SHARED real_t Bdu1[max_Q1D];
+
+      MFEM_SHARED real_t r[max_Q1D];
+
+      MFEM_SHARED real_t BG[2*max_D1D*max_Q1D];
+      DeviceMatrix B(BG, Q1D, D1D);
+      DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D);
+
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(p,x,Q1D)
+         {
+            for (int d = 0; d < D1D; ++d)
+            {
+               B(p,d) = B_(p,d);
+               G(p,d) = G_(p,d);
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // copy edge values to u0, u1 and copy edge normals to du0, du1
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            u[d] = x(d, side, f);
+            du[d] = dxdn(d, side, f);
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // eval @ quad points
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         real_t *Bu = (side == 0) ? Bu0 : Bu1;
+         real_t *Bdu = (side == 0) ? Bdu0 : Bdu1;
+
+         MFEM_FOREACH_THREAD(p,x,Q1D)
+         {
+            const real_t Je_side[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)};
+
+            Bu[p] = 0.0;
+            Bdu[p] = 0.0;
+
+            for (int d = 0; d < D1D; ++d)
+            {
+               const real_t b = B(p,d);
+               const real_t g = G(p,d);
+
+               Bu[p] += b*u[d];
+               Bdu[p] += Je_side[0] * b * du[d] + Je_side[1] * g * u[d];
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // term - < {Q du/dn}, [v] > +  kappa * < {Q/h} [u], [v] >:
+      if (MFEM_THREAD_ID(y) == 0)
+      {
+         MFEM_FOREACH_THREAD(p,x,Q1D)
+         {
+            const real_t q = pa(0, p, f);
+            const real_t hi = pa(1, p, f);
+            const real_t jump = Bu0[p] - Bu1[p];
+            const real_t avg = Bdu0[p] + Bdu1[p]; // = {Q du/dn} * w * det(J)
+            r[p] = -avg + hi * q * jump;
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(d,x,D1D)
+      {
+         real_t Br = 0.0;
+
+         for (int p = 0; p < Q1D; ++p)
+         {
+            Br += B(p, d) * r[p];
+         }
+
+         u0[d] =  Br; // overwrite u0, u1
+         u1[d] = -Br;
+      } // for d
+      MFEM_SYNC_THREAD;
+
+
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            du[d] = 0.0;
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // term sigma * < [u], {Q dv/dn} >
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t * const du = (side == 0) ? du0 : du1;
+         real_t * const u = (side == 0) ? u0 : u1;
+
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            for (int p = 0; p < Q1D; ++p)
+            {
+               const real_t Je[] = {pa(2 + 2*side, p, f), pa(2 + 2*side + 1, p, f)};
+               const real_t jump = Bu0[p] - Bu1[p];
+               const real_t r_p = Je[0] * jump; // normal
+               const real_t w_p = Je[1] * jump; // tangential
+               du[d] += sigma * B(p, d) * r_p;
+               u[d] += sigma * G(p, d) * w_p;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side,y,2)
+      {
+         real_t *u = (side == 0) ? u0 : u1;
+         real_t *du = (side == 0) ? du0 : du1;
+         MFEM_FOREACH_THREAD(d,x,D1D)
+         {
+            y(d, side, f) += u[d];
+            dydn(d, side, f) += du[d];
+         }
+      }
+   }); // mfem::forall
 }
 
-DGDiffusionIntegrator::DGDiffusionIntegrator(const real_t s, const real_t k)
-   : sigma(s), kappa(k)
+template <int T_D1D = 0, int T_Q1D = 0>
+static void PADGDiffusionApply3D(const int NF,
+                                 const Array<real_t>& b,
+                                 const Array<real_t>& bt,
+                                 const Array<real_t>& g,
+                                 const Array<real_t>& gt,
+                                 const real_t sigma,
+                                 const Vector& pa_data,
+                                 const Vector& x_,
+                                 const Vector& dxdn_,
+                                 Vector& y_,
+                                 Vector& dydn_,
+                                 const int d1d = 0,
+                                 const int q1d = 0)
 {
-   static Kernels kernels;
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+
+   auto B_ = Reshape(b.Read(), Q1D, D1D);
+   auto G_ = Reshape(g.Read(), Q1D, D1D);
+
+   // (J0[0], J0[1], J0[2], J1[0], J1[1], J1[2], q/h)
+   auto pa = Reshape(pa_data.Read(), 7, Q1D, Q1D, NF);
+
+   auto x =    Reshape(x_.Read(),         D1D, D1D, 2, NF);
+   auto y =    Reshape(y_.ReadWrite(),    D1D, D1D, 2, NF);
+   auto dxdn = Reshape(dxdn_.Read(),      D1D, D1D, 2, NF);
+   auto dydn = Reshape(dydn_.ReadWrite(), D1D, D1D, 2, NF);
+
+   const int NBX = std::max(D1D, Q1D);
+
+   mfem::forall_3D(NF, NBX, NBX, 2, [=] MFEM_HOST_DEVICE (int f) -> void
+   {
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      MFEM_SHARED real_t u0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t u1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t du0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t du1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Gu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Gu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Bu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Bu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t Bdu0[max_Q1D][max_Q1D];
+      MFEM_SHARED real_t Bdu1[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t kappa_Qh[max_Q1D][max_Q1D];
+
+      MFEM_SHARED real_t nJe[2][max_Q1D][max_Q1D][3];
+      MFEM_SHARED real_t BG[2*max_D1D*max_Q1D];
+
+      // some buffers are reused multiple times, but for clarity have new names:
+      real_t (*Bj0)[max_Q1D] = Bu0;
+      real_t (*Bj1)[max_Q1D] = Bu1;
+      real_t (*Bjn0)[max_Q1D] = Bdu0;
+      real_t (*Bjn1)[max_Q1D] = Bdu1;
+      real_t (*Gj0)[max_Q1D] = Gu0;
+      real_t (*Gj1)[max_Q1D] = Gu1;
+
+      DeviceMatrix B(BG, Q1D, D1D);
+      DeviceMatrix G(BG + D1D*Q1D, Q1D, D1D);
+
+      // copy face values to u0, u1 and copy normals to du0, du1
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               u[d2][d1] = x(d1, d2, side, f); // copy transposed for better memory access
+               du[d2][d1] = dxdn(d1, d2, side, f);
+            }
+         }
+
+         MFEM_FOREACH_THREAD(p1, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(p2, y, Q1D)
+            {
+               for (int l=0; l < 3; ++l)
+               {
+                  nJe[side][p2][p1][l] = pa(3*side + l, p1, p2, f);
+               }
+
+               if (side == 0)
+               {
+                  kappa_Qh[p2][p1] = pa(6, p1, p2, f);
+               }
+            }
+         }
+
+         if (side == 0)
+         {
+            MFEM_FOREACH_THREAD(p, x, Q1D)
+            {
+               MFEM_FOREACH_THREAD(d, y, D1D)
+               {
+                  B(p, d) = B_(p, d);
+                  G(p, d) = G_(p, d);
+               }
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // eval u and normal derivative @ quad points
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
+         real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
+         real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
+
+         MFEM_FOREACH_THREAD(p1, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(d2, y, D1D)
+            {
+               real_t bu = 0.0;
+               real_t bdu = 0.0;
+               real_t gu = 0.0;
+
+               for (int d1=0; d1 < D1D; ++d1)
+               {
+                  const real_t b = B(p1, d1);
+                  const real_t g = G(p1, d1);
+
+                  bu += b * u[d2][d1];
+                  bdu += b * du[d2][d1];
+                  gu += g * u[d2][d1];
+               }
+
+               Bu[p1][d2] = bu;
+               Bdu[p1][d2] = bdu;
+               Gu[p1][d2] = gu;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t (*Bu)[max_Q1D] = (side == 0) ? Bu0 : Bu1;
+         real_t (*Gu)[max_Q1D] = (side == 0) ? Gu0 : Gu1;
+         real_t (*Bdu)[max_Q1D] = (side == 0) ? Bdu0 : Bdu1;
+
+         MFEM_FOREACH_THREAD(p2, x, Q1D)
+         {
+            MFEM_FOREACH_THREAD(p1, y, Q1D)
+            {
+               const real_t * Je = nJe[side][p2][p1];
+
+               real_t bbu = 0.0;
+               real_t bgu = 0.0;
+               real_t gbu = 0.0;
+               real_t bbdu = 0.0;
+
+               for (int d2 = 0; d2 < D1D; ++d2)
+               {
+                  const real_t b = B(p2, d2);
+                  const real_t g = G(p2, d2);
+                  bbu += b * Bu[p1][d2];
+                  gbu += g * Bu[p1][d2];
+                  bgu += b * Gu[p1][d2];
+                  bbdu += b * Bdu[p1][d2];
+               }
+
+               u[p2][p1] = bbu;
+               // du <- Q du/dn * w * det(J)
+               du[p2][p1] = Je[0] * bbdu + Je[1] * bgu + Je[2] * gbu;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
+         real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
+         real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
+
+         MFEM_FOREACH_THREAD(d1, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(p2, y, Q1D)
+            {
+               real_t bj = 0.0;
+               real_t bjn = 0.0;
+               real_t gj = 0.0;
+               real_t br = 0.0;
+
+               for (int p1 = 0; p1 < Q1D; ++p1)
+               {
+                  const real_t b = B(p1, d1);
+                  const real_t g = G(p1, d1);
+
+                  const real_t * Je = nJe[side][p2][p1];
+
+                  const real_t jump = u0[p2][p1] - u1[p2][p1];
+                  const real_t avg = du0[p2][p1] + du1[p2][p1];
+
+                  // r = - < {Q du/dn}, [v] > + kappa * < {Q/h} [u], [v] >
+                  const real_t r = -avg + kappa_Qh[p2][p1] * jump;
+
+                  // bj, gj, bjn contribute to sigma term
+                  bj += b * Je[0] * jump;
+                  gj += g * Je[1] * jump;
+                  bjn += b * Je[2] * jump;
+
+                  br += b * r;
+               }
+
+               Bj[d1][p2] = sigma * bj;
+               Bjn[d1][p2] = sigma * bjn;
+
+               // group br and gj together since we will multiply them both by B
+               // and then sum
+               const real_t sgn = (side == 0) ? 1.0 : -1.0;
+               Gj[d1][p2] = sgn * br + sigma * gj;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+         real_t (*Bj)[max_Q1D] = (side == 0) ? Bj0 : Bj1;
+         real_t (*Bjn)[max_Q1D] = (side == 0) ? Bjn0 : Bjn1;
+         real_t (*Gj)[max_Q1D] = (side == 0) ? Gj0 : Gj1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               real_t bbj = 0.0;
+               real_t gbj = 0.0;
+               real_t bgj = 0.0;
+
+               for (int p2 = 0; p2 < Q1D; ++p2)
+               {
+                  const real_t b = B(p2, d2);
+                  const real_t g = G(p2, d2);
+
+                  bbj += b * Bj[d1][p2];
+                  bgj += b * Gj[d1][p2];
+                  gbj += g * Bjn[d1][p2];
+               }
+
+               du[d2][d1] = bbj;
+               u[d2][d1] = bgj + gbj;
+            }
+         }
+      }
+      MFEM_SYNC_THREAD;
+
+      // map back to y and dydn
+      MFEM_FOREACH_THREAD(side, z, 2)
+      {
+         const real_t (*u)[max_Q1D] = (side == 0) ? u0 : u1;
+         const real_t (*du)[max_Q1D] = (side == 0) ? du0 : du1;
+
+         MFEM_FOREACH_THREAD(d2, x, D1D)
+         {
+            MFEM_FOREACH_THREAD(d1, y, D1D)
+            {
+               y(d1, d2, side, f) += u[d2][d1];
+               dydn(d1, d2, side, f) += du[d2][d1];
+            }
+         }
+      }
+   });
 }
 
-DGDiffusionIntegrator::DGDiffusionIntegrator(Coefficient &q, const real_t s,
-                                             const real_t k)
-   : DGDiffusionIntegrator(s, k)
-{
-   Q = &q;
-}
-
-DGDiffusionIntegrator::DGDiffusionIntegrator(MatrixCoefficient &q,
-                                             const real_t s, const real_t k)
-   : DGDiffusionIntegrator(s, k)
-{
-   MQ = &q;
-}
-
-/// \cond DO_NOT_DOCUMENT
-
-DGDiffusionIntegrator::ApplyKernelType
-DGDiffusionIntegrator::ApplyPAKernels::Fallback(int dim, int, int)
+static void PADGDiffusionApply(const int dim,
+                               const int D1D,
+                               const int Q1D,
+                               const int NF,
+                               const Array<real_t> &B,
+                               const Array<real_t> &Bt,
+                               const Array<real_t> &G,
+                               const Array<real_t> &Gt,
+                               const real_t sigma,
+                               const Vector &pa_data,
+                               const Vector &x,
+                               const Vector &dxdn,
+                               Vector &y,
+                               Vector &dydn)
 {
    if (dim == 2)
    {
-      return internal::PADGDiffusionApply2D;
+      auto kernel = PADGDiffusionApply2D<0,0>;
+      switch ((D1D << 4 ) | Q1D)
+      {
+         case 0x23: kernel = PADGDiffusionApply2D<2,3>; break;
+         case 0x34: kernel = PADGDiffusionApply2D<3,4>; break;
+         case 0x45: kernel = PADGDiffusionApply2D<4,5>; break;
+         case 0x56: kernel = PADGDiffusionApply2D<5,6>; break;
+         case 0x67: kernel = PADGDiffusionApply2D<6,7>; break;
+         case 0x78: kernel = PADGDiffusionApply2D<7,8>; break;
+         case 0x89: kernel = PADGDiffusionApply2D<8,9>; break;
+         case 0x9A: kernel = PADGDiffusionApply2D<9,10>; break;
+      }
+      kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D);
    }
    else if (dim == 3)
    {
-      return internal::PADGDiffusionApply3D;
+      auto kernel = PADGDiffusionApply3D<0,0>;
+      switch ((D1D << 4) | Q1D)
+      {
+         case 0x24: kernel = PADGDiffusionApply3D<2,4>; break;
+         case 0x35: kernel = PADGDiffusionApply3D<3,5>; break;
+         case 0x46: kernel = PADGDiffusionApply3D<4,6>; break;
+         case 0x57: kernel = PADGDiffusionApply3D<5,7>; break;
+         case 0x68: kernel = PADGDiffusionApply3D<6,8>; break;
+         case 0x79: kernel = PADGDiffusionApply3D<7,9>; break;
+         case 0x8A: kernel = PADGDiffusionApply3D<8,10>; break;
+         case 0x9B: kernel = PADGDiffusionApply3D<9,11>; break;
+      }
+      kernel(NF, B, Bt, G, Gt, sigma, pa_data, x, dxdn, y, dydn, D1D, Q1D);
    }
    else
    {
-      MFEM_ABORT("");
+      MFEM_ABORT("Unsupported dimension");
    }
 }
 
-DGDiffusionIntegrator::Kernels::Kernels()
+void DGDiffusionIntegrator::AddMultPAFaceNormalDerivatives(
+   const Vector &x, const Vector &dxdn, Vector &y, Vector &dydn) const
 {
-   DGDiffusionIntegrator::AddSpecialization<2, 2, 3>();
-   DGDiffusionIntegrator::AddSpecialization<2, 3, 4>();
-   DGDiffusionIntegrator::AddSpecialization<2, 4, 5>();
-   DGDiffusionIntegrator::AddSpecialization<2, 5, 6>();
-   DGDiffusionIntegrator::AddSpecialization<2, 6, 7>();
-   DGDiffusionIntegrator::AddSpecialization<2, 7, 8>();
-   DGDiffusionIntegrator::AddSpecialization<2, 8, 9>();
-   DGDiffusionIntegrator::AddSpecialization<2, 9, 10>();
-
-   DGDiffusionIntegrator::AddSpecialization<3, 2, 4>();
-   DGDiffusionIntegrator::AddSpecialization<3, 3, 5>();
-   DGDiffusionIntegrator::AddSpecialization<3, 4, 6>();
-   DGDiffusionIntegrator::AddSpecialization<3, 5, 7>();
-   DGDiffusionIntegrator::AddSpecialization<3, 6, 8>();
-   DGDiffusionIntegrator::AddSpecialization<3, 7, 9>();
-   DGDiffusionIntegrator::AddSpecialization<3, 8, 10>();
-   DGDiffusionIntegrator::AddSpecialization<3, 9, 11>();
+   PADGDiffusionApply(dim, dofs1D, quad1D, nf,
+                      maps->B, maps->Bt, maps->G, maps->Gt,
+                      sigma, pa_data, x, dxdn, y, dydn);
 }
 
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/bilininteg_dgtrace_kernels.hpp

@@ -1,793 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef BILININTEG_DGTRACE_KERNELS_HPP
-#define BILININTEG_DGTRACE_KERNELS_HPP
-
-#include "../../general/forall.hpp"
-#include "../bilininteg.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-#include "../restriction.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-
-// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApply2D(const int NF, const Array<real_t> &b,
-                             const Array<real_t> &bt, const Vector &op_,
-                             const Vector &x_, Vector &y_, const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][VDIM];
-      real_t u1[max_D1D][VDIM];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            u0[d][c] = x(d, c, 0, f);
-            u1[d][c] = x(d, c, 1, f);
-         }
-      }
-      real_t Bu0[max_Q1D][VDIM];
-      real_t Bu1[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            Bu0[q][c] = 0.0;
-            Bu1[q][c] = 0.0;
-         }
-         for (int d = 0; d < D1D; ++d)
-         {
-            const real_t b = B(q, d);
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q][c] += b * u0[d][c];
-               Bu1[q][c] += b * u1[d][c];
-            }
-         }
-      }
-      real_t DBu[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            DBu[q][c] = op(q, 0, 0, f) * Bu0[q][c] + op(q, 1, 0, f) * Bu1[q][c];
-         }
-      }
-      real_t BDBu[max_D1D][VDIM];
-      for (int d = 0; d < D1D; ++d)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            BDBu[d][c] = 0.0;
-         }
-         for (int q = 0; q < Q1D; ++q)
-         {
-            const real_t b = Bt(d, q);
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBu[d][c] += b * DBu[q][c];
-            }
-         }
-         for (int c = 0; c < VDIM; c++)
-         {
-            y(d, c, 0, f) += BDBu[d][c];
-            y(d, c, 1, f) += -BDBu[d][c];
-         }
-      }
-   });
-}
-
-// PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApply3D(const int NF, const Array<real_t> &b,
-                             const Array<real_t> &bt, const Vector &op_,
-                             const Vector &x_, Vector &y_, const int d1d = 0,
-                             const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][max_D1D][VDIM];
-      real_t u1[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; d1++)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               u0[d1][d2][c] = x(d1, d2, c, 0, f);
-               u1[d1][d2][c] = x(d1, d2, c, 1, f);
-            }
-         }
-      }
-      real_t Bu0[max_Q1D][max_D1D][VDIM];
-      real_t Bu1[max_Q1D][max_D1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q][d2][c] = 0.0;
-               Bu1[q][d2][c] = 0.0;
-            }
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q, d1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  Bu0[q][d2][c] += b * u0[d1][d2][c];
-                  Bu1[q][d2][c] += b * u1[d1][d2][c];
-               }
-            }
-         }
-      }
-      real_t BBu0[max_Q1D][max_Q1D][VDIM];
-      real_t BBu1[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; q2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBu0[q1][q2][c] = 0.0;
-               BBu1[q1][q2][c] = 0.0;
-            }
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBu0[q1][q2][c] += b * Bu0[q1][d2][c];
-                  BBu1[q1][q2][c] += b * Bu1[q1][d2][c];
-               }
-            }
-         }
-      }
-      real_t DBBu[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; q2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               DBBu[q1][q2][c] = op(q1, q2, 0, 0, f) * BBu0[q1][q2][c] +
-                                 op(q1, q2, 1, 0, f) * BBu1[q1][q2][c];
-            }
-         }
-      }
-      real_t BDBBu[max_Q1D][max_D1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBBu[q1][d2][c] = 0.0;
-            }
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BDBBu[q1][d2][c] += b * DBBu[q1][q2][c];
-               }
-            }
-         }
-      }
-      real_t BBDBBu[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; ++d1)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBDBBu[d1][d2][c] = 0.0;
-            }
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBDBBu[d1][d2][c] += b * BDBBu[q1][d2][c];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               y(d1, d2, c, 0, f) += BBDBBu[d1][d2][c];
-               y(d1, d2, c, 1, f) += -BBDBBu[d1][d2][c];
-            }
-         }
-      }
-   });
-}
-
-// Optimized PA DGTrace Apply 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADGTraceApply3D(const int NF, const Array<real_t> &b,
-                                 const Array<real_t> &bt, const Vector &op_,
-                                 const Vector &x_, Vector &y_,
-                                 const int d1d = 0, const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
-
-   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D];
-      MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D];
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            u0[tidz][d1][d2] = x(d1, d2, 0, f);
-            u1[tidz][d1][d2] = x(d1, d2, 1, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D];
-      MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t Bu0_ = 0.0;
-            real_t Bu1_ = 0.0;
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q1, d1);
-               Bu0_ += b * u0[tidz][d1][d2];
-               Bu1_ += b * u1[tidz][d1][d2];
-            }
-            Bu0[tidz][q1][d2] = Bu0_;
-            Bu1[tidz][q1][d2] = Bu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            real_t BBu0_ = 0.0;
-            real_t BBu1_ = 0.0;
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               BBu0_ += b * Bu0[tidz][q1][d2];
-               BBu1_ += b * Bu1[tidz][q1][d2];
-            }
-            BBu0[tidz][q1][q2] = BBu0_;
-            BBu1[tidz][q1][q2] = BBu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t DBBu[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            DBBu[tidz][q1][q2] = op(q1, q2, 0, 0, f) * BBu0[tidz][q1][q2] +
-                                 op(q1, q2, 1, 0, f) * BBu1[tidz][q1][q2];
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BDBBu[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BDBBu_ = 0.0;
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               BDBBu_ += b * DBBu[tidz][q1][q2];
-            }
-            BDBBu[tidz][q1][d2] = BDBBu_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BBDBBu_ = 0.0;
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               BBDBBu_ += b * BDBBu[tidz][q1][d2];
-            }
-            y(d1, d2, 0, f) += BBDBBu_;
-            y(d1, d2, 1, f) += -BBDBBu_;
-         }
-      }
-   });
-}
-
-// PA DGTrace Apply 2D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApplyTranspose2D(const int NF, const Array<real_t> &b,
-                                      const Array<real_t> &bt,
-                                      const Vector &op_, const Vector &x_,
-                                      Vector &y_, const int d1d = 0,
-                                      const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][VDIM];
-      real_t u1[max_D1D][VDIM];
-      for (int d = 0; d < D1D; d++)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            u0[d][c] = x(d, c, 0, f);
-            u1[d][c] = x(d, c, 1, f);
-         }
-      }
-      real_t Bu0[max_Q1D][VDIM];
-      real_t Bu1[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            Bu0[q][c] = 0.0;
-            Bu1[q][c] = 0.0;
-         }
-         for (int d = 0; d < D1D; ++d)
-         {
-            const real_t b = B(q, d);
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q][c] += b * u0[d][c];
-               Bu1[q][c] += b * u1[d][c];
-            }
-         }
-      }
-      real_t DBu0[max_Q1D][VDIM];
-      real_t DBu1[max_Q1D][VDIM];
-      for (int q = 0; q < Q1D; ++q)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            DBu0[q][c] =
-               op(q, 0, 0, f) * Bu0[q][c] + op(q, 0, 1, f) * Bu1[q][c];
-            DBu1[q][c] =
-               op(q, 1, 0, f) * Bu0[q][c] + op(q, 1, 1, f) * Bu1[q][c];
-         }
-      }
-      real_t BDBu0[max_D1D][VDIM];
-      real_t BDBu1[max_D1D][VDIM];
-      for (int d = 0; d < D1D; ++d)
-      {
-         for (int c = 0; c < VDIM; c++)
-         {
-            BDBu0[d][c] = 0.0;
-            BDBu1[d][c] = 0.0;
-         }
-         for (int q = 0; q < Q1D; ++q)
-         {
-            const real_t b = Bt(d, q);
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBu0[d][c] += b * DBu0[q][c];
-               BDBu1[d][c] += b * DBu1[q][c];
-            }
-         }
-         for (int c = 0; c < VDIM; c++)
-         {
-            y(d, c, 0, f) += BDBu0[d][c];
-            y(d, c, 1, f) += BDBu1[d][c];
-         }
-      }
-   });
-}
-
-// PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0>
-static void PADGTraceApplyTranspose3D(const int NF, const Array<real_t> &b,
-                                      const Array<real_t> &bt,
-                                      const Vector &op_, const Vector &x_,
-                                      Vector &y_, const int d1d = 0,
-                                      const int q1d = 0)
-{
-   const int VDIM = 1;
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, 2, NF);
-
-   mfem::forall(NF, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int VDIM = 1;
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      real_t u0[max_D1D][max_D1D][VDIM];
-      real_t u1[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; d1++)
-      {
-         for (int d2 = 0; d2 < D1D; d2++)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               u0[d1][d2][c] = x(d1, d2, c, 0, f);
-               u1[d1][d2][c] = x(d1, d2, c, 1, f);
-            }
-         }
-      }
-      real_t Bu0[max_Q1D][max_D1D][VDIM];
-      real_t Bu1[max_Q1D][max_D1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int d2 = 0; d2 < D1D; ++d2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               Bu0[q1][d2][c] = 0.0;
-               Bu1[q1][d2][c] = 0.0;
-            }
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q1, d1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  Bu0[q1][d2][c] += b * u0[d1][d2][c];
-                  Bu1[q1][d2][c] += b * u1[d1][d2][c];
-               }
-            }
-         }
-      }
-      real_t BBu0[max_Q1D][max_Q1D][VDIM];
-      real_t BBu1[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBu0[q1][q2][c] = 0.0;
-               BBu1[q1][q2][c] = 0.0;
-            }
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBu0[q1][q2][c] += b * Bu0[q1][d2][c];
-                  BBu1[q1][q2][c] += b * Bu1[q1][d2][c];
-               }
-            }
-         }
-      }
-      real_t DBu0[max_Q1D][max_Q1D][VDIM];
-      real_t DBu1[max_Q1D][max_Q1D][VDIM];
-      for (int q1 = 0; q1 < Q1D; ++q1)
-      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
-         {
-            const real_t D00 = op(q1, q2, 0, 0, f);
-            const real_t D01 = op(q1, q2, 0, 1, f);
-            const real_t D10 = op(q1, q2, 1, 0, f);
-            const real_t D11 = op(q1, q2, 1, 1, f);
-            for (int c = 0; c < VDIM; c++)
-            {
-               DBu0[q1][q2][c] = D00 * BBu0[q1][q2][c] + D01 * BBu1[q1][q2][c];
-               DBu1[q1][q2][c] = D10 * BBu0[q1][q2][c] + D11 * BBu1[q1][q2][c];
-            }
-         }
-      }
-      real_t BDBu0[max_D1D][max_Q1D][VDIM];
-      real_t BDBu1[max_D1D][max_Q1D][VDIM];
-      for (int d1 = 0; d1 < D1D; ++d1)
-      {
-         for (int q2 = 0; q2 < Q1D; ++q2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BDBu0[d1][q2][c] = 0.0;
-               BDBu1[d1][q2][c] = 0.0;
-            }
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BDBu0[d1][q2][c] += b * DBu0[q1][q2][c];
-                  BDBu1[d1][q2][c] += b * DBu1[q1][q2][c];
-               }
-            }
-         }
-      }
-      real_t BBDBu0[max_D1D][max_D1D][VDIM];
-      real_t BBDBu1[max_D1D][max_D1D][VDIM];
-      for (int d1 = 0; d1 < D1D; ++d1)
-      {
-         for (int d2 = 0; d2 < D1D; ++d2)
-         {
-            for (int c = 0; c < VDIM; c++)
-            {
-               BBDBu0[d1][d2][c] = 0.0;
-               BBDBu1[d1][d2][c] = 0.0;
-            }
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               for (int c = 0; c < VDIM; c++)
-               {
-                  BBDBu0[d1][d2][c] += b * BDBu0[d1][q2][c];
-                  BBDBu1[d1][d2][c] += b * BDBu1[d1][q2][c];
-               }
-            }
-            for (int c = 0; c < VDIM; c++)
-            {
-               y(d1, d2, c, 0, f) += BBDBu0[d1][d2][c];
-               y(d1, d2, c, 1, f) += BBDBu1[d1][d2][c];
-            }
-         }
-      }
-   });
-}
-
-// Optimized PA DGTrace Apply Transpose 3D kernel for Gauss-Lobatto/Bernstein
-template <int T_D1D = 0, int T_Q1D = 0, int T_NBZ = 0>
-static void SmemPADGTraceApplyTranspose3D(const int NF, const Array<real_t> &b,
-                                          const Array<real_t> &bt,
-                                          const Vector &op_, const Vector &x_,
-                                          Vector &y_, const int d1d = 0,
-                                          const int q1d = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D, Q1D, 2, 2, NF);
-   auto x = Reshape(x_.Read(), D1D, D1D, 2, NF);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, 2, NF);
-
-   mfem::forall_2D_batch(NF, Q1D, Q1D, NBZ, [=] MFEM_HOST_DEVICE(int f)
-   {
-      const int tidz = MFEM_THREAD_ID(z);
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      // the following variables are evaluated at compile time
-      constexpr int NBZ = T_NBZ ? T_NBZ : 1;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      MFEM_SHARED real_t u0[NBZ][max_D1D][max_D1D];
-      MFEM_SHARED real_t u1[NBZ][max_D1D][max_D1D];
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            u0[tidz][d1][d2] = x(d1, d2, 0, f);
-            u1[tidz][d1][d2] = x(d1, d2, 1, f);
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t Bu0[NBZ][max_Q1D][max_D1D];
-      MFEM_SHARED real_t Bu1[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t Bu0_ = 0.0;
-            real_t Bu1_ = 0.0;
-            for (int d1 = 0; d1 < D1D; ++d1)
-            {
-               const real_t b = B(q1, d1);
-               Bu0_ += b * u0[tidz][d1][d2];
-               Bu1_ += b * u1[tidz][d1][d2];
-            }
-            Bu0[tidz][q1][d2] = Bu0_;
-            Bu1[tidz][q1][d2] = Bu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BBu0[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED real_t BBu1[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            real_t BBu0_ = 0.0;
-            real_t BBu1_ = 0.0;
-            for (int d2 = 0; d2 < D1D; ++d2)
-            {
-               const real_t b = B(q2, d2);
-               BBu0_ += b * Bu0[tidz][q1][d2];
-               BBu1_ += b * Bu1[tidz][q1][d2];
-            }
-            BBu0[tidz][q1][q2] = BBu0_;
-            BBu1[tidz][q1][q2] = BBu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t DBBu0[NBZ][max_Q1D][max_Q1D];
-      MFEM_SHARED real_t DBBu1[NBZ][max_Q1D][max_Q1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(q2, y, Q1D)
-         {
-            const real_t D00 = op(q1, q2, 0, 0, f);
-            const real_t D01 = op(q1, q2, 0, 1, f);
-            const real_t D10 = op(q1, q2, 1, 0, f);
-            const real_t D11 = op(q1, q2, 1, 1, f);
-            const real_t u0q = BBu0[tidz][q1][q2];
-            const real_t u1q = BBu1[tidz][q1][q2];
-            DBBu0[tidz][q1][q2] = D00 * u0q + D01 * u1q;
-            DBBu1[tidz][q1][q2] = D10 * u0q + D11 * u1q;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_SHARED real_t BDBBu0[NBZ][max_Q1D][max_D1D];
-      MFEM_SHARED real_t BDBBu1[NBZ][max_Q1D][max_D1D];
-      MFEM_FOREACH_THREAD(q1, x, Q1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BDBBu0_ = 0.0;
-            real_t BDBBu1_ = 0.0;
-            for (int q2 = 0; q2 < Q1D; ++q2)
-            {
-               const real_t b = Bt(d2, q2);
-               BDBBu0_ += b * DBBu0[tidz][q1][q2];
-               BDBBu1_ += b * DBBu1[tidz][q1][q2];
-            }
-            BDBBu0[tidz][q1][d2] = BDBBu0_;
-            BDBBu1[tidz][q1][d2] = BDBBu1_;
-         }
-      }
-      MFEM_SYNC_THREAD;
-      MFEM_FOREACH_THREAD(d1, x, D1D)
-      {
-         MFEM_FOREACH_THREAD(d2, y, D1D)
-         {
-            real_t BBDBBu0_ = 0.0;
-            real_t BBDBBu1_ = 0.0;
-            for (int q1 = 0; q1 < Q1D; ++q1)
-            {
-               const real_t b = Bt(d1, q1);
-               BBDBBu0_ += b * BDBBu0[tidz][q1][d2];
-               BBDBBu1_ += b * BDBBu1[tidz][q1][d2];
-            }
-            y(d1, d2, 0, f) += BBDBBu0_;
-            y(d1, d2, 1, f) += BBDBBu1_;
-         }
-      }
-   });
-}
-
-} // namespace internal
-
-template <int DIM, int D1D, int Q1D>
-DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PADGTraceApply2D<D1D, Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      if constexpr (D1D == 3 || D1D == 4)
-      {
-         return internal::SmemPADGTraceApply3D<D1D, Q1D, 2>;
-      }
-      else
-      {
-         return internal::SmemPADGTraceApply3D<D1D, Q1D>;
-      }
-   }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int D1D, int Q1D>
-DGTraceIntegrator::ApplyKernelType DGTraceIntegrator::ApplyPATKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PADGTraceApplyTranspose2D<D1D, Q1D>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::SmemPADGTraceApplyTranspose3D<D1D, Q1D>;
-   }
-   MFEM_ABORT("");
-}
-} // namespace mfem
-
-/// \endcond DO_NOT_DOCUMENT
-#endif

fem/integ/bilininteg_diffusion_ea.cpp

@@ -161,7 +161,8 @@ static void EADiffusionAssemble3D(const int NE,
    auto B = Reshape(b.Read(), Q1D, D1D);
    auto G = Reshape(g.Read(), Q1D, D1D);
    auto D = Reshape(padata.Read(), Q1D, Q1D, Q1D, 6, NE);
-   auto A = Reshape(eadata.ReadWrite(), D1D, D1D, D1D, D1D, D1D, D1D, NE);
+   auto A = Reshape(add ? eadata.ReadWrite() : eadata.Write(),
+                    D1D, D1D, D1D, D1D, D1D, D1D, NE);
    mfem::forall_3D(NE, D1D, D1D, D1D, [=] MFEM_HOST_DEVICE (int e)
    {
       const int D1D = T_D1D ? T_D1D : d1d;
@@ -246,58 +247,60 @@ void DiffusionIntegrator::AssembleEA(const FiniteElementSpace &fes,
                                      Vector &ea_data,
                                      const bool add)
 {
+   MFEM_PERF_FUNCTION;
    AssemblePA(fes);
    ne = fes.GetMesh()->GetNE();
    const Array<real_t> &B = maps->B;
    const Array<real_t> &G = maps->G;
+   decltype(&EADiffusionAssemble1D<>) kernel = nullptr;
    if (dim == 1)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADiffusionAssemble1D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble1D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble1D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble1D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble1D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble1D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble1D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble1D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble1D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: kernel = EADiffusionAssemble1D<2,2>;
+         case 0x33: kernel = EADiffusionAssemble1D<3,3>;
+         case 0x44: kernel = EADiffusionAssemble1D<4,4>;
+         case 0x55: kernel = EADiffusionAssemble1D<5,5>;
+         case 0x66: kernel = EADiffusionAssemble1D<6,6>;
+         case 0x77: kernel = EADiffusionAssemble1D<7,7>;
+         case 0x88: kernel = EADiffusionAssemble1D<8,8>;
+         case 0x99: kernel = EADiffusionAssemble1D<9,9>;
+         default:   kernel = EADiffusionAssemble1D<>;
       }
    }
    else if (dim == 2)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x22: return EADiffusionAssemble2D<2,2>(ne,B,G,pa_data,ea_data,add);
-         case 0x33: return EADiffusionAssemble2D<3,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x44: return EADiffusionAssemble2D<4,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x55: return EADiffusionAssemble2D<5,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x66: return EADiffusionAssemble2D<6,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x77: return EADiffusionAssemble2D<7,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x88: return EADiffusionAssemble2D<8,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x99: return EADiffusionAssemble2D<9,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble2D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x22: kernel = EADiffusionAssemble2D<2,2>;
+         case 0x33: kernel = EADiffusionAssemble2D<3,3>;
+         case 0x44: kernel = EADiffusionAssemble2D<4,4>;
+         case 0x55: kernel = EADiffusionAssemble2D<5,5>;
+         case 0x66: kernel = EADiffusionAssemble2D<6,6>;
+         case 0x77: kernel = EADiffusionAssemble2D<7,7>;
+         case 0x88: kernel = EADiffusionAssemble2D<8,8>;
+         case 0x99: kernel = EADiffusionAssemble2D<9,9>;
+         default:   kernel = EADiffusionAssemble2D<>;
       }
    }
    else if (dim == 3)
    {
       switch ((dofs1D << 4 ) | quad1D)
       {
-         case 0x23: return EADiffusionAssemble3D<2,3>(ne,B,G,pa_data,ea_data,add);
-         case 0x34: return EADiffusionAssemble3D<3,4>(ne,B,G,pa_data,ea_data,add);
-         case 0x45: return EADiffusionAssemble3D<4,5>(ne,B,G,pa_data,ea_data,add);
-         case 0x56: return EADiffusionAssemble3D<5,6>(ne,B,G,pa_data,ea_data,add);
-         case 0x67: return EADiffusionAssemble3D<6,7>(ne,B,G,pa_data,ea_data,add);
-         case 0x78: return EADiffusionAssemble3D<7,8>(ne,B,G,pa_data,ea_data,add);
-         case 0x89: return EADiffusionAssemble3D<8,9>(ne,B,G,pa_data,ea_data,add);
-         default:   return EADiffusionAssemble3D(ne,B,G,pa_data,ea_data,add,
-                                                    dofs1D,quad1D);
+         case 0x23: kernel = EADiffusionAssemble3D<2,3>;
+         case 0x34: kernel = EADiffusionAssemble3D<3,4>;
+         case 0x45: kernel = EADiffusionAssemble3D<4,5>;
+         case 0x56: kernel = EADiffusionAssemble3D<5,6>;
+         case 0x67: kernel = EADiffusionAssemble3D<6,7>;
+         case 0x78: kernel = EADiffusionAssemble3D<7,8>;
+         case 0x89: kernel = EADiffusionAssemble3D<8,9>;
+         default:   kernel = EADiffusionAssemble3D<>;
       }
    }
-   MFEM_ABORT("Unknown kernel.");
+   MFEM_VERIFY(kernel != nullptr, "Unknown kernel.");
+   kernel(ne,B,G,pa_data,ea_data,add,dofs1D,quad1D);
+   // Free the PA data:
+   pa_data.Destroy();
 }
 
 }

fem/integ/bilininteg_diffusion_kernels.cpp

@@ -19,8 +19,6 @@ namespace mfem
 DiffusionIntegrator::Kernels::Kernels()
 {
    // 2D
-   // Q = P+1
-   DiffusionIntegrator::AddSpecialization<2,1,1>();
    DiffusionIntegrator::AddSpecialization<2,2,2>();
    DiffusionIntegrator::AddSpecialization<2,3,3>();
    DiffusionIntegrator::AddSpecialization<2,4,4>();
@@ -29,39 +27,17 @@ DiffusionIntegrator::Kernels::Kernels()
    DiffusionIntegrator::AddSpecialization<2,7,7>();
    DiffusionIntegrator::AddSpecialization<2,8,8>();
    DiffusionIntegrator::AddSpecialization<2,9,9>();
-   // Q = P+2
-   DiffusionIntegrator::AddSpecialization<2,1,2>();
-   DiffusionIntegrator::AddSpecialization<2,2,3>();
-   DiffusionIntegrator::AddSpecialization<2,3,4>();
-   DiffusionIntegrator::AddSpecialization<2,4,5>();
-   DiffusionIntegrator::AddSpecialization<2,5,6>();
-   DiffusionIntegrator::AddSpecialization<2,6,7>();
-   DiffusionIntegrator::AddSpecialization<2,7,8>();
-   DiffusionIntegrator::AddSpecialization<2,8,9>();
-   DiffusionIntegrator::AddSpecialization<2,9,10>();
-   // others
    // 3D
-   // Q = P+1
-   DiffusionIntegrator::AddSpecialization<3,1,1>();
    DiffusionIntegrator::AddSpecialization<3,2,2>();
-   DiffusionIntegrator::AddSpecialization<3,3,3>();
-   DiffusionIntegrator::AddSpecialization<3,4,4>();
-   DiffusionIntegrator::AddSpecialization<3,5,5>();
-   DiffusionIntegrator::AddSpecialization<3,6,6>();
-   DiffusionIntegrator::AddSpecialization<3,7,7>();
-   DiffusionIntegrator::AddSpecialization<3,8,8>();
-   // Q = P+2
-   DiffusionIntegrator::AddSpecialization<3,1,2>();
    DiffusionIntegrator::AddSpecialization<3,2,3>();
    DiffusionIntegrator::AddSpecialization<3,3,4>();
    DiffusionIntegrator::AddSpecialization<3,4,5>();
+   DiffusionIntegrator::AddSpecialization<3,4,6>();
    DiffusionIntegrator::AddSpecialization<3,5,6>();
+   DiffusionIntegrator::AddSpecialization<3,5,8>();
    DiffusionIntegrator::AddSpecialization<3,6,7>();
    DiffusionIntegrator::AddSpecialization<3,7,8>();
    DiffusionIntegrator::AddSpecialization<3,8,9>();
-   // others
-   DiffusionIntegrator::AddSpecialization<3,4,6>();
-   DiffusionIntegrator::AddSpecialization<3,5,8>();
 }
 
 namespace internal

fem/integ/bilininteg_diffusion_kernels.hpp

@@ -672,12 +672,12 @@ inline void SmemPADiffusionApply2D(const int NE,
       real_t (*Gt)[MQ1] = (real_t (*)[MQ1]) (sBG+1);
       MFEM_SHARED real_t Xz[NBZ][MD1][MD1];
       MFEM_SHARED real_t GD[2][NBZ][MD1][MQ1];
-      MFEM_SHARED real_t GQ[2][NBZ][MQ1][MQ1];
+      MFEM_SHARED real_t GQ[2][NBZ][MD1][MQ1];
       real_t (*X)[MD1] = (real_t (*)[MD1])(Xz + tidz);
-      real_t (*DQ0)[MQ1] = (real_t (*)[MQ1])(GD[0] + tidz);
-      real_t (*DQ1)[MQ1] = (real_t (*)[MQ1])(GD[1] + tidz);
-      real_t (*QQ0)[MQ1] = (real_t (*)[MQ1])(GQ[0] + tidz);
-      real_t (*QQ1)[MQ1] = (real_t (*)[MQ1])(GQ[1] + tidz);
+      real_t (*DQ0)[MD1] = (real_t (*)[MD1])(GD[0] + tidz);
+      real_t (*DQ1)[MD1] = (real_t (*)[MD1])(GD[1] + tidz);
+      real_t (*QQ0)[MD1] = (real_t (*)[MD1])(GQ[0] + tidz);
+      real_t (*QQ1)[MD1] = (real_t (*)[MD1])(GQ[1] + tidz);
       MFEM_FOREACH_THREAD(dy,y,D1D)
       {
          MFEM_FOREACH_THREAD(dx,x,D1D)
@@ -1221,9 +1221,9 @@ using DiagonalKernelType = DiffusionIntegrator::DiagonalKernelType;
 template<int DIM, int T_D1D, int T_Q1D>
 ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 2) { return internal::SmemPADiffusionApply2D<T_D1D,T_Q1D>; }
+   else if (DIM == 3) { return internal::SmemPADiffusionApply3D<T_D1D, T_Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
 inline
@@ -1237,9 +1237,9 @@ ApplyKernelType DiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int)
 template<int DIM, int D1D, int Q1D>
 DiagonalKernelType DiffusionIntegrator::DiagonalPAKernels::Kernel()
 {
-   if constexpr (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 2) { return internal::SmemPADiffusionDiagonal2D<D1D,Q1D>; }
+   else if (DIM == 3) { return internal::SmemPADiffusionDiagonal3D<D1D, Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
 inline DiagonalKernelType

fem/integ/bilininteg_diffusion_pa.cpp

@@ -39,6 +39,8 @@ void DiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
 // PA Diffusion Apply kernel
 void DiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+   MFEM_PERF_FUNCTION;
+
    if (DeviceCanUseCeed())
    {
       ceedOp->AddMult(x, y);
@@ -88,6 +90,8 @@ void DiffusionIntegrator::AddMultTransposePA(const Vector &x, Vector &y) const
 
 void DiffusionIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
+   MFEM_PERF_FUNCTION;
+
    const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                          Device::GetDeviceMemoryType() : pa_mt;
    // Assuming the same element type

fem/integ/bilininteg_hcurl_kernels.cpp

@@ -599,11 +599,13 @@ void PACurlCurlSetup3D(const int Q1D,
    });
 }
 
-void PACurlCurlAssembleDiagonal2D(const int D1D, const int Q1D, const bool,
-                                  const int NE, const Array<real_t> &bo,
-                                  const Array<real_t> &, const Array<real_t> &,
+void PACurlCurlAssembleDiagonal2D(const int D1D,
+                                  const int Q1D,
+                                  const int NE,
+                                  const Array<real_t> &bo,
                                   const Array<real_t> &gc,
-                                  const Vector &pa_data, Vector &diag)
+                                  const Vector &pa_data,
+                                  Vector &diag)
 {
    auto Bo = Reshape(bo.Read(), Q1D, D1D-1);
    auto Gc = Reshape(gc.Read(), Q1D, D1D);
@@ -651,11 +653,16 @@ void PACurlCurlAssembleDiagonal2D(const int D1D, const int Q1D, const bool,
    }); // end of element loop
 }
 
-void PACurlCurlApply2D(const int D1D, const int Q1D, const bool, const int NE,
-                       const Array<real_t> &bo, const Array<real_t> &,
-                       const Array<real_t> &bot, const Array<real_t> &,
-                       const Array<real_t> &gc, const Array<real_t> &gct,
-                       const Vector &pa_data, const Vector &x, Vector &y,
+void PACurlCurlApply2D(const int D1D,
+                       const int Q1D,
+                       const int NE,
+                       const Array<real_t> &bo,
+                       const Array<real_t> &bot,
+                       const Array<real_t> &gc,
+                       const Array<real_t> &gct,
+                       const Vector &pa_data,
+                       const Vector &x,
+                       Vector &y,
                        const bool useAbs)
 {
 

fem/integ/bilininteg_hcurl_kernels.hpp

@@ -24,7 +24,7 @@
 
 namespace mfem
 {
-/// \cond DO_NOT_DOCUMENT
+
 namespace internal
 {
 
@@ -426,11 +426,8 @@ void PACurlCurlSetup3D(const int Q1D,
 // PA H(curl) curl-curl Diagonal 2D kernel
 void PACurlCurlAssembleDiagonal2D(const int D1D,
                                   const int Q1D,
-                                  const bool symmetric, // unused
                                   const int NE,
                                   const Array<real_t> &bo,
-                                  const Array<real_t> &bc, // unused
-                                  const Array<real_t> &go, // unused
                                   const Array<real_t> &gc,
                                   const Vector &pa_data,
                                   Vector &diag);
@@ -834,12 +831,9 @@ inline void SmemPACurlCurlAssembleDiagonal3D(const int d1d,
 // PA H(curl) curl-curl Apply/AbsApply 2D kernel
 void PACurlCurlApply2D(const int D1D,
                        const int Q1D,
-                       const bool symmetric, // unused
                        const int NE,
                        const Array<real_t> &bo,
-                       const Array<real_t> &bc, // unused
                        const Array<real_t> &bot,
-                       const Array<real_t> &bct, // unused
                        const Array<real_t> &gc,
                        const Array<real_t> &gct,
                        const Vector &pa_data,
@@ -3164,49 +3158,6 @@ inline void SmemPAHcurlL2ApplyTranspose3D(const int d1d,
 
 } // namespace internal
 
-template<int DIM, int T_D1D, int T_Q1D>
-CurlCurlIntegrator::ApplyKernelType CurlCurlIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PACurlCurlApply2D;
-   }
-   else if constexpr (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlApply3D<T_D1D, T_Q1D>;
-      }
-      else
-      {
-         return internal::PACurlCurlApply3D;
-      }
-   }
-   MFEM_ABORT("");
-}
-
-template <int DIM, int T_D1D, int T_Q1D>
-CurlCurlIntegrator::DiagonalKernelType
-CurlCurlIntegrator::DiagonalPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PACurlCurlAssembleDiagonal2D;
-   }
-   else if constexpr (DIM == 3)
-   {
-      if (Device::Allows(Backend::DEVICE_MASK))
-      {
-         return internal::SmemPACurlCurlAssembleDiagonal3D<T_D1D, T_Q1D>;
-      }
-      else
-      {
-         return internal::PACurlCurlAssembleDiagonal3D;
-      }
-   }
-   MFEM_ABORT("");
-}
-/// \endcond DO_NOT_DOCUMENT
 } // namespace mfem
 
 #endif

fem/integ/bilininteg_hcurlhdiv_kernels.hpp

@@ -19,7 +19,6 @@
 #include "../../linalg/vector.hpp"
 #include "../bilininteg.hpp"
 
-/// \cond DO_NOT_DOCUMENT
 namespace mfem
 {
 
@@ -820,6 +819,4 @@ inline void PAHcurlHdivApplyTranspose3D(const int d1d,
 
 } // namespace mfem
 
-/// \endcond DO_NOT_DOCUMENT
-
 #endif

fem/integ/bilininteg_mass_kernels.cpp

@@ -17,8 +17,6 @@ namespace mfem
 MassIntegrator::Kernels::Kernels()
 {
    // 2D
-   // Q=P+1
-   MassIntegrator::AddSpecialization<2,1,1>();
    MassIntegrator::AddSpecialization<2,2,2>();
    MassIntegrator::AddSpecialization<2,3,3>();
    MassIntegrator::AddSpecialization<2,4,4>();
@@ -27,45 +25,17 @@ MassIntegrator::Kernels::Kernels()
    MassIntegrator::AddSpecialization<2,7,7>();
    MassIntegrator::AddSpecialization<2,8,8>();
    MassIntegrator::AddSpecialization<2,9,9>();
-   // Q=P+2
-   MassIntegrator::AddSpecialization<2,1,2>();
-   MassIntegrator::AddSpecialization<2,2,3>();
-   MassIntegrator::AddSpecialization<2,3,4>();
-   MassIntegrator::AddSpecialization<2,4,5>();
-   MassIntegrator::AddSpecialization<2,5,6>();
-   MassIntegrator::AddSpecialization<2,6,7>();
-   MassIntegrator::AddSpecialization<2,7,8>();
-   MassIntegrator::AddSpecialization<2,8,9>();
-   MassIntegrator::AddSpecialization<2,9,10>();
-   // others
-   MassIntegrator::AddSpecialization<2,2,4>();
-   MassIntegrator::AddSpecialization<2,3,6>();
-   MassIntegrator::AddSpecialization<2,4,6>();
    // 3D
-   // Q=P+1
-   MassIntegrator::AddSpecialization<3,1,1>();
    MassIntegrator::AddSpecialization<3,2,2>();
-   MassIntegrator::AddSpecialization<3,3,3>();
-   MassIntegrator::AddSpecialization<3,4,4>();
-   MassIntegrator::AddSpecialization<3,5,5>();
-   MassIntegrator::AddSpecialization<3,6,6>();
-   MassIntegrator::AddSpecialization<3,7,7>();
-   MassIntegrator::AddSpecialization<3,8,8>();
-   MassIntegrator::AddSpecialization<3,9,9>();
-   // Q=P+2
-   MassIntegrator::AddSpecialization<3,1,2>();
    MassIntegrator::AddSpecialization<3,2,3>();
    MassIntegrator::AddSpecialization<3,3,4>();
-   MassIntegrator::AddSpecialization<3,3,6>();
    MassIntegrator::AddSpecialization<3,4,5>();
+   MassIntegrator::AddSpecialization<3,4,6>();
    MassIntegrator::AddSpecialization<3,5,6>();
+   MassIntegrator::AddSpecialization<3,5,8>();
    MassIntegrator::AddSpecialization<3,6,7>();
    MassIntegrator::AddSpecialization<3,7,8>();
    MassIntegrator::AddSpecialization<3,8,9>();
-   // others
-   MassIntegrator::AddSpecialization<3,2,4>();
-   MassIntegrator::AddSpecialization<3,4,6>();
-   MassIntegrator::AddSpecialization<3,5,8>();
 }
 
 namespace internal

fem/integ/bilininteg_mass_kernels.hpp

@@ -1392,10 +1392,10 @@ using DiagonalKernelType = MassIntegrator::DiagonalKernelType;
 template<int DIM, int T_D1D, int T_Q1D>
 ApplyKernelType MassIntegrator::ApplyPAKernels::Kernel()
 {
-   if constexpr (DIM == 1) { return internal::PAMassApply1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassApply3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 1) { return internal::PAMassApply1D; }
+   else if (DIM == 2) { return internal::SmemPAMassApply2D<T_D1D,T_Q1D>; }
+   else if (DIM == 3) { return internal::SmemPAMassApply3D<T_D1D, T_Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
 inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
@@ -1410,10 +1410,10 @@ inline ApplyKernelType MassIntegrator::ApplyPAKernels::Fallback(
 template<int DIM, int T_D1D, int T_Q1D>
 DiagonalKernelType MassIntegrator::DiagonalPAKernels::Kernel()
 {
-   if constexpr (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
-   else if constexpr (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
-   else if constexpr (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<T_D1D, T_Q1D>; }
-   MFEM_ABORT("");
+   if (DIM == 1) { return internal::PAMassAssembleDiagonal1D; }
+   else if (DIM == 2) { return internal::SmemPAMassAssembleDiagonal2D<T_D1D,T_Q1D>; }
+   else if (DIM == 3) { return internal::SmemPAMassAssembleDiagonal3D<T_D1D, T_Q1D>; }
+   else { MFEM_ABORT(""); }
 }
 
 inline DiagonalKernelType MassIntegrator::DiagonalPAKernels::Fallback(

fem/integ/bilininteg_mass_pa.cpp

@@ -23,6 +23,8 @@ namespace mfem
 
 void MassIntegrator::AssemblePA(const FiniteElementSpace &fes)
 {
+   MFEM_PERF_FUNCTION;
+
    const MemoryType mt = (pa_mt == MemoryType::DEFAULT) ?
                          Device::GetDeviceMemoryType() : pa_mt;
 
@@ -139,6 +141,8 @@ void MassIntegrator::AssembleDiagonalPA(Vector &diag)
 
 void MassIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
+   MFEM_PERF_FUNCTION;
+
    if (DeviceCanUseCeed())
    {
       ceedOp->AddMult(x, y);

fem/integ/bilininteg_vecdiffusion_kernels.hpp

@@ -1,355 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP
-#define MFEM_BILININTEG_VECDIFFUSION_KERNELS_HPP
-
-#include "../../general/forall.hpp"
-#include "../bilininteg.hpp"
-#include "../ceed/integrators/diffusion/diffusion.hpp"
-#include "../gridfunc.hpp"
-#include "../qfunction.hpp"
-/// \cond DO_NOT_DOCUMENT
-namespace mfem
-{
-
-namespace internal
-{
-
-// PA Diffusion Apply 2D kernel
-template <int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0>
-static void
-PAVectorDiffusionApply2D(const int NE, const Array<real_t> &b,
-                         const Array<real_t> &g, const Array<real_t> &bt,
-                         const Array<real_t> &gt, const Vector &d_,
-                         const Vector &x_, Vector &y_, const int d1d = 0,
-                         const int q1d = 0, const int vdim = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   const int VDIM = T_VDIM ? T_VDIM : vdim;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto D = Reshape(d_.Read(), Q1D * Q1D, 3, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      const int VDIM = T_VDIM ? T_VDIM : vdim;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-
-      real_t grad[max_Q1D][max_Q1D][2];
-      for (int c = 0; c < VDIM; c++)
-      {
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               grad[qy][qx][0] = 0.0;
-               grad[qy][qx][1] = 0.0;
-            }
-         }
-         for (int dy = 0; dy < D1D; ++dy)
-         {
-            real_t gradX[max_Q1D][2];
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               gradX[qx][0] = 0.0;
-               gradX[qx][1] = 0.0;
-            }
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               const real_t s = x(dx, dy, c, e);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] += s * B(qx, dx);
-                  gradX[qx][1] += s * G(qx, dx);
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               const real_t wy = B(qy, dy);
-               const real_t wDy = G(qy, dy);
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qy][qx][0] += gradX[qx][1] * wy;
-                  grad[qy][qx][1] += gradX[qx][0] * wDy;
-               }
-            }
-         }
-         // Calculate Dxy, xDy in plane
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const int q = qx + qy * Q1D;
-               const real_t O11 = D(q, 0, e);
-               const real_t O12 = D(q, 1, e);
-               const real_t O22 = D(q, 2, e);
-               const real_t gradX = grad[qy][qx][0];
-               const real_t gradY = grad[qy][qx][1];
-               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
-               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
-            }
-         }
-         for (int qy = 0; qy < Q1D; ++qy)
-         {
-            real_t gradX[max_D1D][2];
-            for (int dx = 0; dx < D1D; ++dx)
-            {
-               gradX[dx][0] = 0.0;
-               gradX[dx][1] = 0.0;
-            }
-            for (int qx = 0; qx < Q1D; ++qx)
-            {
-               const real_t gX = grad[qy][qx][0];
-               const real_t gY = grad[qy][qx][1];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t wx = Bt(dx, qx);
-                  const real_t wDx = Gt(dx, qx);
-                  gradX[dx][0] += gX * wDx;
-                  gradX[dx][1] += gY * wx;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               const real_t wy = Bt(dy, qy);
-               const real_t wDy = Gt(dy, qy);
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  y(dx, dy, c, e) +=
-                     ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
-               }
-            }
-         }
-      }
-   });
-}
-
-// PA Diffusion Apply 3D kernel
-template <const int T_D1D = 0, const int T_Q1D = 0>
-static void
-PAVectorDiffusionApply3D(const int NE, const Array<real_t> &b,
-                         const Array<real_t> &g, const Array<real_t> &bt,
-                         const Array<real_t> &gt, const Vector &op_,
-                         const Vector &x_, Vector &y_, const int d1d = 0,
-                         const int q1d = 0, const int sdim = 0)
-{
-   const int D1D = T_D1D ? T_D1D : d1d;
-   const int Q1D = T_Q1D ? T_Q1D : q1d;
-   constexpr int VDIM = 3;
-   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
-   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
-   auto B = Reshape(b.Read(), Q1D, D1D);
-   auto G = Reshape(g.Read(), Q1D, D1D);
-   auto Bt = Reshape(bt.Read(), D1D, Q1D);
-   auto Gt = Reshape(gt.Read(), D1D, Q1D);
-   auto op = Reshape(op_.Read(), Q1D * Q1D * Q1D, 6, NE);
-   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
-   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
-   mfem::forall(NE, [=] MFEM_HOST_DEVICE(int e)
-   {
-      const int D1D = T_D1D ? T_D1D : d1d;
-      const int Q1D = T_Q1D ? T_Q1D : q1d;
-      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      for (int c = 0; c < VDIM; ++c)
-      {
-         real_t grad[max_Q1D][max_Q1D][max_Q1D][3];
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  grad[qz][qy][qx][0] = 0.0;
-                  grad[qz][qy][qx][1] = 0.0;
-                  grad[qz][qy][qx][2] = 0.0;
-               }
-            }
-         }
-         for (int dz = 0; dz < D1D; ++dz)
-         {
-            real_t gradXY[max_Q1D][max_Q1D][3];
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradXY[qy][qx][0] = 0.0;
-                  gradXY[qy][qx][1] = 0.0;
-                  gradXY[qy][qx][2] = 0.0;
-               }
-            }
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               real_t gradX[max_Q1D][2];
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  gradX[qx][0] = 0.0;
-                  gradX[qx][1] = 0.0;
-               }
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  const real_t s = x(dx, dy, dz, c, e);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     gradX[qx][0] += s * B(qx, dx);
-                     gradX[qx][1] += s * G(qx, dx);
-                  }
-               }
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  const real_t wy = B(qy, dy);
-                  const real_t wDy = G(qy, dy);
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     const real_t wx = gradX[qx][0];
-                     const real_t wDx = gradX[qx][1];
-                     gradXY[qy][qx][0] += wDx * wy;
-                     gradXY[qy][qx][1] += wx * wDy;
-                     gradXY[qy][qx][2] += wx * wy;
-                  }
-               }
-            }
-            for (int qz = 0; qz < Q1D; ++qz)
-            {
-               const real_t wz = B(qz, dz);
-               const real_t wDz = G(qz, dz);
-               for (int qy = 0; qy < Q1D; ++qy)
-               {
-                  for (int qx = 0; qx < Q1D; ++qx)
-                  {
-                     grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
-                     grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
-                     grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
-                  }
-               }
-            }
-         }
-         // Calculate Dxyz, xDyz, xyDz in plane
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const int q = qx + (qy + qz * Q1D) * Q1D;
-                  const real_t O11 = op(q, 0, e);
-                  const real_t O12 = op(q, 1, e);
-                  const real_t O13 = op(q, 2, e);
-                  const real_t O22 = op(q, 3, e);
-                  const real_t O23 = op(q, 4, e);
-                  const real_t O33 = op(q, 5, e);
-                  const real_t gradX = grad[qz][qy][qx][0];
-                  const real_t gradY = grad[qz][qy][qx][1];
-                  const real_t gradZ = grad[qz][qy][qx][2];
-                  grad[qz][qy][qx][0] =
-                     (O11 * gradX) + (O12 * gradY) + (O13 * gradZ);
-                  grad[qz][qy][qx][1] =
-                     (O12 * gradX) + (O22 * gradY) + (O23 * gradZ);
-                  grad[qz][qy][qx][2] =
-                     (O13 * gradX) + (O23 * gradY) + (O33 * gradZ);
-               }
-            }
-         }
-         for (int qz = 0; qz < Q1D; ++qz)
-         {
-            real_t gradXY[max_D1D][max_D1D][3];
-            for (int dy = 0; dy < D1D; ++dy)
-            {
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  gradXY[dy][dx][0] = 0;
-                  gradXY[dy][dx][1] = 0;
-                  gradXY[dy][dx][2] = 0;
-               }
-            }
-            for (int qy = 0; qy < Q1D; ++qy)
-            {
-               real_t gradX[max_D1D][3];
-               for (int dx = 0; dx < D1D; ++dx)
-               {
-                  gradX[dx][0] = 0;
-                  gradX[dx][1] = 0;
-                  gradX[dx][2] = 0;
-               }
-               for (int qx = 0; qx < Q1D; ++qx)
-               {
-                  const real_t gX = grad[qz][qy][qx][0];
-                  const real_t gY = grad[qz][qy][qx][1];
-                  const real_t gZ = grad[qz][qy][qx][2];
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     const real_t wx = Bt(dx, qx);
-                     const real_t wDx = Gt(dx, qx);
-                     gradX[dx][0] += gX * wDx;
-                     gradX[dx][1] += gY * wx;
-                     gradX[dx][2] += gZ * wx;
-                  }
-               }
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  const real_t wy = Bt(dy, qy);
-                  const real_t wDy = Gt(dy, qy);
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     gradXY[dy][dx][0] += gradX[dx][0] * wy;
-                     gradXY[dy][dx][1] += gradX[dx][1] * wDy;
-                     gradXY[dy][dx][2] += gradX[dx][2] * wy;
-                  }
-               }
-            }
-            for (int dz = 0; dz < D1D; ++dz)
-            {
-               const real_t wz = Bt(dz, qz);
-               const real_t wDz = Gt(dz, qz);
-               for (int dy = 0; dy < D1D; ++dy)
-               {
-                  for (int dx = 0; dx < D1D; ++dx)
-                  {
-                     y(dx, dy, dz, c, e) +=
-                        ((gradXY[dy][dx][0] * wz) + (gradXY[dy][dx][1] * wz) +
-                         (gradXY[dy][dx][2] * wDz));
-                  }
-               }
-            }
-         }
-      }
-   });
-}
-} // namespace internal
-
-template <int DIM, int VDIM, int T_D1D, int T_Q1D>
-VectorDiffusionIntegrator::ApplyKernelType
-VectorDiffusionIntegrator::ApplyPAKernels::Kernel()
-{
-   if constexpr (DIM == 2)
-   {
-      return internal::PAVectorDiffusionApply2D<T_D1D, T_Q1D, VDIM>;
-   }
-   else if constexpr (DIM == 3)
-   {
-      return internal::PAVectorDiffusionApply3D;
-   }
-   MFEM_ABORT("");
-}
-
-} // namespace mfem
-/// \endcond DO_NOT_DOCUMENT
-#endif

fem/integ/bilininteg_vecdiffusion_pa.cpp

@@ -15,58 +15,9 @@
 #include "../qfunction.hpp"
 #include "../ceed/integrators/diffusion/diffusion.hpp"
 
-#include "bilininteg_vecdiffusion_kernels.hpp"
-
 namespace mfem
 {
 
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(const IntegrationRule *ir)
-   : BilinearFormIntegrator(ir)
-{
-   static Kernels kernels;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q)
-   : VectorDiffusionIntegrator()
-{
-   Q = &q;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(int vector_dimension)
-   : VectorDiffusionIntegrator()
-{
-   vdim = vector_dimension;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q,
-                                                     const IntegrationRule *ir)
-   : VectorDiffusionIntegrator(ir)
-{
-   Q = &q;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(Coefficient &q,
-                                                     int vector_dimension)
-   : VectorDiffusionIntegrator()
-{
-   Q = &q;
-   vdim = vector_dimension;
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(VectorCoefficient &vq)
-   : VectorDiffusionIntegrator()
-{
-   VQ = &vq;
-   vdim = vq.GetVDim();
-}
-
-VectorDiffusionIntegrator::VectorDiffusionIntegrator(MatrixCoefficient &mq)
-   : VectorDiffusionIntegrator()
-{
-   MQ = &mq;
-   vdim = mq.GetVDim();
-}
-
 // PA Diffusion Assemble 2D kernel
 static void PAVectorDiffusionSetup2D(const int Q1D,
                                      const int NE,
@@ -474,6 +425,322 @@ void VectorDiffusionIntegrator::AssembleDiagonalPA(Vector &diag)
    }
 }
 
+// PA Diffusion Apply 2D kernel
+template<int T_D1D = 0, int T_Q1D = 0, int T_VDIM = 0> static
+void PAVectorDiffusionApply2D(const int NE,
+                              const Array<real_t> &b,
+                              const Array<real_t> &g,
+                              const Array<real_t> &bt,
+                              const Array<real_t> &gt,
+                              const Vector &d_,
+                              const Vector &x_,
+                              Vector &y_,
+                              const int d1d = 0,
+                              const int q1d = 0,
+                              const int vdim = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   const int VDIM = T_VDIM ? T_VDIM : vdim;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Gt = Reshape(gt.Read(), D1D, Q1D);
+   auto D = Reshape(d_.Read(), Q1D*Q1D, 3, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      const int VDIM = T_VDIM ? T_VDIM : vdim;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+
+      real_t grad[max_Q1D][max_Q1D][2];
+      for (int c = 0; c < VDIM; c++)
+      {
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               grad[qy][qx][0] = 0.0;
+               grad[qy][qx][1] = 0.0;
+            }
+         }
+         for (int dy = 0; dy < D1D; ++dy)
+         {
+            real_t gradX[max_Q1D][2];
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               gradX[qx][0] = 0.0;
+               gradX[qx][1] = 0.0;
+            }
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               const real_t s = x(dx,dy,c,e);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] += s * B(qx,dx);
+                  gradX[qx][1] += s * G(qx,dx);
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               const real_t wy  = B(qy,dy);
+               const real_t wDy = G(qy,dy);
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qy][qx][0] += gradX[qx][1] * wy;
+                  grad[qy][qx][1] += gradX[qx][0] * wDy;
+               }
+            }
+         }
+         // Calculate Dxy, xDy in plane
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const int q = qx + qy * Q1D;
+               const real_t O11 = D(q,0,e);
+               const real_t O12 = D(q,1,e);
+               const real_t O22 = D(q,2,e);
+               const real_t gradX = grad[qy][qx][0];
+               const real_t gradY = grad[qy][qx][1];
+               grad[qy][qx][0] = (O11 * gradX) + (O12 * gradY);
+               grad[qy][qx][1] = (O12 * gradX) + (O22 * gradY);
+            }
+         }
+         for (int qy = 0; qy < Q1D; ++qy)
+         {
+            real_t gradX[max_D1D][2];
+            for (int dx = 0; dx < D1D; ++dx)
+            {
+               gradX[dx][0] = 0.0;
+               gradX[dx][1] = 0.0;
+            }
+            for (int qx = 0; qx < Q1D; ++qx)
+            {
+               const real_t gX = grad[qy][qx][0];
+               const real_t gY = grad[qy][qx][1];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t wx  = Bt(dx,qx);
+                  const real_t wDx = Gt(dx,qx);
+                  gradX[dx][0] += gX * wDx;
+                  gradX[dx][1] += gY * wx;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               const real_t wy  = Bt(dy,qy);
+               const real_t wDy = Gt(dy,qy);
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  y(dx,dy,c,e) += ((gradX[dx][0] * wy) + (gradX[dx][1] * wDy));
+               }
+            }
+         }
+      }
+   });
+}
+
+// PA Diffusion Apply 3D kernel
+template<const int T_D1D = 0,
+         const int T_Q1D = 0> static
+void PAVectorDiffusionApply3D(const int NE,
+                              const Array<real_t> &b,
+                              const Array<real_t> &g,
+                              const Array<real_t> &bt,
+                              const Array<real_t> &gt,
+                              const Vector &op_,
+                              const Vector &x_,
+                              Vector &y_,
+                              int d1d = 0, int q1d = 0)
+{
+   const int D1D = T_D1D ? T_D1D : d1d;
+   const int Q1D = T_Q1D ? T_Q1D : q1d;
+   constexpr int VDIM = 3;
+   MFEM_VERIFY(D1D <= DeviceDofQuadLimits::Get().MAX_D1D, "");
+   MFEM_VERIFY(Q1D <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
+   auto B = Reshape(b.Read(), Q1D, D1D);
+   auto G = Reshape(g.Read(), Q1D, D1D);
+   auto Bt = Reshape(bt.Read(), D1D, Q1D);
+   auto Gt = Reshape(gt.Read(), D1D, Q1D);
+   auto op = Reshape(op_.Read(), Q1D*Q1D*Q1D, 6, NE);
+   auto x = Reshape(x_.Read(), D1D, D1D, D1D, VDIM, NE);
+   auto y = Reshape(y_.ReadWrite(), D1D, D1D, D1D, VDIM, NE);
+   mfem::forall(NE, [=] MFEM_HOST_DEVICE (int e)
+   {
+      const int D1D = T_D1D ? T_D1D : d1d;
+      const int Q1D = T_Q1D ? T_Q1D : q1d;
+      constexpr int max_D1D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int max_Q1D = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      for (int c = 0; c < VDIM; ++ c)
+      {
+         real_t grad[max_Q1D][max_Q1D][max_Q1D][3];
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  grad[qz][qy][qx][0] = 0.0;
+                  grad[qz][qy][qx][1] = 0.0;
+                  grad[qz][qy][qx][2] = 0.0;
+               }
+            }
+         }
+         for (int dz = 0; dz < D1D; ++dz)
+         {
+            real_t gradXY[max_Q1D][max_Q1D][3];
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradXY[qy][qx][0] = 0.0;
+                  gradXY[qy][qx][1] = 0.0;
+                  gradXY[qy][qx][2] = 0.0;
+               }
+            }
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               real_t gradX[max_Q1D][2];
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  gradX[qx][0] = 0.0;
+                  gradX[qx][1] = 0.0;
+               }
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  const real_t s = x(dx,dy,dz,c,e);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     gradX[qx][0] += s * B(qx,dx);
+                     gradX[qx][1] += s * G(qx,dx);
+                  }
+               }
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  const real_t wy  = B(qy,dy);
+                  const real_t wDy = G(qy,dy);
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     const real_t wx  = gradX[qx][0];
+                     const real_t wDx = gradX[qx][1];
+                     gradXY[qy][qx][0] += wDx * wy;
+                     gradXY[qy][qx][1] += wx  * wDy;
+                     gradXY[qy][qx][2] += wx  * wy;
+                  }
+               }
+            }
+            for (int qz = 0; qz < Q1D; ++qz)
+            {
+               const real_t wz  = B(qz,dz);
+               const real_t wDz = G(qz,dz);
+               for (int qy = 0; qy < Q1D; ++qy)
+               {
+                  for (int qx = 0; qx < Q1D; ++qx)
+                  {
+                     grad[qz][qy][qx][0] += gradXY[qy][qx][0] * wz;
+                     grad[qz][qy][qx][1] += gradXY[qy][qx][1] * wz;
+                     grad[qz][qy][qx][2] += gradXY[qy][qx][2] * wDz;
+                  }
+               }
+            }
+         }
+         // Calculate Dxyz, xDyz, xyDz in plane
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const int q = qx + (qy + qz * Q1D) * Q1D;
+                  const real_t O11 = op(q,0,e);
+                  const real_t O12 = op(q,1,e);
+                  const real_t O13 = op(q,2,e);
+                  const real_t O22 = op(q,3,e);
+                  const real_t O23 = op(q,4,e);
+                  const real_t O33 = op(q,5,e);
+                  const real_t gradX = grad[qz][qy][qx][0];
+                  const real_t gradY = grad[qz][qy][qx][1];
+                  const real_t gradZ = grad[qz][qy][qx][2];
+                  grad[qz][qy][qx][0] = (O11*gradX)+(O12*gradY)+(O13*gradZ);
+                  grad[qz][qy][qx][1] = (O12*gradX)+(O22*gradY)+(O23*gradZ);
+                  grad[qz][qy][qx][2] = (O13*gradX)+(O23*gradY)+(O33*gradZ);
+               }
+            }
+         }
+         for (int qz = 0; qz < Q1D; ++qz)
+         {
+            real_t gradXY[max_D1D][max_D1D][3];
+            for (int dy = 0; dy < D1D; ++dy)
+            {
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradXY[dy][dx][0] = 0;
+                  gradXY[dy][dx][1] = 0;
+                  gradXY[dy][dx][2] = 0;
+               }
+            }
+            for (int qy = 0; qy < Q1D; ++qy)
+            {
+               real_t gradX[max_D1D][3];
+               for (int dx = 0; dx < D1D; ++dx)
+               {
+                  gradX[dx][0] = 0;
+                  gradX[dx][1] = 0;
+                  gradX[dx][2] = 0;
+               }
+               for (int qx = 0; qx < Q1D; ++qx)
+               {
+                  const real_t gX = grad[qz][qy][qx][0];
+                  const real_t gY = grad[qz][qy][qx][1];
+                  const real_t gZ = grad[qz][qy][qx][2];
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     const real_t wx  = Bt(dx,qx);
+                     const real_t wDx = Gt(dx,qx);
+                     gradX[dx][0] += gX * wDx;
+                     gradX[dx][1] += gY * wx;
+                     gradX[dx][2] += gZ * wx;
+                  }
+               }
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  const real_t wy  = Bt(dy,qy);
+                  const real_t wDy = Gt(dy,qy);
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     gradXY[dy][dx][0] += gradX[dx][0] * wy;
+                     gradXY[dy][dx][1] += gradX[dx][1] * wDy;
+                     gradXY[dy][dx][2] += gradX[dx][2] * wy;
+                  }
+               }
+            }
+            for (int dz = 0; dz < D1D; ++dz)
+            {
+               const real_t wz  = Bt(dz,qz);
+               const real_t wDz = Gt(dz,qz);
+               for (int dy = 0; dy < D1D; ++dy)
+               {
+                  for (int dx = 0; dx < D1D; ++dx)
+                  {
+                     y(dx,dy,dz,c,e) +=
+                        ((gradXY[dy][dx][0] * wz) +
+                         (gradXY[dy][dx][1] * wz) +
+                         (gradXY[dy][dx][2] * wDz));
+                  }
+               }
+            }
+         }
+      }
+   });
+}
+
 // PA Diffusion Apply kernel
 void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
 {
@@ -490,29 +757,27 @@ void VectorDiffusionIntegrator::AddMultPA(const Vector &x, Vector &y) const
       const Array<real_t> &Bt = maps->Bt;
       const Array<real_t> &Gt = maps->Gt;
       const Vector &D = pa_data;
-      ApplyPAKernels::Run(dim, sdim, D1D, Q1D, ne, B, G, Bt, Gt, D, x, y, D1D,
-                          Q1D, sdim);
+
+      if (dim == 2 && sdim == 3)
+      {
+         switch ((dofs1D << 4 ) | quad1D)
+         {
+            case 0x22: return PAVectorDiffusionApply2D<2,2,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x33: return PAVectorDiffusionApply2D<3,3,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x44: return PAVectorDiffusionApply2D<4,4,3>(ne,B,G,Bt,Gt,D,x,y);
+            case 0x55: return PAVectorDiffusionApply2D<5,5,3>(ne,B,G,Bt,Gt,D,x,y);
+            default:
+               return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim);
+         }
+      }
+      if (dim == 2 && sdim == 2)
+      { return PAVectorDiffusionApply2D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D,sdim); }
+
+      if (dim == 3 && sdim == 3)
+      { return PAVectorDiffusionApply3D(ne,B,G,Bt,Gt,D,x,y,D1D,Q1D); }
+
+      MFEM_ABORT("Unknown kernel.");
    }
 }
 
-/// \cond DO_NOT_DOCUMENT
-
-VectorDiffusionIntegrator::ApplyKernelType
-VectorDiffusionIntegrator::ApplyPAKernels::Fallback(int DIM, int, int, int)
-{
-   if (DIM == 2) { return internal::PAVectorDiffusionApply2D; }
-   else if (DIM == 3) { return internal::PAVectorDiffusionApply3D; }
-   else { MFEM_ABORT(""); }
-}
-
-VectorDiffusionIntegrator::Kernels::Kernels()
-{
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 2, 2>();
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 3, 3>();
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 4, 4>();
-   VectorDiffusionIntegrator::AddSpecialization<2, 3, 5, 5>();
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/lininteg_domain.cpp

@@ -9,19 +9,183 @@
 // terms of the BSD-3 license. We welcome feedback and contributions, see file
 // CONTRIBUTING.md for details.
 
-#include "../../fem/kernels.hpp"
 #include "../../general/forall.hpp"
+#include "../../fem/kernels.hpp"
 #include "../fem.hpp"
 
-#include "lininteg_domain_kernels.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-
 namespace mfem
 {
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void DLFEvalAssemble2D(const int vdim, const int ne, const int d,
+                              const int q,
+                              const int map_type, const int *markers, const real_t *b,
+                              const real_t *detj, const real_t *weights,
+                              const Vector &coeff, real_t *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, ne);
+   const auto B = Reshape(b, q, d);
+   const auto DETJ = Reshape(detj, q, q, ne);
+   const auto W = Reshape(weights, q, q);
+   const bool cst = coeff.Size() == vdim;
+   const auto C = cst ? Reshape(F,vdim,1,1,1) : Reshape(F,vdim,q,q,ne);
+   auto Y = Reshape(y, d,d, vdim, ne);
+
+   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE (int e)
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+
+      MFEM_SHARED real_t sBt[Q*D];
+      MFEM_SHARED real_t sQQ[Q*Q];
+      MFEM_SHARED real_t sQD[Q*D];
+
+      const DeviceMatrix Bt(sBt, d, q);
+      kernels::internal::LoadB<D,Q>(d, q, B, sBt);
+
+      const DeviceMatrix QQ(sQQ, q, q);
+      const DeviceMatrix QD(sQD, q, d);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const real_t cst_val = C(c,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               const real_t detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,e) : 1.0;
+               const real_t coeff_val = cst ? cst_val : C(c,x,y,e);
+               QQ(y,x) = W(x,y) * coeff_val * detJ;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qy,y,q)
+         {
+            MFEM_FOREACH_THREAD(dx,x,d)
+            {
+               real_t u = 0.0;
+               for (int qx = 0; qx < q; ++qx) { u += QQ(qy,qx) * Bt(dx,qx); }
+               QD(qy,dx) = u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dy,y,d)
+         {
+            MFEM_FOREACH_THREAD(dx,x,d)
+            {
+               real_t u = 0.0;
+               for (int qy = 0; qy < q; ++qy) { u += QD(qy,dx) * Bt(dy,qy); }
+               Y(dx,dy,c,e) += u;
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
+template<int T_D1D = 0, int T_Q1D = 0>
+static void DLFEvalAssemble3D(const int vdim, const int ne, const int d,
+                              const int q,
+                              const int map_type, const int *markers, const real_t *b,
+                              const real_t *detj, const real_t *weights,
+                              const Vector &coeff, real_t *y)
+{
+   const auto F = coeff.Read();
+   const auto M = Reshape(markers, ne);
+   const auto B = Reshape(b, q,d);
+   const auto DETJ = Reshape(detj, q, q, q, ne);
+   const auto W = Reshape(weights, q,q,q);
+   const bool cst_coeff = coeff.Size() == vdim;
+   const auto C = cst_coeff ? Reshape(F,vdim,1,1,1,1):Reshape(F,vdim,q,q,q,ne);
+
+   auto Y = Reshape(y, d,d,d, vdim, ne);
+
+   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE (int e)
+   {
+      if (M(e) == 0) { return; } // ignore
+
+      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
+      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
+      constexpr int MQD = (Q >= D) ? Q : D;
+
+      real_t u[D];
+
+      MFEM_SHARED real_t sBt[Q*D];
+      const DeviceMatrix Bt(sBt, d,q);
+      kernels::internal::LoadB<D,Q>(d,q,B,sBt);
+
+      MFEM_SHARED real_t sQQQ[MQD*MQD*MQD];
+      const DeviceCube QQQ(sQQQ, MQD, MQD, MQD);
+
+      for (int c = 0; c < vdim; ++c)
+      {
+         const real_t cst_val = C(c,0,0,0,0);
+         MFEM_FOREACH_THREAD(x,x,q)
+         {
+            MFEM_FOREACH_THREAD(y,y,q)
+            {
+               for (int z = 0; z < q; ++z)
+               {
+                  const real_t detJ = (map_type == FiniteElement::VALUE) ? DETJ(x,y,z,e) : 1.0;
+                  const real_t coeff_val = cst_coeff ? cst_val : C(c,x,y,z,e);
+                  QQQ(z,y,x) = W(x,y,z) * coeff_val * detJ;
+               }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(qx,x,q)
+         {
+            MFEM_FOREACH_THREAD(qy,y,q)
+            {
+               for (int dz = 0; dz < d; ++dz) { u[dz] = 0.0; }
+               for (int qz = 0; qz < q; ++qz)
+               {
+                  const real_t ZYX = QQQ(qz,qy,qx);
+                  for (int dz = 0; dz < d; ++dz) { u[dz] += ZYX * Bt(dz,qz); }
+               }
+               for (int dz = 0; dz < d; ++dz) { QQQ(dz,qy,qx) = u[dz]; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,y,d)
+         {
+            MFEM_FOREACH_THREAD(qx,x,q)
+            {
+               for (int dy = 0; dy < d; ++dy) { u[dy] = 0.0; }
+               for (int qy = 0; qy < q; ++qy)
+               {
+                  const real_t zYX = QQQ(dz,qy,qx);
+                  for (int dy = 0; dy < d; ++dy) { u[dy] += zYX * Bt(dy,qy); }
+               }
+               for (int dy = 0; dy < d; ++dy) { QQQ(dz,dy,qx) = u[dy]; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+         MFEM_FOREACH_THREAD(dz,y,d)
+         {
+            MFEM_FOREACH_THREAD(dy,x,d)
+            {
+               for (int dx = 0; dx < d; ++dx) { u[dx] = 0.0; }
+               for (int qx = 0; qx < q; ++qx)
+               {
+                  const real_t zyX = QQQ(dz,dy,qx);
+                  for (int dx = 0; dx < d; ++dx) { u[dx] += zyX * Bt(dx,qx); }
+               }
+               for (int dx = 0; dx < d; ++dx) { Y(dx,dy,dz,c,e) += u[dx]; }
+            }
+         }
+         MFEM_SYNC_THREAD;
+      }
+   });
+}
+
 static void DLFEvalAssemble(const FiniteElementSpace &fes,
                             const IntegrationRule *ir,
-                            const Array<int> &markers, const Vector &coeff,
+                            const Array<int> &markers,
+                            const Vector &coeff,
                             Vector &y)
 {
    Mesh *mesh = fes.GetMesh();
@@ -33,21 +197,53 @@ static void DLFEvalAssemble(const FiniteElementSpace &fes,
    constexpr int flags = GeometricFactors::DETERMINANTS;
    const GeometricFactors *geom = mesh->GetGeometricFactors(*ir, flags, mt);
    const int map_type = fes.GetTypicalFE()->GetMapType();
+   decltype(&DLFEvalAssemble2D<>) ker =
+      dim == 2 ? DLFEvalAssemble2D<> : DLFEvalAssemble3D<>;
+
+   if (dim==2)
+   {
+      if (d==1 && q==1) { ker=DLFEvalAssemble2D<1,1>; }
+      if (d==2 && q==2) { ker=DLFEvalAssemble2D<2,2>; }
+      if (d==3 && q==3) { ker=DLFEvalAssemble2D<3,3>; }
+      if (d==4 && q==4) { ker=DLFEvalAssemble2D<4,4>; }
+      if (d==5 && q==5) { ker=DLFEvalAssemble2D<5,5>; }
+      if (d==2 && q==3) { ker=DLFEvalAssemble2D<2,3>; }
+      if (d==3 && q==4) { ker=DLFEvalAssemble2D<3,4>; }
+      if (d==4 && q==5) { ker=DLFEvalAssemble2D<4,5>; }
+      if (d==5 && q==6) { ker=DLFEvalAssemble2D<5,6>; }
+   }
+
+   if (dim==3)
+   {
+      if (d==1 && q==1) { ker=DLFEvalAssemble3D<1,1>; }
+      if (d==2 && q==2) { ker=DLFEvalAssemble3D<2,2>; }
+      if (d==3 && q==3) { ker=DLFEvalAssemble3D<3,3>; }
+      if (d==4 && q==4) { ker=DLFEvalAssemble3D<4,4>; }
+      if (d==5 && q==5) { ker=DLFEvalAssemble3D<5,5>; }
+      if (d==2 && q==3) { ker=DLFEvalAssemble3D<2,3>; }
+      if (d==3 && q==4) { ker=DLFEvalAssemble3D<3,4>; }
+      if (d==4 && q==5) { ker=DLFEvalAssemble3D<4,5>; }
+      if (d==5 && q==6) { ker=DLFEvalAssemble3D<5,6>; }
+   }
+
+   MFEM_VERIFY(ker, "No kernel ndof " << d << " nqpt " << q);
 
    const int vdim = fes.GetVDim();
    const int ne = fes.GetMesh()->GetNE();
-   const real_t *B = maps.B.Read();
    const int *M = markers.Read();
+   const real_t *B = maps.B.Read();
    const real_t *detJ = geom->detJ.Read();
    const real_t *W = ir->GetWeights().Read();
    real_t *Y = y.ReadWrite();
-   DomainLFIntegrator::AssembleKernels::Run(dim, d, q, vdim, ne, d, q, map_type,
-                                            M, B, detJ, W, coeff, Y);
+   ker(vdim, ne, d, q, map_type, M, B, detJ, W, coeff, Y);
 }
 
 void DomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
-                                        const Array<int> &markers, Vector &b)
+                                        const Array<int> &markers,
+                                        Vector &b)
 {
+   MFEM_PERF_FUNCTION;
+
    const FiniteElement &fe = *fes.GetTypicalFE();
    const int qorder = oa * fe.GetOrder() + ob;
    const Geometry::Type gtype = fe.GetGeomType();
@@ -72,49 +268,4 @@ void VectorDomainLFIntegrator::AssembleDevice(const FiniteElementSpace &fes,
    DLFEvalAssemble(fes, ir, markers, coeff, b);
 }
 
-DomainLFIntegrator::AssembleKernelType
-DomainLFIntegrator::AssembleKernels::Fallback(int DIM, int, int)
-{
-   switch (DIM)
-   {
-      case 1:
-         return DLFEvalAssemble1D<0, 0>;
-      case 2:
-         return DLFEvalAssemble2D<0, 0>;
-      case 3:
-         return DLFEvalAssemble3D<0, 0>;
-   }
-   MFEM_ABORT("");
-}
-
-DomainLFIntegrator::Kernels::Kernels()
-{
-   // 2D
-   // Q = P+1
-   DomainLFIntegrator::AddSpecialization<2, 1, 1>();
-   DomainLFIntegrator::AddSpecialization<2, 2, 2>();
-   DomainLFIntegrator::AddSpecialization<2, 3, 3>();
-   DomainLFIntegrator::AddSpecialization<2, 4, 4>();
-   DomainLFIntegrator::AddSpecialization<2, 5, 5>();
-   // Q = P+2
-   DomainLFIntegrator::AddSpecialization<2, 2, 3>();
-   DomainLFIntegrator::AddSpecialization<2, 3, 4>();
-   DomainLFIntegrator::AddSpecialization<2, 4, 5>();
-   DomainLFIntegrator::AddSpecialization<2, 5, 6>();
-   // 3D
-   // Q = P+1
-   DomainLFIntegrator::AddSpecialization<3, 1, 1>();
-   DomainLFIntegrator::AddSpecialization<3, 2, 2>();
-   DomainLFIntegrator::AddSpecialization<3, 3, 3>();
-   DomainLFIntegrator::AddSpecialization<3, 4, 4>();
-   DomainLFIntegrator::AddSpecialization<3, 5, 5>();
-   // Q = P+2
-   DomainLFIntegrator::AddSpecialization<3, 2, 3>();
-   DomainLFIntegrator::AddSpecialization<3, 3, 4>();
-   DomainLFIntegrator::AddSpecialization<3, 4, 5>();
-   DomainLFIntegrator::AddSpecialization<3, 5, 6>();
-}
-
-/// \endcond DO_NOT_DOCUMENT
-
 } // namespace mfem

fem/integ/lininteg_domain_kernels.hpp

@@ -1,318 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_LININTEG_DOMAIN_KERNELS_HPP
-#define MFEM_LININTEG_DOMAIN_KERNELS_HPP
-
-#include "../../fem/kernels.hpp"
-#include "../../general/forall.hpp"
-#include "../fem.hpp"
-
-/// \cond DO_NOT_DOCUMENT
-
-namespace mfem
-{
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void DLFEvalAssemble1D(const int vdim, const int ne, const int d,
-                              const int q, const int map_type,
-                              const int *markers, const real_t *b,
-                              const real_t *detj, const real_t *weights,
-                              const Vector &coeff, real_t *y)
-{
-   {
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      MFEM_VERIFY(q <= Q, "");
-      MFEM_VERIFY(d <= D, "");
-   }
-
-   const auto F = coeff.Read();
-   const auto B = Reshape(b, q, d);
-   const auto DETJ = Reshape(detj, q, ne);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F, vdim, 1, 1) : Reshape(F, vdim, q, ne);
-   auto Y = Reshape(y, d, vdim, ne);
-
-   mfem::forall_2D(ne, d, 1, [=] MFEM_HOST_DEVICE(int e)
-   {
-      if (markers[e] == 0)
-      {
-         return;
-      } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t sBt[Q * D];
-      const DeviceMatrix Bt(sBt, d, q);
-      kernels::internal::LoadB<D, Q>(d, q, B, sBt);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const real_t cst_val = C(c, 0, 0);
-         MFEM_FOREACH_THREAD(dx, x, d)
-         {
-            real_t u = 0;
-            for (int qx = 0; qx < q; ++qx)
-            {
-               const real_t detJ =
-                  (map_type == FiniteElement::VALUE) ? DETJ(qx, e) : 1.0;
-               const real_t coeff_val = cst ? cst_val : C(c, qx, e);
-               u += weights[qx] * coeff_val * detJ * Bt(dx, qx);
-            }
-            Y(dx, c, e) += u;
-         }
-      }
-   });
-}
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void DLFEvalAssemble2D(const int vdim, const int ne, const int d,
-                              const int q, const int map_type,
-                              const int *markers, const real_t *b,
-                              const real_t *detj, const real_t *weights,
-                              const Vector &coeff, real_t *y)
-{
-   {
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      MFEM_VERIFY(q <= Q, "");
-      MFEM_VERIFY(d <= D, "");
-   }
-
-   const auto F = coeff.Read();
-   const auto B = Reshape(b, q, d);
-   const auto DETJ = Reshape(detj, q, q, ne);
-   const auto W = Reshape(weights, q, q);
-   const bool cst = coeff.Size() == vdim;
-   const auto C = cst ? Reshape(F, vdim, 1, 1, 1) : Reshape(F, vdim, q, q, ne);
-   auto Y = Reshape(y, d, d, vdim, ne);
-
-   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE(int e)
-   {
-      if (markers[e] == 0)
-      {
-         return;
-      } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-
-      MFEM_SHARED real_t sBt[Q * D];
-      MFEM_SHARED real_t sQQ[Q * Q];
-      MFEM_SHARED real_t sQD[Q * D];
-
-      const DeviceMatrix Bt(sBt, d, q);
-      kernels::internal::LoadB<D, Q>(d, q, B, sBt);
-
-      const DeviceMatrix QQ(sQQ, q, q);
-      const DeviceMatrix QD(sQD, q, d);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const real_t cst_val = C(c, 0, 0, 0);
-         MFEM_FOREACH_THREAD(x, x, q)
-         {
-            MFEM_FOREACH_THREAD(y, y, q)
-            {
-               const real_t detJ =
-                  (map_type == FiniteElement::VALUE) ? DETJ(x, y, e) : 1.0;
-               const real_t coeff_val = cst ? cst_val : C(c, x, y, e);
-               QQ(y, x) = W(x, y) * coeff_val * detJ;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qy, y, q)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d)
-            {
-               real_t u = 0.0;
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  u += QQ(qy, qx) * Bt(dx, qx);
-               }
-               QD(qy, dx) = u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dy, y, d)
-         {
-            MFEM_FOREACH_THREAD(dx, x, d)
-            {
-               real_t u = 0.0;
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  u += QD(qy, dx) * Bt(dy, qy);
-               }
-               Y(dx, dy, c, e) += u;
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template <int T_D1D = 0, int T_Q1D = 0>
-static void DLFEvalAssemble3D(const int vdim, const int ne, const int d,
-                              const int q, const int map_type,
-                              const int* markers, const real_t *b,
-                              const real_t *detj, const real_t *weights,
-                              const Vector &coeff, real_t *y)
-{
-   {
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      MFEM_VERIFY(q <= Q, "");
-      MFEM_VERIFY(d <= D, "");
-   }
-
-   const auto F = coeff.Read();
-   const auto B = Reshape(b, q, d);
-   const auto DETJ = Reshape(detj, q, q, q, ne);
-   const auto W = Reshape(weights, q, q, q);
-   const bool cst_coeff = coeff.Size() == vdim;
-   const auto C =
-      cst_coeff ? Reshape(F, vdim, 1, 1, 1, 1) : Reshape(F, vdim, q, q, q, ne);
-
-   auto Y = Reshape(y, d, d, d, vdim, ne);
-
-   mfem::forall_2D(ne, q, q, [=] MFEM_HOST_DEVICE(int e)
-   {
-      if (markers[e] == 0)
-      {
-         return;
-      } // ignore
-
-      constexpr int Q = T_Q1D ? T_Q1D : DofQuadLimits::MAX_Q1D;
-      constexpr int D = T_D1D ? T_D1D : DofQuadLimits::MAX_D1D;
-      constexpr int MQD = (Q >= D) ? Q : D;
-
-      real_t u[D];
-
-      MFEM_SHARED real_t sBt[Q * D];
-      const DeviceMatrix Bt(sBt, d, q);
-      kernels::internal::LoadB<D, Q>(d, q, B, sBt);
-
-      MFEM_SHARED real_t sQQQ[MQD * MQD * MQD];
-      const DeviceCube QQQ(sQQQ, MQD, MQD, MQD);
-
-      for (int c = 0; c < vdim; ++c)
-      {
-         const real_t cst_val = C(c, 0, 0, 0, 0);
-         MFEM_FOREACH_THREAD(x, x, q)
-         {
-            MFEM_FOREACH_THREAD(y, y, q)
-            {
-               for (int z = 0; z < q; ++z)
-               {
-                  const real_t detJ = (map_type == FiniteElement::VALUE)
-                                      ? DETJ(x, y, z, e)
-                                      : 1.0;
-                  const real_t coeff_val =
-                     cst_coeff ? cst_val : C(c, x, y, z, e);
-                  QQQ(z, y, x) = W(x, y, z) * coeff_val * detJ;
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(qx, x, q)
-         {
-            MFEM_FOREACH_THREAD(qy, y, q)
-            {
-               for (int dz = 0; dz < d; ++dz)
-               {
-                  u[dz] = 0.0;
-               }
-               for (int qz = 0; qz < q; ++qz)
-               {
-                  const real_t ZYX = QQQ(qz, qy, qx);
-                  for (int dz = 0; dz < d; ++dz)
-                  {
-                     u[dz] += ZYX * Bt(dz, qz);
-                  }
-               }
-               for (int dz = 0; dz < d; ++dz)
-               {
-                  QQQ(dz, qy, qx) = u[dz];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz, y, d)
-         {
-            MFEM_FOREACH_THREAD(qx, x, q)
-            {
-               for (int dy = 0; dy < d; ++dy)
-               {
-                  u[dy] = 0.0;
-               }
-               for (int qy = 0; qy < q; ++qy)
-               {
-                  const real_t zYX = QQQ(dz, qy, qx);
-                  for (int dy = 0; dy < d; ++dy)
-                  {
-                     u[dy] += zYX * Bt(dy, qy);
-                  }
-               }
-               for (int dy = 0; dy < d; ++dy)
-               {
-                  QQQ(dz, dy, qx) = u[dy];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-         MFEM_FOREACH_THREAD(dz, y, d)
-         {
-            MFEM_FOREACH_THREAD(dy, x, d)
-            {
-               for (int dx = 0; dx < d; ++dx)
-               {
-                  u[dx] = 0.0;
-               }
-               for (int qx = 0; qx < q; ++qx)
-               {
-                  const real_t zyX = QQQ(dz, dy, qx);
-                  for (int dx = 0; dx < d; ++dx)
-                  {
-                     u[dx] += zyX * Bt(dx, qx);
-                  }
-               }
-               for (int dx = 0; dx < d; ++dx)
-               {
-                  Y(dx, dy, dz, c, e) += u[dx];
-               }
-            }
-         }
-         MFEM_SYNC_THREAD;
-      }
-   });
-}
-
-template <int DIM, int T_D1D, int T_Q1D>
-DomainLFIntegrator::AssembleKernelType
-DomainLFIntegrator::AssembleKernels::Kernel()
-{
-   switch (DIM)
-   {
-      case 1:
-         return DLFEvalAssemble1D<T_D1D, T_Q1D>;
-      case 2:
-         return DLFEvalAssemble2D<T_D1D, T_Q1D>;
-      case 3:
-         return DLFEvalAssemble3D<T_D1D, T_Q1D>;
-   }
-   MFEM_ABORT("");
-}
-/// \endcond DO_NOT_DOCUMENT
-
-} // namespace mfem
-#endif

fem/intrules.cpp

@@ -947,7 +947,6 @@ int Quadrature1D::CheckOpen(int type)
       case OpenUniform:
       case ClosedUniform:
       case OpenHalfUniform:
-      case ClosedGL:
          return type; // all types can work as open
       default:
          return Invalid;

fem/kernel_dispatch.hpp

@@ -78,7 +78,7 @@ namespace mfem
   public:                                                                      \
     const char *kernel_name = MFEM_KERNEL_NAME(KernelName);                    \
     using KernelSignature = KernelType;                                        \
-    template <MFEM_PARAM_LIST P3> static KernelSignature Kernel();             \
+    template <MFEM_PARAM_LIST P3> static MFEM_EXPORT KernelSignature Kernel(); \
     static MFEM_EXPORT KernelSignature Fallback(MFEM_PARAM_LIST P1);           \
     static MFEM_EXPORT KernelName &Get() {                                     \
       static KernelName table;                                                 \

fem/linearform.cpp

@@ -161,6 +161,7 @@ bool LinearForm::SupportsDevice() const
 
 void LinearForm::UseFastAssembly(bool use_fa)
 {
+   MFEM_PERF_FUNCTION;
    fast_assembly = use_fa;
 
    if (fast_assembly && SupportsDevice() && !ext)
@@ -171,6 +172,8 @@ void LinearForm::UseFastAssembly(bool use_fa)
 
 void LinearForm::Assemble()
 {
+   MFEM_PERF_FUNCTION;
+
    Array<int> vdofs;
    ElementTransformation *eltrans;
    Vector elemvect;

fem/linearform_ext.cpp

@@ -15,10 +15,16 @@
 namespace mfem
 {
 
-LinearFormExtension::LinearFormExtension(LinearForm *lf): lf(lf) { Update(); }
+LinearFormExtension::LinearFormExtension(LinearForm *lf): lf(lf)
+{
+   MFEM_PERF_FUNCTION;
+   Update();
+}
 
 void LinearFormExtension::Assemble()
 {
+   MFEM_PERF_FUNCTION;
+
    const FiniteElementSpace &fes = *lf->FESpace();
    MFEM_VERIFY(lf->SupportsDevice(), "Not supported.");
    MFEM_VERIFY(lf->Size() == fes.GetVSize(), "LinearForm size does not "
@@ -113,6 +119,8 @@ void LinearFormExtension::Assemble()
 
 void LinearFormExtension::Update()
 {
+   MFEM_PERF_FUNCTION;
+
    const FiniteElementSpace &fes = *lf->FESpace();
    const Mesh &mesh = *fes.GetMesh();
    constexpr ElementDofOrdering ordering = ElementDofOrdering::LEXICOGRAPHIC;

fem/linearform_ext.hpp

@@ -14,6 +14,7 @@
 
 #include "../general/array.hpp"
 #include "../linalg/vector.hpp"
+#include "fespace.hpp"
 
 namespace mfem
 {

fem/lininteg.cpp

@@ -35,19 +35,6 @@ void LinearFormIntegrator::AssembleRHSElementVect(
    mfem_error("LinearFormIntegrator::AssembleRHSElementVect(...)");
 }
 
-DomainLFIntegrator::DomainLFIntegrator(Coefficient &QF, int a, int b)
-   : DeltaLFIntegrator(QF), Q(QF), oa(a), ob(b)
-{
-   static Kernels kernels;
-}
-
-DomainLFIntegrator::DomainLFIntegrator(Coefficient &QF,
-                                       const IntegrationRule *ir)
-   : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1)
-{
-   static Kernels kernels;
-}
-
 void DomainLFIntegrator::AssembleRHSElementVect(const FiniteElement &el,
                                                 ElementTransformation &Tr,
                                                 Vector &elvect)
@@ -279,13 +266,6 @@ void BoundaryTangentialLFIntegrator::AssembleRHSElementVect(
    }
 }
 
-VectorDomainLFIntegrator::VectorDomainLFIntegrator(VectorCoefficient &QF,
-                                                   const IntegrationRule *ir)
-   : DeltaLFIntegrator(QF, ir), Q(QF)
-{
-   static DomainLFIntegrator::Kernels kernels;
-}
-
 void VectorDomainLFIntegrator::AssembleRHSElementVect(
    const FiniteElement &el, ElementTransformation &Tr, Vector &elvect)
 {

fem/lininteg.hpp

@@ -18,8 +18,6 @@
 #include <random>
 #include "integrator.hpp"
 
-#include "kernel_dispatch.hpp"
-
 namespace mfem
 {
 
@@ -111,12 +109,14 @@ class DomainLFIntegrator : public DeltaLFIntegrator
    int oa, ob;
 public:
    /// Constructs a domain integrator with a given Coefficient
-   /// the old default was a = 1, b = 1
-   /// for simple elliptic problems a = 2, b = -2 is OK
-   DomainLFIntegrator(Coefficient &QF, int a = 2, int b = 0);
+   DomainLFIntegrator(Coefficient &QF, int a = 2, int b = 0)
+   // the old default was a = 1, b = 1
+   // for simple elliptic problems a = 2, b = -2 is OK
+      : DeltaLFIntegrator(QF), Q(QF), oa(a), ob(b) { }
 
    /// Constructs a domain integrator with a given Coefficient
-   DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir);
+   DomainLFIntegrator(Coefficient &QF, const IntegrationRule *ir)
+      : DeltaLFIntegrator(QF, ir), Q(QF), oa(1), ob(1) { }
 
    bool SupportsDevice() const override { return true; }
 
@@ -136,22 +136,6 @@ public:
                                  Vector &elvect) override;
 
    using LinearFormIntegrator::AssembleRHSElementVect;
-
-   /// args: vdim, ne, d1d, q1d, map_type, markers, B, detJ, W, coeff, y
-   using AssembleKernelType = void (*)(const int, const int, const int,
-                                       const int, const int, const int *,
-                                       const real_t *, const real_t *,
-                                       const real_t *, const Vector &coeff,
-                                       real_t *y);
-
-   /// parameters: use DIM, T_D1D, T_Q1D
-   MFEM_REGISTER_KERNELS(AssembleKernels, AssembleKernelType, (int, int, int));
-   struct Kernels { Kernels(); };
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      AssembleKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
 };
 
 /// Class for domain integrator $ L(v) := (f, \nabla v) $
@@ -272,13 +256,14 @@ private:
 
 public:
    /// Constructs a domain integrator with a given VectorCoefficient
-   VectorDomainLFIntegrator(VectorCoefficient &QF,
-                            const IntegrationRule *ir = nullptr);
+   VectorDomainLFIntegrator(VectorCoefficient &QF)
+      : DeltaLFIntegrator(QF), Q(QF) { }
 
    bool SupportsDevice() const override { return true; }
 
    /// Method defining assembly on device
-   void AssembleDevice(const FiniteElementSpace &fes, const Array<int> &markers,
+   void AssembleDevice(const FiniteElementSpace &fes,
+                       const Array<int> &markers,
                        Vector &b) override;
 
    /** Given a particular Finite Element and a transformation (Tr)
@@ -292,12 +277,6 @@ public:
                                  Vector &elvect) override;
 
    using LinearFormIntegrator::AssembleRHSElementVect;
-
-   template <int DIM, int D1D, int Q1D> static void AddSpecialization()
-   {
-      // uses the same kernels for assembly
-      DomainLFIntegrator::AssembleKernels::Specialization<DIM, D1D, Q1D>::Add();
-   }
 };
 
 /** Class for domain integrator $ L(v) := (f, \nabla v) $, where
@@ -565,7 +544,7 @@ public:
     Specifically, given the Dirichlet data $u_D$, the linear form assembles the
     following integrals on the boundary:
    $$
-    \sigma \langle u_D, (Q \nabla v) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle,
+    \sigma \langle u_D, (Q \nabla v)) \cdot n \rangle + \kappa \langle {h^{-1} Q} u_D, v \rangle,
    $$
     where Q is a scalar or matrix diffusion coefficient and v is the test
     function. The parameters $\sigma$ and $\kappa$ should be the same as the ones

fem/lor/lor.cpp

@@ -365,6 +365,8 @@ FiniteElementSpace &LORBase::GetFESpace() const
 
 void LORBase::AssembleSystem(BilinearForm &a_ho, const Array<int> &ess_dofs)
 {
+   MFEM_PERF_FUNCTION;
+
    A.Clear();
    delete a;
    if (BatchedLORAssembly::FormIsSupported(a_ho))

fem/lor/lor_batched.cpp

@@ -14,11 +14,9 @@
 #include "../../general/forall.hpp"
 #include <climits>
 #include "../pbilinearform.hpp"
-#include "../../fem/fe/face_map_utils.hpp"
 
 // Specializations
 #include "lor_h1.hpp"
-#include "lor_dg.hpp"
 #include "lor_nd.hpp"
 #include "lor_rt.hpp"
 
@@ -56,18 +54,17 @@ bool BatchedLORAssembly::FormIsSupported(BilinearForm &a)
    // Batched LOR requires all tensor elements
    if (!UsesTensorBasis(*a.FESpace())) { return false; }
 
-   if (dynamic_cast<const H1_FECollection*>(fec) ||
-       dynamic_cast<const DG_FECollection*>(fec))
+   if (dynamic_cast<const H1_FECollection*>(fec))
    {
-      return HasIntegrators<DiffusionIntegrator, MassIntegrator>(a);
+      if (HasIntegrators<DiffusionIntegrator, MassIntegrator>(a)) { return true; }
    }
    else if (dynamic_cast<const ND_FECollection*>(fec))
    {
-      return HasIntegrators<CurlCurlIntegrator, VectorFEMassIntegrator>(a);
+      if (HasIntegrators<CurlCurlIntegrator, VectorFEMassIntegrator>(a)) { return true; }
    }
    else if (dynamic_cast<const RT_FECollection*>(fec))
    {
-      return HasIntegrators<DivDivIntegrator, VectorFEMassIntegrator>(a);
+      if (HasIntegrators<DivDivIntegrator, VectorFEMassIntegrator>(a)) { return true; }
    }
    return false;
 }
@@ -78,14 +75,12 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
    Mesh &mesh_ho = *fes_ho.GetMesh();
    mesh_ho.EnsureNodes();
 
-   const bool dg = fes_ho.IsDGSpace();
-
    // Get nodal points at the LOR vertices
    const int dim = mesh_ho.Dimension();
    const int sdim = mesh_ho.SpaceDimension();
    const int nel_ho = mesh_ho.GetNE();
    const int order = fes_ho.GetMaxElementOrder();
-   const int nd1d = dg ? order + 2 : order + 1;
+   const int nd1d = order + 1;
    const int ndof_per_el = static_cast<int>(pow(nd1d, dim));
 
    const GridFunction *nodal_gf = mesh_ho.GetNodes();
@@ -97,8 +92,7 @@ void BatchedLORAssembly::FormLORVertexCoordinates(FiniteElementSpace &fes_ho,
    Vector nodal_evec(nodal_restriction->Height());
    nodal_restriction->Mult(*nodal_gf, nodal_evec);
 
-   const IntegrationRule ir = GetLobattoIntRule(
-                                 mesh_ho.GetTypicalElementGeometry(), nd1d);
+   IntegrationRule ir = GetCollocatedIntRule(fes_ho);
 
    // Map from nodal E-vector to Q-vector at the LOR vertex points
    X_vert.SetSize(sdim*ndof_per_el*nel_ho);
@@ -165,7 +159,6 @@ int BatchedLORAssembly::FillI(SparseMatrix &A) const
    const auto K = dof_glob2loc_offsets_.Read();
    const auto map = Reshape(sparse_mapping.Read(), nnz_per_row, ndof_per_el);
 
-
    auto I = A.WriteI();
 
    mfem::forall(nvdof + 1, [=] MFEM_HOST_DEVICE (int ii) { I[ii] = 0; });
@@ -365,179 +358,10 @@ void BatchedLORAssembly::FillJAndData(SparseMatrix &A) const
    });
 }
 
-void BatchedLORAssembly::SparseIJToCSR_DG(OperatorHandle &A) const
-{
-   const int ndof_per_el = fes_ho.GetFE(0)->GetDof();
-   const int nel_ho = fes_ho.GetNE();
-   const int nnz_per_row = sparse_ij.Size()/ndof_per_el/nel_ho;
-   const int dim = fes_ho.GetMesh()->Dimension();
-   const int nrows = nel_ho*ndof_per_el;
-   const int p = fes_ho.GetMaxElementOrder();
-   const int pp1 = p + 1;
-   const int nnz = nrows*nnz_per_row;
-
-   const int face_nbr_vsize = [&]()
-   {
-#ifdef MFEM_USE_MPI
-      if (auto *par_fes = dynamic_cast<ParFiniteElementSpace*>(&fes_ho))
-      {
-         return par_fes->GetFaceNbrVSize();
-      }
-#endif
-      return 0;
-   }();
-
-   // If A contains an existing SparseMatrix, reuse it (and try to reuse its
-   // I, J, A arrays if they are big enough)
-   SparseMatrix *A_mat = A.Is<SparseMatrix>();
-   if (!A_mat)
-   {
-      A_mat = new SparseMatrix;
-      A.Reset(A_mat);
-   }
-
-   // The second argument (nrows + face_nbr_vsize) accounts for additional
-   // columns contributed by DG face neighbors in parallel finite element
-   // spaces. In serial, face_nbr_vsize is set to 0.
-   A_mat->OverrideSize(nrows, nrows + face_nbr_vsize);
-
-   EnsureCapacity(A_mat->GetMemoryI(), nrows + 1);
-   EnsureCapacity(A_mat->GetMemoryJ(), nnz);
-   EnsureCapacity(A_mat->GetMemoryData(), nnz);
-
-   Array<int> nbr_info(nel_ho*3*2*dim);
-   auto h_nbr_info = Reshape(nbr_info.HostWrite(), nel_ho, 2*dim, 3);
-   const int num_faces = fes_ho.GetMesh()->GetNumFaces();
-   for (int f = 0; f < num_faces; f++)
-   {
-      Mesh::FaceInformation finfo = fes_ho.GetMesh()->GetFaceInformation(f);
-      int e0 = finfo.element[0].index;
-      int f0 = finfo.element[0].local_face_id;
-      if (finfo.IsBoundary())
-      {
-         h_nbr_info(e0,f0,0) = -1;
-         h_nbr_info(e0,f0,1)= -1;
-         h_nbr_info(e0,f0,2)= -1;
-      }
-      else if (finfo.IsShared())
-      {
-         // Face neighbors elements are indexed after the last local element
-         h_nbr_info(e0,f0,0) = nel_ho + finfo.element[1].index;
-         h_nbr_info(e0,f0,1)= finfo.element[1].orientation;
-         h_nbr_info(e0,f0,2)= finfo.element[1].local_face_id;
-      }
-      else if (finfo.IsInterior())
-      {
-         int e1 = finfo.element[1].index;
-         int f1 = finfo.element[1].local_face_id;
-         h_nbr_info(e0,f0,0) = e1;
-         h_nbr_info(e0,f0,1)= finfo.element[1].orientation;
-         h_nbr_info(e0,f0,2)= f1;
-         h_nbr_info(e1,f1,0) = e0;
-         h_nbr_info(e1,f1,1) = finfo.element[1].orientation;
-         h_nbr_info(e1,f1,2) = f0;
-      }
-   };
-
-   auto h_I = A_mat->HostWriteI();
-   h_I[0] = 0;
-   for (int i = 0; i < nrows; ++i)
-   {
-      const int iel_ho = i / ndof_per_el;
-      const int iloc = i % ndof_per_el;
-      static const int lex_map_2[4] = {3, 1, 0, 2};
-      static const int lex_map_3[6] = {4, 2, 1, 3, 0, 5};
-      const int local_i[3] = {iloc % pp1, (iloc/pp1)%pp1, iloc/pp1/pp1};
-      int bdr_count = 0;
-      for (int n_idx = 0; n_idx < dim; ++n_idx)
-      {
-         for (int e_i = 0; e_i < 2; ++e_i)
-         {
-            const int j_lex = e_i + n_idx*2;
-            const int f = (dim == 3) ? lex_map_3[j_lex]:lex_map_2[j_lex];
-            const bool boundary = (local_i[n_idx] == e_i * p);
-            if (boundary)
-            {
-               int neighbor_idx = h_nbr_info(iel_ho, f, 0);
-               if (neighbor_idx == -1)
-               {
-                  ++bdr_count;
-               }
-            }
-         }
-      }
-      h_I[i+1] = h_I[i] + (nnz_per_row - bdr_count);
-   }
-
-   const auto V = Reshape(sparse_ij.Read(), nnz_per_row, ndof_per_el, nel_ho);
-   auto J = A_mat->WriteJ();
-   auto AV = A_mat->WriteData();
-   auto I = A_mat->ReadI();
-
-   auto d_nbr_info = Reshape(nbr_info.Read(), nel_ho, 2*dim, 3);
-   mfem::forall(nrows, [=] MFEM_HOST_DEVICE (int i)
-   {
-      const int e = i / ndof_per_el;
-      const int iloc = i % ndof_per_el;
-      const int local_x = iloc % pp1;
-      const int local_y = (iloc/pp1)%pp1;
-      const int local_z = iloc/pp1/pp1;
-      const int local_i[3] = {local_x, local_y, local_z};
-      int offset = I[i];
-      static const int lex_map_2[4] = {3, 1, 0, 2};
-      static const int lex_map_3[6] = {4,2,1,3,0,5};
-      const int *lex_map = (dim == 2) ? lex_map_2 : lex_map_3;
-      AV[offset] = V(0, iloc, e);
-      J[offset] = i;
-      ++offset;
-      for (int n_idx = 0; n_idx < dim; ++n_idx)
-      {
-         // qi is the face lexicographic index, obtained by taking the
-         // lexicographic index of the coordinates ommiting n_idx.
-         int qi = 0;
-         int stride = 1;
-         for (int d = 0; d < dim; ++d)
-         {
-            if (d != n_idx)
-            {
-               qi += local_i[d]*stride;
-               stride *= pp1;
-            }
-         }
-         for (int e_i = 0; e_i < 2; ++e_i)
-         {
-            const int j_lex = e_i + n_idx*2;
-            const int f = lex_map[j_lex];
-            const bool bdr = (local_i[n_idx] == e_i * p);
-            if (bdr)
-            {
-               const int nbr_e = d_nbr_info(e, f, 0);
-               const int nbr_ori = d_nbr_info(e, f, 1);
-               const int nbr_f = d_nbr_info(e, f, 2);
-               if (nbr_e != -1)
-               {
-                  const int nbr_loc_idx = internal::FaceIdxToVolIdx(
-                                             dim, qi, pp1, f, nbr_f, 1, nbr_ori);
-                  J[offset] = nbr_e*ndof_per_el + nbr_loc_idx;
-                  AV[offset] = V(f+1, iloc, e);
-                  ++offset;
-               }
-            }
-            else
-            {
-               int shift = (e_i == 0) ? -1 : 1;
-               for (int n = 0; n < n_idx; ++n) { shift *= pp1; }
-               J[offset] = i + shift;
-               AV[offset] = V(f+1, iloc, e);
-               ++offset;
-            }
-         }
-      }
-   });
-}
-
 void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
 {
+   MFEM_PERF_FUNCTION;
+
    const int nvdof = fes_ho.GetVSize();
 
    // If A contains an existing SparseMatrix, reuse it (and try to reuse its
@@ -550,11 +374,12 @@ void BatchedLORAssembly::SparseIJToCSR(OperatorHandle &A) const
    }
 
    A_mat->OverrideSize(nvdof, nvdof);
-   EnsureCapacity(A_mat->GetMemoryI(), nvdof + 1);
 
-   const int nnz = FillI(*A_mat);
-   EnsureCapacity(A_mat->GetMemoryJ(), nnz);
-   EnsureCapacity(A_mat->GetMemoryData(), nnz);
+   A_mat->GetMemoryI().New(nvdof+1, Device::GetDeviceMemoryType());
+   int nnz = FillI(*A_mat);
+
+   A_mat->GetMemoryJ().New(nnz, Device::GetDeviceMemoryType());
+   A_mat->GetMemoryData().New(nnz, Device::GetDeviceMemoryType());
    FillJAndData(*A_mat);
 }
 
@@ -594,6 +419,8 @@ static void Assemble_(LOR_KERNEL &kernel, int dim, int sdim, int order)
 template <typename LOR_KERNEL>
 void BatchedLORAssembly::AssemblyKernel(BilinearForm &a)
 {
+   MFEM_PERF_FUNCTION;
+
    LOR_KERNEL kernel(a, fes_ho, X_vert, sparse_ij, sparse_mapping);
 
    const int dim = fes_ho.GetMesh()->Dimension();
@@ -608,19 +435,6 @@ void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
    // Assemble the matrix, depending on what the form is.
    // This fills in the arrays sparse_ij and sparse_mapping.
    const FiniteElementCollection *fec = fes_ho.FEColl();
-
-   // Handle DG case separately, because assembly of CSR matrix requires
-   // handling face terms.
-   if (dynamic_cast<const DG_FECollection*>(fec))
-   {
-      if (HasIntegrators<DiffusionIntegrator, MassIntegrator>(a))
-      {
-         AssemblyKernel<BatchedLOR_DG>(a);
-      }
-      SparseIJToCSR_DG(A);
-      return;
-   }
-
    if (dynamic_cast<const H1_FECollection*>(fec))
    {
       if (HasIntegrators<DiffusionIntegrator, MassIntegrator>(a))
@@ -643,47 +457,10 @@ void BatchedLORAssembly::AssembleWithoutBC(BilinearForm &a, OperatorHandle &A)
       }
    }
 
-   SparseIJToCSR(A);
+   return SparseIJToCSR(A);
 }
 
 #ifdef MFEM_USE_MPI
-void BatchedLORAssembly::ParAssemble_DG(SparseMatrix &A_local,
-                                        OperatorHandle &A)
-{
-   auto &par_fes = static_cast<ParFiniteElementSpace&>(fes_ho);
-
-   // handle the case when 'a' contains off-diagonal
-   const int lvsize = par_fes.GetVSize();
-   const Array<HYPRE_BigInt> &face_nbr_glob_ldof =
-      par_fes.GetFaceNbrGlobalDofMapArray();
-   const HYPRE_BigInt ldof_offset = par_fes.GetMyDofOffset();
-
-   const int nnz_local = A_local.NumNonZeroElems();
-   Array<HYPRE_BigInt> glob_J(nnz_local);
-
-   const HYPRE_BigInt *d_face_nbr_glob_ldof = face_nbr_glob_ldof.Read();
-   const int *d_J = A_local.ReadJ();
-   HYPRE_BigInt *d_glob_J = glob_J.Write();
-
-   mfem::forall(nnz_local, [=] MFEM_HOST_DEVICE (int i)
-   {
-      if (d_J[i] < lvsize)
-      {
-         d_glob_J[i] = d_J[i] + ldof_offset;
-      }
-      else
-      {
-         d_glob_J[i] = d_face_nbr_glob_ldof[d_J[i] - lvsize];
-      }
-   });
-
-   A.Reset(new HypreParMatrix(
-              par_fes.GetComm(), lvsize, par_fes.GlobalVSize(),
-              par_fes.GlobalVSize(), A_local.HostReadWriteI(),
-              glob_J.HostReadWrite(), A_local.HostReadWriteData(),
-              par_fes.GetDofOffsets(), par_fes.GetDofOffsets()));
-}
-
 void BatchedLORAssembly::ParAssemble(
    BilinearForm &a, const Array<int> &ess_dofs, OperatorHandle &A)
 {
@@ -691,18 +468,13 @@ void BatchedLORAssembly::ParAssemble(
    OperatorHandle A_local;
    AssembleWithoutBC(a, A_local);
 
-   if (dynamic_cast<const DG_FECollection*>(fes_ho.FEColl()))
-   {
-      ParAssemble_DG(*A_local.As<SparseMatrix>(), A);
-   }
-   else
-   {
-      ParBilinearForm *pa =
-         dynamic_cast<ParBilinearForm*>(&a);
-      pa->ParallelRAP(*A_local.As<SparseMatrix>(), A, true);
-      A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
-                                          Operator::DiagonalPolicy::DIAG_ONE);
-   }
+   ParBilinearForm *pa =
+      dynamic_cast<ParBilinearForm*>(&a);
+
+   pa->ParallelRAP(*A_local.As<SparseMatrix>(), A, true);
+
+   A.As<HypreParMatrix>()->EliminateBC(ess_dofs,
+                                       Operator::DiagonalPolicy::DIAG_ONE);
 }
 #endif
 
@@ -736,22 +508,12 @@ BatchedLORAssembly::BatchedLORAssembly(FiniteElementSpace &fes_ho_)
    FormLORVertexCoordinates(fes_ho, X_vert);
 }
 
-IntegrationRule GetLobattoIntRule(Geometry::Type geom, int nd1d)
+IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes)
 {
    IntegrationRules irs(0, Quadrature1D::GaussLobatto);
+   const Geometry::Type geom = fes.GetMesh()->GetTypicalElementGeometry();
+   const int nd1d = fes.GetMaxElementOrder() + 1;
    return irs.Get(geom, 2*nd1d - 3);
 }
 
-IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes)
-{
-   const Geometry::Type geom = fes.GetMesh()->GetTypicalElementGeometry();
-   return GetLobattoIntRule(geom, fes.GetMaxElementOrder() + 1);
-}
-
-IntegrationRule GetCollocatedFaceIntRule(FiniteElementSpace &fes)
-{
-   const Geometry::Type geom = fes.GetMesh()->GetTypicalFaceGeometry();
-   return GetLobattoIntRule(geom, fes.GetMaxElementOrder() + 1);
-}
-
 } // namespace mfem

fem/lor/lor_batched.hpp

@@ -25,7 +25,6 @@ namespace mfem
 /// supported, currently:
 ///
 ///  - H1 diffusion + mass
-///  - DG diffusion + mass (in progress)
 ///  - ND curl-curl + mass
 ///  - RT div-div + mass
 ///
@@ -74,9 +73,6 @@ public:
    /// Return the vertices of the LOR mesh in E-vector format
    const Vector &GetLORVertexCoordinates() { return X_vert; }
 
-   /// Specialized implementation of SparseIJToCSR for DG spaces.
-   void SparseIJToCSR_DG(OperatorHandle &A) const;
-
 protected:
    /// After assembling the "sparse IJ" format, convert it to CSR.
    void SparseIJToCSR(OperatorHandle &A) const;
@@ -109,9 +105,6 @@ public:
    void FillJAndData(SparseMatrix &A) const;
 
 #ifdef MFEM_USE_MPI
-   /// Assemble the parallel DG matrix (with shared faces).
-   void ParAssemble_DG(SparseMatrix &A_local, OperatorHandle &A);
-
    /// Assemble the system in parallel and place the result in @a A.
    void ParAssemble(BilinearForm &a, const Array<int> &ess_dofs,
                     OperatorHandle &A);
@@ -135,8 +128,9 @@ void EnsureCapacity(Memory<T> &mem, int capacity)
 
 /// Return the first domain integrator in the form @a i of type @a T.
 template <typename T>
-static T *GetIntegrator(Array<BilinearFormIntegrator*> *integs)
+static T *GetIntegrator(BilinearForm &a)
 {
+   Array<BilinearFormIntegrator*> *integs = a.GetDBFI();
    if (integs != NULL)
    {
       for (auto *i : *integs)
@@ -150,32 +144,8 @@ static T *GetIntegrator(Array<BilinearFormIntegrator*> *integs)
    return nullptr;
 }
 
-template <typename T>
-static T *GetIntegrator(BilinearForm &a)
-{
-   return GetIntegrator<T>(a.GetDBFI());
-}
-
-template <typename T>
-static T *GetInteriorFaceIntegrator(BilinearForm &a)
-{
-   return GetIntegrator<T>(a.GetFBFI());
-}
-
-/// @brief Return the Gauss-Lobatto rule for geometry @a geom with @a nd1d
-/// points per dimension.
-IntegrationRule GetLobattoIntRule(Geometry::Type geom, int nd1d);
-
-/// @brief Return the Gauss-Lobatto rule collocated with the element nodes.
-///
-/// Assumes @a fes uses Gauss-Lobatto basis.
 IntegrationRule GetCollocatedIntRule(FiniteElementSpace &fes);
 
-/// @brief Return the Gauss-Lobatto rule collocated with face nodes.
-///
-/// Assumes @a fes uses Gauss-Lobatto basis.
-IntegrationRule GetCollocatedFaceIntRule(FiniteElementSpace &fes);
-
 template <typename INTEGRATOR>
 void ProjectLORCoefficient(BilinearForm &a, CoefficientVector &coeff_vector)
 {

fem/lor/lor_dg.hpp

@@ -1,79 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#ifndef MFEM_LOR_DG
-#define MFEM_LOR_DG
-
-#include "lor_batched.hpp"
-
-namespace mfem
-{
-
-// BatchedLORKernel specialization for DG spaces. Not user facing. See the
-// classes BatchedLORAssembly and BatchedLORKernel .
-class BatchedLOR_DG : BatchedLORKernel
-{
-   IntegrationRule ir_face; ///< Collocated Gauss-Lobatto face quadrature rule.
-   real_t kappa; ///< DG penalty parameter.
-public:
-   template <int ORDER, int SDIM> void Assemble2D();
-   template <int ORDER> void Assemble3D();
-   BatchedLOR_DG(BilinearForm &a,
-                 FiniteElementSpace &fes_ho_,
-                 Vector &X_vert_,
-                 Vector &sparse_ij_,
-                 Array<int> &sparse_mapping_)
-      : BatchedLORKernel(fes_ho_, X_vert_, sparse_ij_, sparse_mapping_),
-        ir_face(GetLobattoIntRule(fes_ho_.GetMesh()->GetTypicalFaceGeometry(),
-                                  fes_ho_.GetMaxElementOrder() + 1))
-   {
-      ProjectLORCoefficient<MassIntegrator>(a, c1);
-      ProjectLORCoefficient<DiffusionIntegrator>(a, c2);
-
-      auto *integ = GetInteriorFaceIntegrator<DGDiffusionIntegrator>(a);
-      if (integ)
-      {
-         kappa = integ->GetPenaltyParameter();
-      }
-      else
-      {
-         kappa = 0.0;
-      }
-   }
-
-   /// @brief Compute and return the face info array.
-   ///
-   /// The face info array has shape (6, nf), where @a nf is the number of
-   /// faces. For each face @a i, the column (:,i) has entries (e0, f0, o0, e1,
-   /// f1, o1), where @a e is adjacent element, @a f is the local face index,
-   /// and @a o is the orientation. For boundary and shared faces, (e1, f1, o1)
-   /// are all set to -1.
-   Array<int> GetFaceInfo() const;
-
-   /// @brief Compute and return the boundary penalty factor.
-   ///
-   /// The returned vector has shape (nq, nf), where @a nq is the number of
-   /// nodes per face, and @a nf is the number of faces.
-   ///
-   /// The boundary penalty factor is $J_f / h = J_f^2 / J_e$ (since $h = J_e /
-   /// J_f$), where $J_f$ is the face Jacobian determinant, and $J_e$ is the
-   /// element Jacobian determinant.
-   Vector GetBdrPenaltyFactor() const;
-
-   /// Assemble the face penalty terms in the matrix @a sparse_ij.
-   void AssembleFaceTerms();
-};
-
-}
-
-#include "lor_dg_impl.hpp"
-
-#endif

fem/lor/lor_dg_impl.hpp

@@ -1,392 +0,0 @@
-// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
-// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
-// LICENSE and NOTICE for details. LLNL-CODE-806117.
-//
-// This file is part of the MFEM library. For more information and source code
-// availability visit https://mfem.org.
-//
-// MFEM is free software; you can redistribute it and/or modify it under the
-// terms of the BSD-3 license. We welcome feedback and contributions, see file
-// CONTRIBUTING.md for details.
-
-#pragma once
-
-#include "lor_dg.hpp"
-#include "../fe/face_map_utils.hpp"
-#include "../../linalg/dtensor.hpp"
-#include "../../general/forall.hpp"
-
-namespace mfem
-{
-
-Array<int> BatchedLOR_DG::GetFaceInfo() const
-{
-   Mesh &mesh = *fes_ho.GetMesh();
-   const int nf = mesh.GetNumFaces();
-   Array<int> face_info(nf * 6); // (e0, f0, o0, e1, f1, o1)
-   auto h_face_info = Reshape(face_info.HostWrite(), 6, nf);
-   for (int f = 0; f < nf; ++f)
-   {
-      auto finfo = mesh.GetFaceInformation(f);
-      h_face_info(0, f) = finfo.element[0].index;
-      h_face_info(1, f) = finfo.element[0].local_face_id;
-      h_face_info(2, f) = finfo.element[0].orientation;
-      if (finfo.IsLocal()) // Interior, non-shared face
-      {
-         h_face_info(3, f) = finfo.element[1].index;
-         h_face_info(4, f) = finfo.element[1].local_face_id;
-         h_face_info(5, f) = finfo.element[1].orientation;
-      }
-      else
-      {
-         h_face_info(3, f) = -1;
-         h_face_info(4, f) = -1;
-         h_face_info(5, f) = -1;
-      }
-   }
-   return face_info;
-}
-
-Vector BatchedLOR_DG::GetBdrPenaltyFactor() const
-{
-   Mesh &mesh = *fes_ho.GetMesh();
-
-   const int nf = mesh.GetNumFaces();
-   Array<int> f_int(mesh.GetNFbyType(FaceType::Interior));
-   Array<int> f_bdr(mesh.GetNFbyType(FaceType::Boundary));
-   {
-      int i_int = 0;
-      int i_bdr = 0;
-      for (int i = 0; i < nf; ++i)
-      {
-         const auto f = mesh.GetFaceInformation(i);
-         if (f.IsBoundary())
-         {
-            f_bdr[i_bdr] = i;
-            ++i_bdr;
-         }
-         else if (f.IsInterior())
-         {
-            f_int[i_int] = i;
-            ++i_int;
-         }
-      }
-   }
-
-   const auto geom = fes_ho.GetMesh()->GetGeometricFactors(
-                        ir, GeometricFactors::DETERMINANTS);
-
-   const int nq = ir_face.Size();
-   Vector face_Jh(nq * nf);
-   for (const FaceType ft : {FaceType::Interior, FaceType::Boundary})
-   {
-      const int nft = mesh.GetNFbyType(ft);
-      auto *geom_face = mesh.GetFaceGeometricFactors(
-                           ir_face, FaceGeometricFactors::DETERMINANTS, ft);
-
-      const L2FaceValues fv = (ft == FaceType::Interior)
-                              ? L2FaceValues::DoubleValued
-                              : L2FaceValues::SingleValued;
-      const int m = (fv == L2FaceValues::DoubleValued) ? 2 : 1;
-
-      auto *r = fes_ho.GetFaceRestriction(ElementDofOrdering::LEXICOGRAPHIC, ft, fv);
-      Vector detJ_r(nq * m * nft);
-      r->Mult(geom->detJ, detJ_r);
-
-      const auto *d_i = (ft == FaceType::Interior) ? f_int.Read() : f_bdr.Read();
-      const auto d_detJ_face = Reshape(geom_face->detJ.Read(), nq, nft);
-      const auto d_detJ_r = Reshape(detJ_r.Read(), nq, m, nft);
-      auto d_face_Jh = Reshape(face_Jh.Write(), nq, nf);
-
-      mfem::forall(nft * nq, [=] MFEM_HOST_DEVICE (int ii)
-      {
-         const int i = ii % nq;
-         const int f = ii / nq;
-         const real_t J_el = 0.5*(d_detJ_r(i, 0, f) + d_detJ_r(i, m==2?1:0, f));
-         const real_t J_f = d_detJ_face(i, f);
-         d_face_Jh(i, d_i[f]) = J_f * J_f / J_el;
-      });
-   }
-   return face_Jh;
-}
-
-void BatchedLOR_DG::AssembleFaceTerms()
-{
-   Mesh &mesh = *fes_ho.GetMesh();
-
-   const int nnz_per_row = 1 + mesh.Dimension()*2;
-   const int pp1 = fes_ho.GetMaxElementOrder() + 1;
-   const int nel_ho = mesh.GetNE();
-   const int nf = mesh.GetNumFaces();
-   const int nd_face = ir_face.Size();
-   const int nd = ir.Size();
-   const int dim = mesh.Dimension();
-
-   Array<int> face_info = GetFaceInfo();
-   const auto d_face_info = Reshape(face_info.Read(), 6, nf);
-
-   Vector face_Jh = GetBdrPenaltyFactor();
-   const auto d_face_Jh = Reshape(face_Jh.Read(), nd_face, nf);
-
-   const auto *w_face = ir_face.GetWeights().Read();
-
-   // Penalty parameter (avoid capturing *this in lambda)
-   const real_t d_kappa = kappa;
-
-   // Get diffusion coefficient
-   const bool const_dq = c2.Size() == 1;
-   const auto DQ = const_dq?Reshape(c2.Read(),1,1):Reshape(c2.Read(),nd,nel_ho);
-
-   // Sparse matrix entries
-   auto V = Reshape(sparse_ij.ReadWrite(), nnz_per_row, nd, nel_ho);
-
-   mfem::forall(nf, [=] MFEM_HOST_DEVICE (int f)
-   {
-      const int f_0 = d_face_info(1, f);
-      const int f_1 = d_face_info(4, f);
-      const int nsides = (f_1 >= 0) ? 2 : 1;
-      for (int el_i = 0; el_i < nsides; ++el_i)
-      {
-         const int e = d_face_info(3*el_i, f);
-         const int o = d_face_info(3*el_i + 2, f);
-         const int v_idx = 1 + ((el_i == 0) ? f_0 : f_1);
-         for (int i = 0; i < nd_face; ++i)
-         {
-            const int ii = internal::FaceIdxToVolIdx(dim, i, pp1, f_0, f_1, el_i, o);
-            const real_t Jh = d_face_Jh(i, f);
-            const real_t dq = const_dq ? DQ(0,0) : DQ(ii, e);
-            V(v_idx, ii, e) = -dq*d_kappa*Jh*w_face[i];
-         }
-      }
-   });
-}
-
-template <int ORDER, int SDIM>
-void BatchedLOR_DG::Assemble2D()
-{
-   MFEM_VERIFY(SDIM == 2, "Surface meshes not currently supported for LOR-DG.")
-
-   static constexpr int pp1 = ORDER + 1;
-   static constexpr int ndof_per_el = pp1*pp1;
-   static constexpr int nnz_per_row = 5;
-   const int nel_ho = fes_ho.GetNE();
-
-   // Get element geometric factors; calling before AssembleFaceTerms, since
-   // in AssembleFaceTerms, element Jacobian determinants are used, potentially
-   // saving recomputation.
-   const auto factors = GeometricFactors::DETERMINANTS |
-                        GeometricFactors::JACOBIANS;
-   const auto *geom = fes_ho.GetMesh()->GetGeometricFactors(ir, factors);
-
-   // Sparse matrix entries
-   sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
-   sparse_ij.UseDevice(true);
-   sparse_ij = 0.0;
-   auto V = Reshape(sparse_ij.ReadWrite(), nnz_per_row, pp1, pp1, nel_ho);
-
-   AssembleFaceTerms();
-
-   // Populate Gauss-Lobatto quadrature rule of size (p+1)
-   IntegrationRule ir_pp1;
-   QuadratureFunctions1D::GaussLobatto(pp1, &ir_pp1);
-   Vector glx_pp1(pp1), glw_pp1(pp1);
-   for (int i = 0; i < pp1; ++i)
-   {
-      glx_pp1[i] = ir_pp1[i].x;
-      glw_pp1[i] = ir_pp1[i].weight;
-   }
-   const auto *x_pp1 = glx_pp1.Read();
-   const auto *w_1d = glw_pp1.Read();
-
-   // Get coefficients for mass and diffusion
-   const bool const_mq = c1.Size() == 1;
-   const auto MQ = const_mq
-                   ? Reshape(c1.Read(), 1, 1, 1)
-                   : Reshape(c1.Read(), pp1, pp1, nel_ho);
-   const bool const_dq = c2.Size() == 1;
-   const auto DQ = const_dq
-                   ? Reshape(c2.Read(), 1, 1, 1)
-                   : Reshape(c2.Read(), pp1, pp1, nel_ho);
-
-   const auto detJ = Reshape(geom->detJ.Read(), pp1, pp1, nel_ho);
-   const auto J = Reshape(geom->J.Read(), pp1, pp1, 2, 2, nel_ho);
-   const auto W = Reshape(ir.GetWeights().Read(), pp1, pp1);
-
-   mfem::forall(nel_ho, [=] MFEM_HOST_DEVICE (int iel_ho)
-   {
-      for (int iy = 0; iy < pp1; ++iy)
-      {
-         for (int ix = 0; ix < pp1; ++ix)
-         {
-            const real_t mq = const_mq ? MQ(0,0,0) : MQ(ix, iy, iel_ho);
-            const real_t dq = const_dq ? DQ(0,0,0) : DQ(ix, iy, iel_ho);
-
-            for (int n_idx = 0; n_idx < 2; ++n_idx)
-            {
-               for (int e_i = 0; e_i < 2; ++e_i)
-               {
-                  const int i_0 = (n_idx == 0) ? ix + e_i : ix;
-                  const int j_0 = (n_idx == 1) ? iy + e_i : iy;
-
-                  const bool bdr = (n_idx == 0 && (i_0 == 0 || i_0 == pp1)) ||
-                                   (n_idx == 1 && (j_0 == 0 || j_0 == pp1));
-
-                  if (bdr) { continue; }
-
-                  static constexpr int lex_map[] = {4, 2, 1, 3};
-                  const int v_idx_lex = e_i + n_idx*2;
-                  const int v_idx = lex_map[v_idx_lex];
-
-                  const int w_idx = (n_idx == 0) ? iy : ix;
-                  const int x_idx = (n_idx == 0) ? i_0 : j_0;
-
-                  const real_t J1 = J(ix, iy, n_idx, !n_idx, iel_ho);
-                  const real_t J2 = J(ix, iy, !n_idx, !n_idx, iel_ho);
-                  const real_t Jh = (J1*J1 + J2*J2) / detJ(ix, iy, iel_ho);
-
-                  V(v_idx, ix, iy, iel_ho) =
-                     -dq * Jh * w_1d[w_idx] / (x_pp1[x_idx] - x_pp1[x_idx -1]);
-               }
-            }
-            V(0, ix, iy, iel_ho) = mq * detJ(ix, iy, iel_ho) * W(ix, iy);
-            for (int i = 1; i < nnz_per_row; ++i)
-            {
-               V(0, ix, iy, iel_ho) -= V(i, ix, iy, iel_ho);
-            }
-         }
-      }
-   });
-}
-
-template <int ORDER>
-void BatchedLOR_DG::Assemble3D()
-{
-   static constexpr int pp1 = ORDER + 1;
-   static constexpr int ndof_per_el = pp1*pp1*pp1;
-   static constexpr int nnz_per_row = 7;
-   const int nel_ho = fes_ho.GetNE();
-
-   // Get element geometric factors; calling before AssembleFaceTerms, since
-   // in AssembleFaceTerms, element Jacobian determinants are used, potentially
-   // saving recomputation.
-   const auto factors = GeometricFactors::DETERMINANTS |
-                        GeometricFactors::JACOBIANS;
-   const auto geom = fes_ho.GetMesh()->GetGeometricFactors(ir, factors);
-
-   sparse_ij.SetSize(nnz_per_row*ndof_per_el*nel_ho);
-   sparse_ij.UseDevice(true);
-   sparse_ij = 0.0;
-   auto V = Reshape(sparse_ij.Write(), nnz_per_row, pp1, pp1, pp1, nel_ho);
-
-   AssembleFaceTerms();
-
-   // Populate Gauss-Lobatto quadrature rule of size (p+1)
-   IntegrationRule ir_pp1;
-   QuadratureFunctions1D::GaussLobatto(pp1, &ir_pp1);
-   Vector glx_pp1(pp1), glw_pp1(pp1);
-   for (int i = 0; i < pp1; ++i)
-   {
-      glx_pp1[i] = ir_pp1[i].x;
-      glw_pp1[i] = ir_pp1[i].weight;
-   }
-   const auto *x_pp1 = glx_pp1.Read();
-   const auto *w_1d = glw_pp1.Read();
-
-   const bool const_mq = c1.Size() == 1;
-   const auto MQ = const_mq
-                   ? Reshape(c1.Read(), 1, 1, 1, 1)
-                   : Reshape(c1.Read(), pp1, pp1, pp1, nel_ho);
-   const bool const_dq = c2.Size() == 1;
-   const auto DQ = const_dq
-                   ? Reshape(c2.Read(), 1, 1, 1, 1)
-                   : Reshape(c2.Read(), pp1, pp1, pp1, nel_ho);
-   const auto W = Reshape(ir.GetWeights().Read(), pp1, pp1, pp1);
-
-   const auto detJ = Reshape(geom->detJ.Read(), pp1, pp1, pp1, nel_ho);
-   const auto J = Reshape(geom->J.Read(), pp1, pp1, pp1, 3, 3, nel_ho);
-
-   mfem::forall(nel_ho, [=] MFEM_HOST_DEVICE (int iel_ho)
-   {
-      for (int iz = 0; iz < pp1; ++iz)
-      {
-         for (int iy = 0; iy < pp1; ++iy)
-         {
-            for (int ix = 0; ix < pp1; ++ix)
-            {
-               const real_t mq = const_mq ? MQ(0,0,0,0) : MQ(ix, iy, iz, iel_ho);
-               const real_t dq = const_dq ? DQ(0,0,0,0) : DQ(ix, iy, iz, iel_ho);
-
-               const real_t DETJ = detJ(ix, iy, iz, iel_ho);
-
-               for (int n_idx = 0; n_idx < 3; ++n_idx)
-               {
-                  for (int e_i = 0; e_i < 2; ++e_i)
-                  {
-                     static constexpr int lex_map[] = {5,3,2,4,1,6};
-                     const int v_idx_lex = e_i + n_idx*2;
-                     const int v_idx = lex_map[v_idx_lex];
-
-                     const int i_0 = (n_idx == 0) ? ix + e_i : ix;
-                     const int j_0 = (n_idx == 1) ? iy + e_i : iy;
-                     const int k_0 = (n_idx == 2) ? iz + e_i : iz;
-
-                     const bool bdr =
-                        (n_idx == 0 && (i_0 == 0 || i_0 == pp1)) ||
-                        (n_idx == 1 && (j_0 == 0 || j_0 == pp1)) ||
-                        (n_idx == 2 && (k_0 == 0 || k_0 == pp1));
-
-                     if (bdr) { continue; }
-
-                     int x_idx = (n_idx == 0) ? i_0 : (n_idx == 1) ? j_0 : k_0;
-                     int w_idx_1 = (n_idx == 0) ? iy : (n_idx == 1) ? iz : ix;
-                     int w_idx_2 = (n_idx == 0) ? iz : (n_idx == 1) ? ix : iy;
-
-                     const real_t J00 = J(ix, iy, iz, 0, 0, iel_ho);
-                     const real_t J01 = J(ix, iy, iz, 0, 1, iel_ho);
-                     const real_t J02 = J(ix, iy, iz, 0, 2, iel_ho);
-                     const real_t J10 = J(ix, iy, iz, 1, 0, iel_ho);
-                     const real_t J11 = J(ix, iy, iz, 1, 1, iel_ho);
-                     const real_t J12 = J(ix, iy, iz, 1, 2, iel_ho);
-                     const real_t J20 = J(ix, iy, iz, 2, 0, iel_ho);
-                     const real_t J21 = J(ix, iy, iz, 2, 1, iel_ho);
-                     const real_t J22 = J(ix, iy, iz, 2, 2, iel_ho);
-
-                     real_t JinvJinvT_diag = 0.0;
-                     if (n_idx == 0)
-                     {
-                        JinvJinvT_diag = J02*J02*(J11*J11 + J21*J21) + (J12*J21 - J11*J22)*
-                                         (J12*J21 - J11*J22) - 2*J01*J02*(J11*J12 + J21*J22) + J01*J01*
-                                         (J12*J12 + J22*J22);
-                     }
-                     else if (n_idx == 1)
-                     {
-                        JinvJinvT_diag = J02*J02*(J10*J10 + J20*J20) + (J12*J20 - J10*J22)*
-                                         (J12*J20 - J10*J22) - 2*J00*J02*(J10*J12 + J20*J22) + J00*J00*
-                                         (J12*J12 + J22*J22);
-                     }
-                     else if (n_idx == 2)
-                     {
-                        JinvJinvT_diag = J01*J01*(J10*J10 + J20*J20) + (J11*J20 - J10*J21)*
-                                         (J11*J20 - J10*J21) - 2*J00*J01*(J10*J11 + J20*J21) + J00*J00*
-                                         (J11*J11 + J21*J21);
-                     }
-
-                     const real_t Jh = JinvJinvT_diag / DETJ;
-
-                     V(v_idx, ix, iy, iz, iel_ho) = -dq * Jh * w_1d[w_idx_1] * w_1d[w_idx_2] /
-                                                    (x_pp1[x_idx] - x_pp1[x_idx -1]);
-                  }
-               }
-               V(0, ix, iy, iz, iel_ho) = mq * DETJ * W(ix, iy, iz);
-               for (int i = 1; i < 7; ++i)
-               {
-                  V(0, ix, iy, iz, iel_ho) -= V(i, ix, iy, iz, iel_ho);
-               }
-            }
-         }
-      }
-
-   });
-}
-
-} // namespace mfem

fem/lor/lor_h1_impl.hpp

@@ -184,6 +184,8 @@ void BatchedLOR_H1::Assemble2D()
 template <int ORDER>
 void BatchedLOR_H1::Assemble3D()
 {
+   MFEM_PERF_FUNCTION;
+
    const int nel_ho = fes_ho.GetNE();
    static constexpr int nv = 8;
    static constexpr int dim = 3;

fem/multigrid.cpp

@@ -10,6 +10,7 @@
 // CONTRIBUTING.md for details.
 
 #include "multigrid.hpp"
+#include "../general/annotation.hpp"
 
 namespace mfem
 {
@@ -17,7 +18,10 @@ namespace mfem
 MultigridBase::MultigridBase()
    : cycleType(CycleType::VCYCLE), preSmoothingSteps(1), postSmoothingSteps(1),
      nrhs(0)
-{}
+{
+   coarse_solver = nullptr;
+   own_coarse_solver = false;
+}
 
 MultigridBase::MultigridBase(const Array<Operator*>& operators_,
                              const Array<Solver*>& smoothers_,
@@ -29,12 +33,18 @@ MultigridBase::MultigridBase(const Array<Operator*>& operators_,
 {
    operators_.Copy(operators);
    smoothers_.Copy(smoothers);
+   coarse_solver = nullptr;
    ownedOperators_.Copy(ownedOperators);
    ownedSmoothers_.Copy(ownedSmoothers);
+   own_coarse_solver = false;
 }
 
 MultigridBase::~MultigridBase()
 {
+   if (own_coarse_solver)
+   {
+      delete coarse_solver;
+   }
    for (int i = 0; i < operators.Size(); ++i)
    {
       if (ownedOperators[i])
@@ -56,16 +66,17 @@ void MultigridBase::InitVectors() const
    X.SetSize(M, nrhs);
    Y.SetSize(M, nrhs);
    R.SetSize(M, nrhs);
-   Z.SetSize(M, nrhs);
-   for (int i = 0; i < X.NumRows(); ++i)
+   for (int i = 0; i < M; ++i)
    {
       const int n = operators[i]->Height();
-      for (int j = 0; j < X.NumCols(); ++j)
+      for (int j = 0; j < nrhs; ++j)
       {
-         X(i, j) = new Vector(n);
-         Y(i, j) = new Vector(n);
+         if (i < M - 1)
+         {
+            X(i, j) = new Vector(n);
+            Y(i, j) = new Vector(n);
+         }
          R(i, j) = new Vector(n);
-         Z(i, j) = new Vector(n);
       }
    }
 }
@@ -76,10 +87,12 @@ void MultigridBase::EraseVectors() const
    {
       for (int j = 0; j < X.NumCols(); ++j)
       {
-         delete X(i, j);
-         delete Y(i, j);
+         if (i < X.NumRows() - 1)
+         {
+            delete X(i, j);
+            delete Y(i, j);
+         }
          delete R(i, j);
-         delete Z(i, j);
       }
    }
 }
@@ -95,6 +108,12 @@ void MultigridBase::AddLevel(Operator* op, Solver* smoother,
    ownedSmoothers.Append(ownSmoother);
 }
 
+void MultigridBase::AddCoarseSolver(Solver *c_solver, bool own_c_solver)
+{
+   coarse_solver = c_solver;
+   own_coarse_solver = own_c_solver;
+}
+
 void MultigridBase::SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
                                  int postSmoothingSteps_)
 {
@@ -105,25 +124,24 @@ void MultigridBase::SetCycleType(CycleType cycleType_, int preSmoothingSteps_,
 
 void MultigridBase::Mult(const Vector& x, Vector& y) const
 {
-   Array<const Vector*> X_(1);
-   Array<Vector*> Y_(1);
-   X_[0] = &x;
-   Y_[0] = &y;
+   const Vector *x_array[1] = { &x };
+   Array<const Vector*> X_(x_array, 1); // no heap allocation
+
+   Vector *y_array[1] = { &y };
+   Array<Vector*> Y_(y_array, 1); // no heap allocation
+
    ArrayMult(X_, Y_);
 }
 
 void MultigridBase::ArrayMult(const Array<const Vector*>& X_,
                               Array<Vector*>& Y_) const
 {
+   MFEM_PERF_FUNCTION;
+
    MFEM_ASSERT(operators.Size() > 0,
                "Multigrid solver does not have operators set!");
    MFEM_ASSERT(X_.Size() == Y_.Size(),
                "Number of columns mismatch in MultigridBase::Mult!");
-   if (iterative_mode)
-   {
-      MFEM_WARNING("Multigrid solver does not use iterative_mode and ignores "
-                   "the initial guess!");
-   }
 
    // Add capacity as necessary
    nrhs = X_.Size();
@@ -134,96 +152,163 @@ void MultigridBase::ArrayMult(const Array<const Vector*>& X_,
    for (int j = 0; j < nrhs; ++j)
    {
       MFEM_ASSERT(X_[j] && Y_[j], "Missing Vector in MultigridBase::Mult!");
-      *X(M - 1, j) = *X_[j];
-      *Y(M - 1, j) = 0.0;
-   }
-   Cycle(M - 1);
-   for (int j = 0; j < nrhs; ++j)
-   {
-      *Y_[j] = *Y(M - 1, j);
+      X(M - 1, j) = const_cast<Vector*>(X_[j]);
+      Y(M - 1, j) = Y_[j];
    }
+   const bool zero = !iterative_mode;
+   Cycle(M - 1, zero);
 }
 
 void MultigridBase::SmoothingStep(int level, bool zero, bool transpose) const
 {
+   MFEM_PERF_FUNCTION;
+
    // y = y + S (x - A y) or y = y + S^T (x - A y)
+
+   // Note: 'zero' == true means that Y(level,*) are not initialized and we
+   // should assume that the input they typically provide to this call is zeros.
+
+   // We can't use the smoothers' iterative mode since we don't know if they
+   // actually support it, so we always turn the iterative mode off to properly
+   // use smoothers that do support it.
+   smoothers[level]->iterative_mode = false;
+
    if (zero)
    {
-      Array<Vector *> X_(X[level], nrhs), Y_(Y[level], nrhs);
-      GetSmootherAtLevel(level)->ArrayMult(X_, Y_);
+      MFEM_ASSERT(!transpose, "internal error!");
+      const Array<const Vector *> cX_((const Vector **)(X[level]), nrhs);
+      Array<Vector *> Y_(Y[level], nrhs);
+
+      GetSmootherAtLevel(level)->ArrayMult(cX_, Y_);
    }
    else
    {
-      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs),
-            Z_(Z[level], nrhs);
+      const Array<const Vector *> cY_((const Vector **)(Y[level]), nrhs),
+            cR_((const Vector **)(R[level]), nrhs);
+      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs);
+
+      GetOperatorAtLevel(level)->ArrayMult(cY_, R_);
       for (int j = 0; j < nrhs; ++j)
       {
-         *R_[j] = *X(level, j);
+         // *R_[j] = *X(level, j) - *R_[j]
+         subtract(*X(level, j), *R_[j], *R_[j]);
       }
-      GetOperatorAtLevel(level)->ArrayAddMult(Y_, R_, -1.0);
       if (transpose)
       {
-         GetSmootherAtLevel(level)->ArrayMultTranspose(R_, Z_);
+         GetSmootherAtLevel(level)->ArrayAddMultTranspose(cR_, Y_);
       }
       else
       {
-         GetSmootherAtLevel(level)->ArrayMult(R_, Z_);
-      }
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y_[j] += *Z_[j];
+         GetSmootherAtLevel(level)->ArrayAddMult(cR_, Y_);
       }
    }
 }
 
-void MultigridBase::Cycle(int level) const
+void MultigridBase::CoarseSolve(bool zero) const
 {
-   // Coarse solve
-   if (level == 0)
+   MFEM_PERF_FUNCTION;
+
+   // See the comment about iterative mode in SmoothingStep()
+   coarse_solver->iterative_mode = false;
+
+   if (zero)
    {
-      SmoothingStep(0, true, false);
+      const Array<const Vector *> cX_((const Vector **)(X[0]), nrhs);
+      Array<Vector *> Y_(Y[0], nrhs);
+
+      coarse_solver->ArrayMult(cX_, Y_);
+   }
+   else
+   {
+      const Array<const Vector *> cY_((const Vector **)(Y[0]), nrhs),
+            cR_((const Vector **)(R[0]), nrhs);
+      Array<Vector *> Y_(Y[0], nrhs), R_(R[0], nrhs);
+
+      GetOperatorAtLevel(0)->ArrayMult(cY_, R_);
+      for (int j = 0; j < nrhs; ++j)
+      {
+         // *R_[j] = *X(0, j) - *R_[j]
+         subtract(*X(0, j), *R_[j], *R_[j]);
+      }
+      coarse_solver->ArrayAddMult(cR_, Y_);
+   }
+}
+
+void MultigridBase::Cycle(int level, bool zero) const
+{
+   // Note: 'zero' == true means that Y(level,*) are not initialized and we
+   // should assume that the input they typically provide to this call is zeros.
+
+   // Coarse solve
+   if (level == 0 && !coarse_solver)
+   {
+      SmoothingStep(0, zero, false);
       return;
    }
 
    // Pre-smooth
    for (int i = 0; i < preSmoothingSteps; ++i)
    {
-      SmoothingStep(level, (cycleType == CycleType::VCYCLE && i == 0), false);
+      SmoothingStep(level, zero && (i == 0), false);
+   }
+
+   // Coarse solve with 'coarse_solver'
+   if (level == 0)
+   {
+      CoarseSolve(preSmoothingSteps == 0 && zero);
+      goto mg_post_smooth;
    }
 
    // Compute residual and restrict
+   if (preSmoothingSteps == 0 && zero)
    {
-      Array<Vector *> Y_(Y[level], nrhs), R_(R[level], nrhs),
-            X_(X[level - 1], nrhs);
+      const Array<const Vector *> cX_l((const Vector **)(X[level]), nrhs);
+      Array<Vector *> X_lm1(X[level - 1], nrhs);
+
+      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(cX_l, X_lm1);
+   }
+   else
+   {
+      const Array<const Vector *> cY_((const Vector **)(Y[level]), nrhs),
+            cR_((const Vector **)(R[level]), nrhs);
+      Array<Vector *> R_(R[level], nrhs), X_(X[level - 1], nrhs);
+
+      GetOperatorAtLevel(level)->ArrayMult(cY_, R_);
       for (int j = 0; j < nrhs; ++j)
       {
-         *R_[j] = *X(level, j);
-      }
-      GetOperatorAtLevel(level)->ArrayAddMult(Y_, R_, -1.0);
-      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(R_, X_);
-      for (int j = 0; j < nrhs; ++j)
-      {
-         *Y(level - 1, j) = 0.0;
+         // *R_[j] = *X(level, j) - *R_[j]
+         subtract(*X(level, j), *R_[j], *R_[j]);
       }
+      GetProlongationAtLevel(level - 1)->ArrayMultTranspose(cR_, X_);
    }
 
    // Corrections
-   Cycle(level - 1);
+   Cycle(level - 1, true);
    if (cycleType == CycleType::WCYCLE)
    {
-      Cycle(level - 1);
+      // If the coarse solve at level 0 is "exact" solve, then we don't want to
+      // repeat it.
+      // To support multiple level 0 coarse-grid corrections, one can wrap that
+      // smoother in an SLI solver and use that instead.
+      if (level > 1) { Cycle(level - 1, false); }
    }
 
    // Prolongate and add
    {
-      Array<Vector *> Y_(Y[level - 1], nrhs), Z_(Z[level], nrhs);
-      GetProlongationAtLevel(level - 1)->ArrayMult(Y_, Z_);
-      for (int j = 0; j < nrhs; ++j)
+      const Array<const Vector *> cY_lm1((const Vector **)(Y[level - 1]), nrhs);
+      Array<Vector *> Y_l(Y[level], nrhs);
+
+      if (preSmoothingSteps == 0 && zero)
       {
-         *Y(level, j) += *Z_[j];
+         GetProlongationAtLevel(level - 1)->ArrayMult(cY_lm1, Y_l);
+      }
+      else
+      {
+         GetProlongationAtLevel(level - 1)->ArrayAddMult(cY_lm1, Y_l);
       }
    }
 
+mg_post_smooth:
    // Post-smooth
    for (int i = 0; i < postSmoothingSteps; ++i)
    {

fem/multigrid.hpp

@@ -36,12 +36,14 @@ protected:
    Array<Solver*> smoothers;
    Array<bool> ownedOperators;
    Array<bool> ownedSmoothers;
+   Solver *coarse_solver; /// can be NULL, see AddCoarseSolver()
+   bool own_coarse_solver;
 
    CycleType cycleType;
    int preSmoothingSteps;
    int postSmoothingSteps;
 
-   mutable Array2D<Vector*> X, Y, R, Z;
+   mutable Array2D<Vector*> X, Y, R;
    mutable int nrhs;
 
 public:
@@ -65,6 +67,16 @@ public:
    void AddLevel(Operator* op, Solver* smoother, bool ownOperator,
                  bool ownSmoother);
 
+   /// Adds a coarse solver for level 0 to work in tandem with the smoother
+   /** If this coarse solver is not given, the smoother at level 0 is used as
+       the coarse solver. When this coarse solver is given, the smoother at
+       level 0 is used similar to the smoothers at other levels. Thus, the
+       action at level 0 consists of:
+       - pre-smoothing steps with smoother 0,
+       - solve step with @a c_solver,
+       - post-smoothing steps with smoother 0. */
+   void AddCoarseSolver(Solver *c_solver, bool own_c_solver);
+
    /// Returns the number of levels
    int NumLevels() const { return operators.Size(); }
 
@@ -118,11 +130,14 @@ public:
 
 private:
    /// Application of a multigrid cycle at particular level
-   void Cycle(int level) const;
+   void Cycle(int level, bool zero) const;
 
    /// Application of a pre-/post-smoothing step at particular level
    void SmoothingStep(int level, bool zero, bool transpose) const;
 
+   /// Perform a coarse solve with 'coarse_solve' (must be non-NULL)
+   void CoarseSolve(bool zero) const;
+
    /// Allocate or destroy temporary storage
    void InitVectors() const;
    void EraseVectors() const;
@@ -202,6 +217,9 @@ public:
 
    /// Recover the solution of a linear system formed with FormFineLinearSystem()
    void RecoverFineFEMSolution(const Vector& X, const Vector& b, Vector& x);
+
+   const Array<int> &GetFineEssentialTrueDofs() const
+   { return *essentialTrueDofs.Last(); }
 };
 
 } // namespace mfem

fem/nonlinearform.cpp

@@ -788,10 +788,12 @@ BlockNonlinearForm::BlockNonlinearForm(Array<FiniteElementSpace *> &f) :
 }
 
 void BlockNonlinearForm::SetEssentialBC(
-   const Array<Array<int>*> &bdr_attr_is_ess, Array<Vector*> &rhs)
+   const Array<Array<int> *> &bdr_attr_is_ess, Array<Vector *> &rhs)
 {
    for (int s = 0; s < fes.Size(); ++s)
    {
+      ess_tdofs[s]->SetSize(ess_tdofs.Size());
+
       fes[s]->GetEssentialTrueDofs(*bdr_attr_is_ess[s], *ess_tdofs[s]);
 
       if (rhs[s])
@@ -801,19 +803,6 @@ void BlockNonlinearForm::SetEssentialBC(
    }
 }
 
-void BlockNonlinearForm::SetEssentialTrueDofs(
-   const Array<Array<int>*> &ess_tdof_list, Array<Vector*> &rhs)
-{
-   for (int s = 0; s < fes.Size(); ++s)
-   {
-      *ess_tdofs[s] = *ess_tdof_list[s];
-      if (rhs[s])
-      {
-         rhs[s]->SetSubVector(*ess_tdofs[s], 0.0);
-      }
-   }
-}
-
 real_t BlockNonlinearForm::GetEnergyBlocked(const BlockVector &bx) const
 {
    Array<Array<int> *> vdofs(fes.Size());

fem/nonlinearform.hpp

@@ -363,39 +363,8 @@ public:
                              Array<int> &bdr_marker)
    { bfnfi.Append(nlfi); bfnfi_marker.Append(&bdr_marker); }
 
-   /** @brief Set essential boundary conditions to each finite element space
-       using boundary attribute markers.
-
-       This method calls `FiniteElementSpace::GetEssentialTrueDofs()` for each
-       space and stores ess_tdof_lists internally.
-
-       If `rhs` vectors are non-null, the entries corresponding to these
-       essential DoFs are set to zero. This ensures compatibility with the
-       output of the `Mult()` method, which also zeroes out these entries.
-
-       @param[in] bdr_attr_is_ess A list of boundary attribute markers for each
-       space.
-       @param[in,out] rhs         An array of optional right-hand side vectors.
-       If a vector at `rhs[i]` is non-null, its essential DoFs will be set
-       to zero. */
-   virtual void SetEssentialBC(const Array<Array<int>*> &bdr_attr_is_ess,
-                               Array<Vector*> &rhs);
-
-   /** @brief Set essential boundary conditions to each finite element space
-       using essential true dof lists.
-
-       This method stores a copy of the provided essential true dof lists.
-
-       If `rhs` vectors are non-null, the entries corresponding to these
-       essential DoFs are set to zero. This ensures compatibility with the
-       output of the `Mult()` method, which also zeroes out these entries.
-
-       @param[in] ess_tdof_list A list of essential true dofs for each space.
-       @param[in,out] rhs       An array of optional right-hand side vectors.
-       If a vector at `rhs[i]` is non-null, its essential DoFs will be set
-       to zero. */
-   virtual void SetEssentialTrueDofs(const Array<Array<int>*> &ess_tdof_list,
-                                     Array<Vector*> &rhs);
+   virtual void SetEssentialBC(const Array<Array<int> *>&bdr_attr_is_ess,
+                               Array<Vector *> &rhs);
 
    virtual real_t GetEnergy(const Vector &x) const;
 

fem/pbilinearform.cpp

@@ -124,6 +124,8 @@ void ParBilinearForm::pAllocMat()
 void ParBilinearForm::ParallelRAP(SparseMatrix &loc_A, OperatorHandle &A,
                                   bool steal_loc_A)
 {
+   MFEM_PERF_FUNCTION;
+
    ParFiniteElementSpace &pfespace = *ParFESpace();
 
    // Create a block diagonal parallel matrix

fem/pderefmat_op.cpp

@@ -332,7 +332,7 @@ ParDerefineMatrixOp::ParDerefineMatrixOp(ParFiniteElementSpace &fespace_,
    pack_col_idcs.SetSize(send_len);
    // memory manager doesn't appear to have a graceful fallback for
    // HOST_PINNED if not built with CUDA or HIP
-#if defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP)
+#if defined(MFEM_USE_CUDA) or defined(MFEM_USE_HIP)
    xghost_send.SetSize(send_len * fespace->GetVDim(),
                        Device::GetGPUAwareMPI() ? MemoryType::DEFAULT
                        : MemoryType::HOST_PINNED);

fem/pfespace.cpp

@@ -63,9 +63,11 @@ ParFiniteElementSpace::ParFiniteElementSpace(
 
 ParFiniteElementSpace::ParFiniteElementSpace(
    ParMesh *pm, const FiniteElementCollection *f, int dim, int ordering)
-   : FiniteElementSpace(pm, f, dim, ordering)
+   : FiniteElementSpace((MFEM_PERF_BEGIN(_MFEM_FUNC_NAME), pm),
+                        f, dim, ordering)
 {
    ParInit(pm);
+   MFEM_PERF_END(_MFEM_FUNC_NAME);
 }
 
 ParFiniteElementSpace::ParFiniteElementSpace(
@@ -92,6 +94,7 @@ ParNURBSExtension *ParFiniteElementSpace::MakeLocalNURBSext(
 
 void ParFiniteElementSpace::ParInit(ParMesh *pm)
 {
+   MFEM_PERF_FUNCTION;
    pmesh = pm;
    pncmesh = nullptr;
 
@@ -181,6 +184,7 @@ void ParFiniteElementSpace::CommunicateGhostOrder()
 
 void ParFiniteElementSpace::Construct()
 {
+   MFEM_PERF_FUNCTION;
    if (NURBSext)
    {
       ConstructTrueNURBSDofs();
@@ -839,6 +843,8 @@ void ParFiniteElementSpace::Build_Dof_TrueDof_Matrix() const // matrix P
 
    if (P) { return; }
 
+   MFEM_PERF_FUNCTION;
+
    if (!nd_strias)
    {
       // Safe to assume 1-1 correspondence between shared dofs
@@ -1424,6 +1430,7 @@ const Operator *ParFiniteElementSpace::GetRestrictionOperator() const
       if (NRanks == 1)
       {
          R_transpose.reset(new IdentityOperator(GetTrueVSize()));
+         Rconf = new IdentityOperator(GetTrueVSize());
       }
       else
       {
@@ -1436,8 +1443,8 @@ const Operator *ParFiniteElementSpace::GetRestrictionOperator() const
             R_transpose.reset(
                new DeviceConformingProlongationOperator(*this, true));
          }
+         Rconf = new TransposeOperator(*R_transpose);
       }
-      Rconf = new TransposeOperator(*R_transpose);
       return Rconf;
    }
    else

fem/pfespace.hpp

@@ -481,7 +481,6 @@ public:
        that the number of DOFs is @a ndofs. */
    const FiniteElement *GetFaceNbrFE(int i, int ndofs = 0) const;
    const FiniteElement *GetFaceNbrFaceFE(int i) const;
-   const Array<HYPRE_BigInt> &GetFaceNbrGlobalDofMapArray() { return face_nbr_glob_dof_map; }
    const HYPRE_BigInt *GetFaceNbrGlobalDofMap() { return face_nbr_glob_dof_map; }
    ElementTransformation *GetFaceNbrElementTransformation(int i) const
    { return pmesh->GetFaceNbrElementTransformation(i); }

fem/plinearform.cpp

@@ -45,6 +45,8 @@ void ParLinearForm::MakeRef(ParFiniteElementSpace *pf, Vector &v, int v_offset)
 
 void ParLinearForm::Assemble()
 {
+   MFEM_PERF_FUNCTION;
+
    LinearForm::Assemble();
 
    if (interior_face_integs.Size())

fem/pnonlinearform.cpp

@@ -199,8 +199,9 @@ const ParFiniteElementSpace *ParBlockNonlinearForm::ParFESpace(int k) const
 }
 
 // Here, rhs is a true dof vector
-void ParBlockNonlinearForm::SetEssentialBC(
-   const Array<Array<int>*> &bdr_attr_is_ess, Array<Vector*> &rhs)
+void ParBlockNonlinearForm::SetEssentialBC(const
+                                           Array<Array<int> *>&bdr_attr_is_ess,
+                                           Array<Vector *> &rhs)
 {
    Array<Vector *> nullarray(fes.Size());
    nullarray = NULL;
@@ -216,23 +217,6 @@ void ParBlockNonlinearForm::SetEssentialBC(
    }
 }
 
-void ParBlockNonlinearForm::SetEssentialTrueDofs(
-   const Array<Array<int>*> &ess_tdof_list, Array<Vector*> &rhs)
-{
-   Array<Vector *> nullarray(fes.Size());
-   nullarray = nullptr;
-
-   BlockNonlinearForm::SetEssentialTrueDofs(ess_tdof_list, nullarray);
-
-   for (int s = 0; s < fes.Size(); ++s)
-   {
-      if (rhs[s])
-      {
-         rhs[s]->SetSubVector(*ess_tdofs[s], 0.0);
-      }
-   }
-}
-
 real_t ParBlockNonlinearForm::GetEnergy(const Vector &x) const
 {
    // xs_true is not modified, so const_cast is okay