Compare commits

..
Author SHA1 Message Date
camierjs 9bef1cc28f Cleanup pass 2026-03-01 15:30:39 -08:00
camierjs 6b45d9a69b Merge branch 'master' into checkpointing_examples_gpu 2026-03-01 14:38:23 -08:00
camierjs db32ae9807 Cleanup 2026-03-01 14:34:24 -08:00
camierjs 545a257ab5 LE style, compile fix 2026-02-10 12:43:39 -08:00
blaz 4b4aff736d adjoint struct 2026-02-09 00:23:04 -08:00
blaz 727339e0ae check for objective gradients 2026-02-08 22:07:05 -08:00
blaz d8d4d5b874 Merge branch 'dfem-transpose' into checkpointing_examples 2026-02-08 20:24:10 -08:00
blaz 5a7a017a4e objective integradn 2026-02-08 20:21:33 -08:00
blaz 224aff5ac9 forward part of the adjoint computations 2026-02-07 21:09:30 -08:00
blaz e42762365f adjoint initial version 2026-02-07 20:09:35 -08:00
blaz 765fb4a608 added spectral matrix preconditioner 2026-02-07 19:38:59 -08:00
John Camier 591156be2f Merge branch 'master' into dfem-transpose 2026-01-30 18:42:29 -08:00
blaz 8e2478f964 Checkpointing examples 2026-01-25 19:45:26 -08:00
blaz 06c485bb3e updates 2026-01-21 11:03:43 -08:00
bslazarov 1cd27605f0 memory transfer to Device for q-functions 2026-01-20 14:43:06 -08:00
blaz 547fd02911 anisotrpic elasticity structure 2026-01-20 09:57:29 -08:00
blaz aa92498033 explicit linear elasticity 2026-01-20 09:51:15 -08:00
blaz 024576620a initial REVOLVE 2026-01-16 11:29:25 -08:00
blaz 5b670bda3f file and segement storage 2026-01-15 22:59:32 -08:00
blaz 6b0e135bdb rename files 2026-01-15 22:18:33 -08:00
blaz 204b9df2d6 checkpinting into file - multiple files - one file per snapshot 2026-01-15 22:16:31 -08:00
blaz 8b49379f87 fixed slot in-memory or file storage for checkpointing 2026-01-15 22:15:09 -08:00
blaz d32a00fb2f CMake list 2026-01-15 19:57:10 -08:00
blaz 59c9ca70e4 initial version 2026-01-15 19:49:02 -08:00
camierjs 9646ce4d1a Fix dFEM linearized sections leaks 2026-01-12 17:10:40 -08:00
Julian Andrej f46a81bf3a clearer comment 2026-01-12 13:09:39 -08:00
Julian Andrej 460e5ba483 correct sparsematrix test 2026-01-12 13:09:04 -08:00
Julian Andrej 34593dccac properly initialize matrix pointer 2026-01-09 16:37:16 -08:00
Julian Andrej 248ab78e77 missing dual types 2026-01-09 16:04:01 -08:00
Julian Andrej 6b9bdb11df fix hypreparmatrix assemble with multiple integrators 2026-01-09 13:18:37 -08:00
Julian Andrej 505d2e5123 fix multiple integrators 2026-01-09 12:59:25 -08:00
Julian Andrej 47cff96e2e documentation 2026-01-08 10:57:34 -08:00
Julian Andrej 010f455382 documentation 2026-01-06 08:51:46 -08:00
Julian Andrej acebc322b8 cleanup 2026-01-06 08:43:02 -08:00
Julian Andrej 25ec78df9a make tests more strict 2026-01-05 16:21:58 -08:00
Julian Andrej 8c78387136 remove periodic tests for transposes 2025-12-30 13:40:59 -08:00
Julian Andrej 8684d9d5d1 working 2025-12-30 12:41:29 -08:00
Julian Andrej 2375953e95 debugging state 2025-12-29 13:35:24 -08:00
Julian Andrej ecbb196551 bug 2025-12-23 08:27:01 -08:00
Julian Andrej 90ca758a98 again 2025-12-23 07:40:55 -08:00
Julian Andrej 93d674a7a2 argument order 2025-12-23 07:31:27 -08:00
Julian Andrej 03ec3f739b correct size in parallel for ldof vectors 2025-12-23 07:27:36 -08:00
Julian Andrej a3eb9c8e6b doxygen 2025-12-22 16:30:39 -08:00
Julian Andrej 30f3e123cd more warnings 2025-12-22 16:27:42 -08:00
Julian Andrej 4d4c2325f9 warnings 2025-12-22 16:25:39 -08:00
Julian Andrej db84abedfc make dual numbers work 2025-12-22 16:21:48 -08:00
Julian Andrej 2609e3af6e transpose action using cache 2025-12-22 09:05:50 -08:00
367 changed files with 22144 additions and 55928 deletions
@@ -94,16 +94,6 @@ inputs:
description: If true, do not set any CXXFLAGS or LDFLAGS.
default: false
# Unfortunately, "uses:" fields cannot have references to variables like
# ${{env.MFEM_ACTIONS_VERSION}}, so the branch/tag name has to be hard coded.
# Therefore, in the future, when updating the version of the
# mfem/github-actions to use, we'll have to replace:
# - all definitions of MFEM_ACTIONS_VERSION and
# - all "uses:" fields that refer to mfem/github-actions.
MFEM_ACTIONS_VERSION:
description: Version (branch or tag) of the mfem/github-actions to use.
default: v2.7
runs:
using: 'composite'
steps:
@@ -128,7 +118,6 @@ runs:
echo UBSAN_LDFLAGS=${{inputs.UBSAN_LDFLAGS}} >> $GITHUB_ENV
echo MSAN_CXXFLAGS=${{inputs.MSAN_CXXFLAGS}} >> $GITHUB_ENV
echo MSAN_LDFLAGS=${{inputs.MSAN_LDFLAGS}} >> $GITHUB_ENV
echo MFEM_ACTIONS_VERSION=${{inputs.MFEM_ACTIONS_VERSION}} >> $GITHUB_ENV
shell: bash
- name: Env (dir)
+3 -3
View File
@@ -25,7 +25,7 @@ runs:
steps:
- uses: ./.github/actions/sanitize/config
- uses: actions/cache@v5
- uses: actions/cache@v4
if: ${{env.DEBUG == 'true'}}
id: debug
with:
@@ -53,7 +53,7 @@ runs:
run: echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.UBSAN_CXXFLAGS}} >> $GITHUB_ENV
shell: bash
- uses: mfem/github-actions/build-mfem@v2.7
- uses: mfem/github-actions/build-mfem@v2.5
if: ${{steps.debug.outputs.cache-hit != 'true'}}
env:
CXXFLAGS: ${{env.CXXFLAGS}}
@@ -82,7 +82,7 @@ runs:
run: find . -type f -name '*.o' -delete
shell: bash
- uses: actions/upload-artifact@v7
- uses: actions/upload-artifact@v4
with:
name: build-${{inputs.par}}-${{inputs.sanitizer}}
path: mfem/build
-6
View File
@@ -12,11 +12,6 @@
name: 'Install MPI'
description: 'Installs MPI and set up its environment variables'
inputs:
NO_FLAGS:
description: If true, do not set any CXXFLAGS or LDFLAGS.
default: false
runs:
using: 'composite'
steps:
@@ -32,7 +27,6 @@ runs:
shell: bash
- name: Env (bis)
if: ${{ inputs.NO_FLAGS != 'true' }}
run: |
echo CXXFLAGS=${{env.CXXFLAGS}} ${{env.MPI_INC}} >> $GITHUB_ENV
echo LDFLAGS=${{env.LDFLAGS}} ${{env.MPI_LIB}} >> $GITHUB_ENV
+2 -2
View File
@@ -36,7 +36,7 @@ runs:
steps:
- uses: ./.github/actions/sanitize/config
- uses: actions/cache@v5
- uses: actions/cache@v4
if: ${{env.DEBUG == 'true' && inputs.cache-skip != 'true'}}
id: debug
with:
@@ -49,7 +49,7 @@ runs:
par: ${{inputs.par}}
sanitizer: ${{inputs.sanitizer}}
- uses: actions/download-artifact@v8
- uses: actions/download-artifact@v4
with:
name: build-${{inputs.par}}-${{inputs.sanitizer}}
path: mfem/build
+7 -7
View File
@@ -23,7 +23,7 @@ inputs:
runs:
using: 'composite'
steps:
- uses: actions/cache/restore@v5 # Cache for LLVM libcxx
- uses: actions/cache/restore@v4 # Cache for LLVM libcxx
with:
path: ${{env.LLVM_DIR}}
fail-on-cache-miss: true
@@ -32,32 +32,32 @@ runs:
- uses: ./.github/actions/sanitize/mpi
if: ${{inputs.par == 'true'}}
- uses: actions/cache/restore@v5 # Cache for Hypre
- uses: actions/cache/restore@v4 # Cache for Hypre
if: ${{inputs.par == 'true'}}
with:
path: ${{env.HYPRE_DIR}}
fail-on-cache-miss: true
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{env.MFEM_ACTIONS_VERSION}}
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
- uses: actions/cache/restore@v5 # Cache for Metis
- uses: actions/cache/restore@v4 # Cache for Metis
if: ${{inputs.par == 'true'}}
with:
path: ${{env.METIS_DIR}}
fail-on-cache-miss: true
key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
- name: Hypre/Metis links
if: ${{inputs.par == 'true'}}
run: ln -s -f ${{env.HYPRE_DIR}} hypre && ln -s -f ${{env.METIS_DIR}} metis-4.0
shell: bash
- uses: actions/cache/restore@v5 # Cache for LSAN suppression file
- uses: actions/cache/restore@v4 # Cache for LSAN suppression file
with:
path: ${{env.LSAN_DIR}}
fail-on-cache-miss: true
key: build-lsan-suppression-file
- uses: actions/checkout@v6 # Checkout the repository
- uses: actions/checkout@v4 # Checkout the repository
with:
path: mfem
# ref: ${{env.BRANCH}}
-42
View File
@@ -1,42 +0,0 @@
# MFEM Pull Request Review Agent Guide
## Purpose and scope
Review MFEM PRs for correctness, maintainability, performance, portability, test coverage, and MFEM consistency. Use the diff and PR context; reference source files, tests, and CI results when available. Follow `CONTRIBUTING.md`, especially Developer Guidelines, PR rules, checklist, and testing.
## Critical review pillars
- Correctness and numerical behavior
- API and user-facing impact
- Performance implications
- Maintainability and portability
## Review workflow
1. Read the PR description, linked issues, and intended behavior.
2. Inspect the diff before commenting.
3. Identify affected MFEM components, examples, tests, build or docs changes, and downstream APIs.
4. Analyze the code against the critical review pillars.
5. Compare the change against nearby code and MFEM patterns; flag unmotivated deviations.
6. Check whether tests and documentation were updated appropriately.
7. Review CI results and suggest actions.
8. Produce a structured review with prioritized findings.
9. Always limit conclusions to available evidence.
## MFEM-specific review checklist
- Component-aware scope: identify the touched subsystem (FEM, solvers, preconditioners, linear algebra, mesh, examples, miniapps, build, or docs) and assess its impact against the review pillars.
- Numerical and algorithmic behavior: assess issues in convergence, stability, tolerances, precision, iteration limits, and failure handling. If clear opportunities exist to improve the algorithmic approach, call them out with expected impact.
- API and user-facing impact: assess backward compatibility, user-visible behavior and default changes, migration impact, deprecations, and whether documentation clearly explains user-facing API changes.
- Data structure and memory semantics: assess ownership, lifetime, aliasing, container behavior, and device-host synchronization.
- Parallel and serial behavior: assess whether the change preserves equivalent semantics in serial and parallel modes where applicable; if logic is currently mode-specific, check whether extension to the other mode is straightforward (clear abstractions, no hard-wired assumptions), document constraints, and call out expected behavior differences explicitly.
- Backend and portability impact: assess likely cross-backend risks in CPU, CUDA, HIP, OCCA, RAJA, partial assembly, fallback paths, compiler compatibility, and platform assumptions.
- Build, dependency, and configuration impact: assess CMake or make changes, optional dependency behavior, and feature-flag interactions.
- Tests and docs alignment: check available regression or unit coverage evidence for changed behavior, and ensure docs are updated for new flags, APIs, options, or behavior changes.
- MFEM developer-guideline fit: keep code lean, simple, general, logically separated, and portable; suggest C++17 improvements when they clearly improve safety, clarity, or maintainability.
- New source files, examples, or miniapps: if a PR adds source/header files, verify they are properly wired into the relevant `makefile` and `CMakeLists.txt`, referenced in docs where applicable (including `doc/CodeDocumentation.dox`), and added to top-level `.gitignore` only when generated artifacts require it.
- Changelog: verify `CHANGELOG` is updated if the PR introduces significant new features or user-facing changes.
- MFEM conventions: use `real_t`; use `mfem::out`/`mfem::err` instead of `std::cout`/`std::cerr` in library code; flag large/binary files; if AI assistance is apparent but undisclosed, suggest following `CONTRIBUTING.md`.
- Edge cases: if the PR touches complex or error-prone areas, suggest additional tests for edge cases, failure modes, and parallel behavior.
## Commenting guidelines
- Keep comments concise, actionable, and grounded in the diff.
- Focus on correctness, behavior changes, and user impact over style nits.
- Be professional, concise, collaborative, technically precise, and avoid unsupported assumptions.
+7 -3
View File
@@ -13,7 +13,7 @@ Note that some of these scripts use the shared MFEM GitHub Actions from the exte
<https://github.com/mfem/github-actions>
For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch (or tag) in the above from which the action is taken.
For a particular action, e.g. `mfem/github-actions/build-mfem@v2.5`, the `v2.5` suffix denotes the branch in the above from which the action is taken.
The current CI workflows are:
@@ -29,12 +29,16 @@ Runs a number of static repository-level sanity checks.
- `branch-history` guards against accidental commits of large files using the `--history` option of the `config/githooks/pre-push` script.
## `mfem-analysis.yml` (`build-analysis`)
Checks if the code builds and satisfies minimal requirements.
- `gitignore` builds hypre, METIS, and MFEM using `mfem/github-actions/build-hypre`, `mfem/github-actions/build-metis`, and `mfem/github-actions/build-mfem` and checks for correct `.gitignore` settings by running the `tests/scripts/gitignore` script.
## `builds-and-tests.yml`
Runs a matrix of builds and tests runs with different compilers, OS, mfem/hypre settings, etc. Also processes and upload Codecov reports.
One matrix job runs `tests/scripts/gitignore` after `make test-noclean` to check generated artifacts against `.gitignore`.
Uses the following GitHub Actions from <https://github.com/mfem/github-actions>:
- `mfem/github-actions/build-hypre`
+1 -1
View File
@@ -43,7 +43,7 @@ jobs:
remove-docker-images: 'true'
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v4
# It's easier to reference named variables than indexes of the matrix
- name: Set Environment
+28 -31
View File
@@ -40,7 +40,6 @@ env:
METIS_ARCHIVE_MAC: metis-4.0.3-mac.tgz
METIS_TOP_DIR: metis-4.0.3
MFEM_TOP_DIR: mfem
MFEM_ACTIONS_VERSION: v2.7
# Note for future improvements:
#
@@ -111,7 +110,6 @@ jobs:
build-system: make
hypre-target: int64
precision: fp64
gitignore-check: YES
- os: ubuntu-latest
target: opt
codecov: NO
@@ -155,7 +153,7 @@ jobs:
# /home/runner/work/mfem/mfem/mfem
# Note: Done now to access "install-hypre" and "install-metis" actions.
- name: checkout mfem
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
path: ${{ env.MFEM_TOP_DIR }}
# Fetch the complete history for codecov to access commits ID
@@ -172,6 +170,20 @@ jobs:
env
shell: bash
# For info on Xcode see:
# - https://github.com/actions/runner-images/issues/12541
# - https://github.com/actions/runner-images/blob/releases/macos-15-arm64/20250811/images/macos/macos-15-arm64-Readme.md#xcode
- name: Xcode version setup (MacOS)
if: matrix.os == 'macos-latest'
run: |
XCODE_PATH="/Applications/Xcode_16.4.app"
echo "> sudo xcode-select -s ${XCODE_PATH}"
sudo xcode-select -s ${XCODE_PATH}
echo "> g++ -v"
g++ -v
echo "> clang++ -v"
clang++ -v
# Only get MPI if defined for the job.
# TODO: It would be nice to have only one step, e.g. with a dedicated
# action, but I (@adrienbernede) don't see how at the moment.
@@ -213,14 +225,14 @@ jobs:
- name: cache hypre
id: hypre-cache
if: matrix.mpi == 'par'
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: ${{ env.HYPRE_TOP_DIR }}
key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-${{ env.MFEM_ACTIONS_VERSION }}
key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-${{ matrix.hypre-target }}-${{ matrix.precision }}-v2.5
- name: get hypre
if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os != 'windows-latest'
uses: mfem/github-actions/build-hypre@v2.7
uses: mfem/github-actions/build-hypre@v2.5
with:
archive: ${{ env.HYPRE_ARCHIVE }}
dir: ${{ env.HYPRE_TOP_DIR }}
@@ -230,7 +242,7 @@ jobs:
- name: get hypre (Windows)
if: matrix.mpi == 'par' && steps.hypre-cache.outputs.cache-hit != 'true' && matrix.os == 'windows-latest'
uses: mfem/github-actions/build-hypre@v2.7
uses: mfem/github-actions/build-hypre@v2.5
with:
archive: ${{ env.HYPRE_ARCHIVE }}
dir: ${{ env.HYPRE_TOP_DIR }}
@@ -243,14 +255,14 @@ jobs:
- name: cache metis
id: metis-cache
if: matrix.mpi == 'par' && matrix.os != 'windows-latest'
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: ${{ env.METIS_TOP_DIR }}
key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-${{ env.MFEM_ACTIONS_VERSION }}
key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
- name: install metis
if: matrix.mpi == 'par' && matrix.os != 'windows-latest' && steps.metis-cache.outputs.cache-hit != 'true'
uses: mfem/github-actions/build-metis@v2.7
uses: mfem/github-actions/build-metis@v2.5
with:
archive: ${{ matrix.os != 'macos-latest' && env.METIS_ARCHIVE || env.METIS_ARCHIVE_MAC }}
dir: ${{ env.METIS_TOP_DIR }}
@@ -258,7 +270,7 @@ jobs:
- name: cache vcpkg (Windows)
id: vcpkg-cache
if: matrix.os == 'windows-latest'
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: vcpkg_cache
key: ${{ runner.os }}-${{ matrix.mpi }}-vcpkg-v1
@@ -283,8 +295,7 @@ jobs:
export HOMEBREW_NO_INSTALL_CLEANUP=1
brew update
brew install enzyme
ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required.*:.*\(llvm[^ ]*\).*/\1/p')
echo "ENZYME_LLVM=$ENZYME_LLVM"
ENZYME_LLVM=$(brew info enzyme | sed -n 's/^Required:.*\(llvm[^ ]*\).*/\1/p')
LLVM_PREFIX=$(brew --prefix $ENZYME_LLVM)
echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV
echo "OMPI_CC=$LLVM_PREFIX/bin/clang" >> $GITHUB_ENV
@@ -292,7 +303,7 @@ jobs:
# MFEM build and test
- name: build
uses: mfem/github-actions/build-mfem@v2.7
uses: mfem/github-actions/build-mfem@v2.5
env:
VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}/vcpkg_cache
with:
@@ -318,13 +329,7 @@ jobs:
- name: tests
if: matrix.build-system == 'make' && (matrix.target == 'opt' || matrix.os == 'ubuntu-latest')
run: |
cd ${{ env.MFEM_TOP_DIR }}
if [[ "${{ matrix.gitignore-check }}" == "YES" ]]; then
make test-noclean
else
make test
fi
shell: bash
cd ${{ env.MFEM_TOP_DIR }} && make test
- name: cmake checks
if: matrix.build-system == 'cmake' && matrix.target == 'dbg'
@@ -369,16 +374,8 @@ jobs:
# Code coverage (process and upload reports)
- name: codecov
if: matrix.codecov == 'YES'
uses: mfem/github-actions/upload-coverage@v2.7
uses: mfem/github-actions/upload-coverage@v2.5
with:
name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}-${{ matrix.precision }}
name: ${{ matrix.os }}-${{ matrix.build-system }}-${{ matrix.target }}-${{ matrix.mpi }}-${{ matrix.hypre-target }}
project_dir: ${{ env.MFEM_TOP_DIR }}
directories: "fem general linalg mesh"
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: gitignore
if: matrix.gitignore-check == 'YES'
run: |
cd ${{ env.MFEM_TOP_DIR }}/tests/scripts
./runtest gitignore
+4 -14
View File
@@ -14,19 +14,9 @@ name: "Static Analysis"
on:
push:
branches: ["master", "next"]
paths-ignore: &docs-only-paths
- "**/*.md"
- "doc/**"
- ".binder/**"
- "CITATION.cff"
- "LICENSE"
- "NOTICE"
- "CHANGELOG"
- "INSTALL"
pull_request:
# The branches below must be a subset of the branches above
branches: ["master"]
paths-ignore: *docs-only-paths
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
@@ -50,11 +40,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -67,7 +57,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v4
uses: github/codeql-action/autobuild@v2
# ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -80,4 +70,4 @@ jobs:
# ./location_of_script_within_repo/buildscript.sh
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4
uses: github/codeql-action/analyze@v2
+100
View File
@@ -0,0 +1,100 @@
# Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
# LICENSE and NOTICE for details. LLNL-CODE-806117.
#
# This file is part of the MFEM library. For more information and source code
# availability visit https://mfem.org.
#
# MFEM is free software; you can redistribute it and/or modify it under the
# terms of the BSD-3 license. We welcome feedback and contributions, see file
# CONTRIBUTING.md for details.
name: "Build Analysis"
permissions:
actions: write
on:
push:
branches:
- master
- next
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
HYPRE_ARCHIVE: v2.19.0.tar.gz
HYPRE_TOP_DIR: hypre-2.19.0
METIS_ARCHIVE: metis-4.0.3.tar.gz
METIS_TOP_DIR: metis-4.0.3
COVERAGE_ENV: mfem-coverage
jobs:
gitignore:
runs-on: ubuntu-latest
steps:
- name: checkout MFEM
uses: actions/checkout@v4
with:
path: mfem
- name: Get MPI (Linux)
run: |
sudo apt-get install openmpi-bin libopenmpi-dev
export OMPI_MCA_rmaps_base_oversubscribe=1
- name: Cache Hypre Install
id: hypre-cache
uses: actions/cache@v4
with:
path: ${{ env.HYPRE_TOP_DIR }}
key: ${{ runner.os }}-ompi-build-${{ env.HYPRE_TOP_DIR }}-v2.5
- name: Get Hypre
if: steps.hypre-cache.outputs.cache-hit != 'true'
uses: mfem/github-actions/build-hypre@v2.5
with:
archive: ${{ env.HYPRE_ARCHIVE }}
dir: ${{ env.HYPRE_TOP_DIR }}
target: int32
- name: Cache Metis Install
id: metis-cache
uses: actions/cache@v4
with:
path: ${{ env.METIS_TOP_DIR }}
key: ${{ runner.os }}-build-${{ env.METIS_TOP_DIR }}-v2.5
- name: Install Metis
if: steps.metis-cache.outputs.cache-hit != 'true'
uses: mfem/github-actions/build-metis@v2.5
with:
archive: ${{ env.METIS_ARCHIVE }}
dir: ${{ env.METIS_TOP_DIR }}
# MFEM build and test
- name: build-mfem
uses: mfem/github-actions/build-mfem@v2.5
with:
os: ${{ runner.os }}
target: opt
codecov: NO
mpi: par
build-system: make
hypre-dir: ${{ env.HYPRE_TOP_DIR }}
metis-dir: ${{ env.METIS_TOP_DIR }}
mfem-dir: mfem
- name: test (no clean)
run: |
cd mfem && make test-noclean
- name: gitignore
run: |
cd mfem/tests/scripts
./runtest gitignore
+4 -4
View File
@@ -38,7 +38,7 @@ jobs:
github.event.pull_request.head.repo.full_name != github.repository)
steps:
- name: checkout mfem
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: copyright check
id: copyright
@@ -93,7 +93,7 @@ jobs:
github.event.pull_request.head.repo.full_name != github.repository)
steps:
- name: checkout mfem
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: get astyle
run: |
@@ -110,7 +110,7 @@ jobs:
github.event.pull_request.head.repo.full_name != github.repository)
steps:
- name: checkout mfem
uses: actions/checkout@v6
uses: actions/checkout@v4
- name: get doxygen and graphviz
run: |
@@ -135,7 +135,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: checkout mfem
uses: actions/checkout@v6
uses: actions/checkout@v4
with:
fetch-depth: 0
+4 -8
View File
@@ -17,24 +17,20 @@ jobs:
runs-on: ubuntu-latest
name: 2.19.0
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/config
with:
NO_FLAGS: true
- name: Cache
id: cache
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: ${{env.HYPRE_DIR}}
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-${{ env.MFEM_ACTIONS_VERSION }}
key: ${{runner.os}}-ompi-build-${{env.HYPRE_DIR}}-int32-fp64-v2.5
- name: Setup
if: steps.cache.outputs.cache-hit != 'true'
uses: ./.github/actions/sanitize/mpi
with:
NO_FLAGS: true
- name: Build
if: steps.cache.outputs.cache-hit != 'true'
uses: mfem/github-actions/build-hypre@v2.7
uses: mfem/github-actions/build-hypre@v2.5
with:
archive: ${{env.HYPRE_TGZ}}
dir: ${{env.HYPRE_DIR}}
+2 -2
View File
@@ -27,13 +27,13 @@ jobs:
llvm_use_sanitizer: "Undefined"
name: ${{matrix.sanitizer}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/config
with:
NO_FLAGS: true
- name: Cache
id: cache
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: ${{env.LLVM_DIR}}
key: build-libcxx-${{env.LLVM_VER}}-${{matrix.sanitizer}}
+2 -2
View File
@@ -17,11 +17,11 @@ jobs:
runs-on: ubuntu-latest
name: lsan.supp
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/config
- name: Cache
id: cache
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: ${{env.LSAN_DIR}}
key: build-lsan-suppression-file
+4 -8
View File
@@ -17,24 +17,20 @@ jobs:
runs-on: ubuntu-latest
name: 4.0.3
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/config
with:
NO_FLAGS: true
- name: Cache
id: cache
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: ${{env.METIS_DIR}}
key: ${{runner.os}}-build-${{env.METIS_DIR}}-${{env.MFEM_ACTIONS_VERSION}}
key: ${{runner.os}}-build-${{env.METIS_DIR}}-v2.5
- name: Setup
if: steps.cache.outputs.cache-hit != 'true'
uses: ./.github/actions/sanitize/mpi
with:
NO_FLAGS: true
- name: Build
if: steps.cache.outputs.cache-hit != 'true'
uses: mfem/github-actions/build-metis@v2.7
uses: mfem/github-actions/build-metis@v2.5
with:
archive: ${{env.METIS_TGZ}}
dir: ${{env.METIS_DIR}}
+9 -9
View File
@@ -28,7 +28,7 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/mfem
with:
par: ${{inputs.par}}
@@ -40,7 +40,7 @@ jobs:
env:
ex: ${{inputs.par && 'ex1p' || 'ex1'}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/restore
id: restore
with:
@@ -58,7 +58,7 @@ jobs:
env:
exclude: ${{inputs.par && '-E "_ser"' || ''}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/restore
id: restore
with:
@@ -82,7 +82,7 @@ jobs:
env:
exclude: ${{inputs.par && '-E "_ser"' || ''}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/restore
id: restore
with:
@@ -107,7 +107,7 @@ jobs:
run: ${{inputs.par && '-R "_cpu_np"' || ''}}
exclude: ${{inputs.par && '"unit_tests|debug"' || '"^unit_tests$|debug"'}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/restore
id: restore
with:
@@ -131,7 +131,7 @@ jobs:
env:
unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/restore
id: restore
with:
@@ -146,7 +146,7 @@ jobs:
if: ${{steps.restore.outputs.cache-hit != 'true'}}
working-directory: mfem/build/tests/unit
run: find . -type f -name '*.o' -delete
- uses: actions/upload-artifact@v7
- uses: actions/upload-artifact@v4
with:
name: tests-${{inputs.par}}-${{inputs.sanitizer}}
path: mfem/build/tests/unit/${{env.unit_tests}}
@@ -165,14 +165,14 @@ jobs:
unit_tests: ${{inputs.par && 'punit_tests' || 'unit_tests'}}
np: ${{inputs.par && '_np=2' || ''}}
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v4
- uses: ./.github/actions/sanitize/restore
id: restore
with:
par: ${{inputs.par}}
sanitizer: ${{inputs.sanitizer}}
cache-path: mfem/build/tests/unit/${{env.unit_tests}}
- uses: actions/download-artifact@v8
- uses: actions/download-artifact@v4
if: ${{steps.restore.outputs.cache-hit != 'true'}}
with:
name: tests-${{inputs.par}}-${{inputs.sanitizer}}
-10
View File
@@ -17,17 +17,7 @@ permissions:
on:
push:
branches: ["master", "next"]
paths-ignore: &docs-only-paths
- "**/*.md"
- "doc/**"
- ".binder/**"
- "CITATION.cff"
- "LICENSE"
- "NOTICE"
- "CHANGELOG"
- "INSTALL"
pull_request:
paths-ignore: *docs-only-paths
workflow_dispatch:
concurrency:
-4
View File
@@ -443,10 +443,6 @@ miniapps/diag-smoothers/mg-abs-l1-jacobi
miniapps/contact/contact
miniapps/contact/ParaView
miniapps/plasma/pic/electrostatic-*
!miniapps/plasma/pic/electrostatic-*.cpp
miniapps/plasma/pic/*.csv
# Unit test binary and outputs
tests/unit/output_meshes
tests/unit/unit_tests
-5
View File
@@ -85,8 +85,3 @@ opt_par_gcc_10_pumi:
extends: .mfem_job_on_dane
variables:
SPEC: "%gcc@10.3.1 +pumi"
opt_par_gcc_10_gslib:
extends: .mfem_job_on_dane
variables:
SPEC: "%gcc@10.3.1 +gslib"
-5
View File
@@ -63,8 +63,3 @@ opt_mpi_cuda_hypre_cuda_gcc:
extends: .mfem_job_on_matrix
variables:
SPEC: "%gcc@10.3.1 +mpi +cuda cuda_arch=90 ^hypre+cuda"
opt_mpi_cuda_gcc_gslib:
extends: .mfem_job_on_matrix
variables:
SPEC: "%gcc@10.3.1 +mpi +cuda +gslib cuda_arch=90 ^hypre+cuda"
+2 -2
View File
@@ -32,9 +32,9 @@ mkdir _${BASELINE_TEST} && cd _${BASELINE_TEST}
# run
if [[ "${MACHINE_NAME}" == "dane" ]]; then
srun --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
salloc --nodes=1 -t 60 --exclusive --reservation=ci ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
elif [[ ${MACHINE_NAME} == "corona" ]]; then
srun --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
salloc --nodes=1 -t 60 -p pbatch ../runtest ../../mfem "${BASELINE_TEST} ${TPLS_DIR}"
else
echo "Unknown machine: MACHINE_NAME=$MACHINE_NAME"
exit 1
+5 -72
View File
@@ -11,91 +11,24 @@
Version 4.9.1 (development)
===========================
- Added policy for AI-assisted contribution to CONTRIBUTING.md.
Discretization improvements
---------------------------
- Improved FindPointsGSLIB surface mesh capability with support for simplices
and an option to specify axis-aligned bounding box padding for near-surface
point queries.
- Added GPU-enabled partial assembly for simplicial Bernstein H1 basis based on
ragged tensor algorithms (see DOI: 10.1137/11082539X) for mass and diffusion
integrators.
- Replaced legacy simplex quadrature rules with symmetric positive weight rules
for triangles (orders 0-25) and tetrahedra (orders 0-20). These rules
guarantee all-positive weights and interior quadrature points, improving
numerical stability. Higher orders fall back to Grundmann-Moller.
* Triangle rules: Witherden and Vincent, DOI: 10.1016/j.camwa.2015.03.017
* Tet rules (d=1-13): Witherden and Vincent (same as above)
* Tet rules (d=14-20): Chuluunbaatar et al., DOI: 10.1016/j.camwa.2022.08.016
- Added support for general 1D Gauss-Jacobi quadrature rules and Stroud conical
quadrature rules on triangles and tetrahedra.
- Improved the GridFunction projection routines. Projections work for Scalar,
- Improved the gridfunction projection routines. Projections work for Scalar,
Vector and VectorFE, also NURBS versions. Optionally different types of
projections can be selected, default behavior has not changed.
projections can be selected, default behaviour has not changed.
- Added GridFunction projection methods for trace spaces, i.e., project
coefficients on the mesh skeleton.
- Added methods to estimate function extremum using piecewise linear bounds plus
- Added methods to estimate function extremum using piecewise linear bounds +
recursive subdivision.
<<<<<<<<< Temporary merge branch 1
- Added class VectorQuadratureSpace that represents a vector (multi-component)
version of the scalar (single-component) class QuadratureSpaceBase.
=========
- Extend FindPointsGSLIB to support surface meshes.
>>>>>>>>> Temporary merge branch 2
Meshing improvements
--------------------
- Added option to guarantee mesh validity during TMOP-based r-adaptivity, using
bounds on the determinant of the mesh transformation Jacobian.
- Added PA support for TMOP's adaptive limiting functionality. Multiple
GridFunctions and Coefficients can be combined to form a composite term.
- Improved support for 1D NURBS meshes with variable order, including using
the patches construct for 1D NURBS meshes.
- Added the option to include material interfaces (faces separating elements
with different element attributes) as additional boundary elements, for
parallel visualization, e.g. with GLVis. This is supported by both the Print
and PrintAsOne methods of ParMesh. See ParMesh::SetPrintInterfaces().
Linear and nonlinear solvers
----------------------------
- Added support for trace spaces in PRefinementTransferOperator. This is used in
PRefinement multigrid methods for problems posed on trace spaces (see e.g. the
DPG miniapps).
GPU computing
-------------
- Added device assembly support for 3D H(curl) VectorFEDomainLFIntegrator.
- Added NVIDIA cuDSS library interface. Implementation examples have been
added to ex1 and ex1p. See https://developer.nvidia.com/cudss for more
details. Supported versions >= 0.6.0.
- Allow specifying GPU kernel launch bounds for native and RAJA GPU backends.
New and updated examples and miniapps
-------------------------------------
- The Lorentz miniapp (in miniapps/electromagnetics) has been updated to
leverage the ParticleSet capability.
- Added (Complex)PRefinementMultigrid solver option in the DPG miniapps.
Miscellaneous
-------------
- Fixed signed DOF handling in ParGridFunction reading (read constructor) and
saving via SaveAsOne(). Simplified the process of applying the DOF signs by
using the new method ApplyDofSigns() in class ParFiniteElementSpace: the
method will return immediately if no sign flips are needed.
- Electromagnetics/lorentz miniapp has been updated to leverage the ParticleSet
capability.
Version 4.9, released on Dec 11, 2025
+13 -42
View File
@@ -433,15 +433,6 @@ if (MFEM_USE_STRUMPACK)
endif()
endif()
# cuDSS can only be enabled in CUDA
if (MFEM_USE_CUDSS)
if (MFEM_USE_CUDA)
find_package(CUDSS REQUIRED)
else()
message(FATAL_ERROR " *** cuDSS requires that CUDA be enabled.")
endif()
endif()
# GnuTLS
if (MFEM_USE_GNUTLS)
find_package(_GnuTLS REQUIRED)
@@ -601,13 +592,6 @@ if (MFEM_USE_ENZYME)
set(ENZYME_INCLUDE_DIRS ${ENZYME_DIR}/include)
endif()
if (MFEM_USE_PROTEUS)
enable_language(C)
find_package(proteus REQUIRED PATHS "${PROTEUS_DIR}")
message(STATUS "${PROTEUS_DIR}/include")
include_directories("${PROTEUS_DIR}/include")
endif()
# MFEM_TIMER_TYPE
if (NOT DEFINED MFEM_TIMER_TYPE)
if (APPLE)
@@ -647,7 +631,7 @@ find_package(Threads REQUIRED)
set(MFEM_TPLS OPENMP HYPRE LAPACK BLAS SuperLUDist STRUMPACK METIS SuiteSparse
SUNDIALS PETSC SLEPC MUMPS AXOM FMS CONDUIT Ginkgo GNUTLS GSLIB HDF5
NETCDF MPFR PUMI HIOP POSIXCLOCKS MFEMBacktrace ZLIB OCCA CEED RAJA UMPIRE
ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CUDSS CALIPER CODIPACK
ADIOS2 MKL_CPARDISO MKL_PARDISO AMGX MAGMA CUSPARSE CUBLAS CALIPER CODIPACK
BENCHMARK PARELAG TRIBOL MPI_CXX HIP HIPBLAS HIPSPARSE MOONOLITH BLITZ
ALGOIM ENZYME CUDA::cudart)
@@ -668,8 +652,6 @@ foreach(TPL IN LISTS MFEM_TPLS)
endif()
endforeach(TPL)
# reverse to remove the first instance of entries in TPL_LIBRARIES
# so later duplicates are kept (for dependency ordering)
list(REVERSE TPL_LIBRARIES)
list(REMOVE_DUPLICATES TPL_LIBRARIES)
list(REVERSE TPL_LIBRARIES)
@@ -744,16 +726,6 @@ mfem_add_library(mfem ${SOURCES} ${HEADERS} ${MASTER_HEADERS})
target_compile_features(mfem PUBLIC cxx_std_${CMAKE_CXX_STANDARD})
# message(STATUS "TPL_LIBRARIES = ${TPL_LIBRARIES}")
target_link_libraries(mfem PUBLIC ${TPL_LIBRARIES} ${TPL_TARGETS})
if (MFEM_USE_PROTEUS)
add_library(ClangProteusFlags INTERFACE IMPORTED)
set_target_properties(ClangProteusFlags PROPERTIES
INTERFACE_COMPILE_OPTIONS "-fpass-plugin=$<TARGET_FILE:ProteusPass>"
)
target_link_libraries(mfem PUBLIC ClangProteusFlags)
target_link_libraries(mfem PUBLIC proteus)
endif()
if (TPL_TARGETS)
add_dependencies(mfem ${TPL_TARGETS})
endif()
@@ -790,7 +762,7 @@ set_target_properties(mfem PROPERTIES SOVERSION "${mfem_VERSION}")
# If building out-of-source, define MFEM_CONFIG_FILE to point to the config file
# inside the build directory.
if (NOT ("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}"))
target_compile_definitions(mfem PUBLIC
target_compile_definitions(mfem PRIVATE
"MFEM_CONFIG_FILE=\"${PROJECT_BINARY_DIR}/config/_config.hpp\"")
endif()
@@ -850,15 +822,16 @@ if (MFEM_ENABLE_TESTING)
add_mfem_target(${MFEM_ALL_TESTS_TARGET_NAME} OFF)
add_subdirectory(tests EXCLUDE_FROM_ALL)
# Create a target for all benchmarks and, optionally, enable it. Some simple
# timer-based benchmarks in tests/benchmarks do not require Google Benchmark.
set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
${MFEM_ENABLE_BENCHMARKS})
if (MFEM_ENABLE_BENCHMARKS)
add_subdirectory(tests/benchmarks) #install benchmarks if enabled
else()
add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
if (MFEM_USE_BENCHMARK)
# Create a target for all benchmarks and, optionally, enable it.
set(MFEM_ALL_BENCHMARKS_TARGET_NAME benchmarks)
add_mfem_target(${MFEM_ALL_BENCHMARKS_TARGET_NAME}
${MFEM_ENABLE_BENCHMARKS})
if (MFEM_ENABLE_BENCHMARKS)
add_subdirectory(tests/benchmarks) #install benchmarks if enabled
else()
add_subdirectory(tests/benchmarks EXCLUDE_FROM_ALL)
endif()
endif()
endif()
@@ -1042,7 +1015,5 @@ install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/data
# Create 'config.mk' from 'config.mk.in' for the build and install locations and
# define install rules for 'config.mk' and 'test.mk'
#-------------------------------------------------------------------------------
if (MFEM_USE_CUDA OR MFEM_USE_HIP)
option(MFEM_EXPORT_GPU_CONFIG "Export config.mk for GPU-enabled downstream packages" ON)
endif()
mfem_export_mk_files()
+65 -72
View File
@@ -3,12 +3,12 @@
</p>
<p align="center">
<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-blue.svg"></a>
<a href="https://github.com/mfem/mfem/releases/latest"><img alt="GitHub release" src="https://img.shields.io/github/v/release/mfem/mfem"></a>
<a href="https://github.com/mfem/mfem/actions/workflows/repo-check.yml?query=branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
<a href="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml?query=branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
<a href="https://github.com/mfem/mfem/blob/master/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-BSD-brightgreen.svg"></a>
<a href="https://github.com/mfem/mfem/actions?query=workflow%3Arepo-check+branch%3Amaster"><img alt="Repo check" src="https://github.com/mfem/mfem/actions/workflows/repo-check.yml/badge.svg?branch=master"></a>
<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuild-analysis+branch%3Amaster"><img alt="Build Analysis" src="https://github.com/mfem/mfem/actions/workflows/mfem-analysis.yml/badge.svg?branch=master"></a>
<a href="https://github.com/mfem/mfem/actions?query=workflow%3Abuilds-and-tests+branch%3Amaster"><img alt="Builds and Tests" src="https://github.com/mfem/mfem/actions/workflows/builds-and-tests.yml/badge.svg?branch=master"></a>
<a href="https://ci.appveyor.com/project/mfem/mfem"><img alt="Build Status" src="https://ci.appveyor.com/api/projects/status/19non9sqm6msi2wy?svg=true"></a>
<a href="https://docs.mfem.org/html/index.html"><img alt="Documentation" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
<a href="https://docs.mfem.org/html/index.html"><img alt="Doxygen" src="https://img.shields.io/badge/code-documented-brightgreen.svg"></a>
</p>
@@ -24,14 +24,6 @@ must be made under this license.
Note also that MFEM has a [Code of Conduct](CODE_OF_CONDUCT.md). By participating
in the MFEM community, you agree to abide by its rules.
## AI Policy
- Use of AI code generation in MFEM is allowed but must be disclosed, e.g. by
selecting the `AI-assisted` label on the PR.
- By submitting a PR, the author acknowledges that they have reviewed and
understand the changes they are proposing.
- PR authors are still responsible for correctness, licensing, and attribution
of all changes.
If you plan on contributing to MFEM, consider reviewing the
[issue tracker](https://github.com/mfem/mfem/issues) first to check if a thread
already exists for your desired feature or the bug you ran into. Use a pull
@@ -84,7 +76,7 @@ Origin](#developers-certificate-of-origin-11) at the end of this file.*
follow the [MFEM PR Rules](#mfem-pr-rules).
- When your contribution is fully working and ready to be reviewed, add
the `ready-for-review` label.
- PRs are treated similarly to journal submission, with an "editor" assigning two
- PRs are treated similarly to journal submission with an "editor" assigning two
reviewers to evaluate the changes.
- The reviewers have 3 weeks to evaluate the PR and work with the author to
fix issues and implement improvements.
@@ -125,7 +117,7 @@ The MFEM source code has the following structure:
│ ├── petsc
│ ├── pumi
│ ├── sundials
└── superlu
| └── superlu
├── fem
│ ├── ceed
│ ├── dfem
@@ -137,6 +129,10 @@ The MFEM source code has the following structure:
│ ├── moonolith
│ ├── qinterp
│ └── tmop
│ | ├── assemble
│ | ├── metrics
│ | ├── mult
│ | └── tools
├── general
├── linalg
│ ├── batched
@@ -149,10 +145,11 @@ The MFEM source code has the following structure:
│ ├── common
│ ├── contact
│ ├── dfem
│ ├── diag-smoothers
│ ├── dpg
│ ├── electromagnetics
│ ├── fluids
│ │ ├── navier
│ │ └── schrodinger-flow
│ ├── gslib
│ ├── hdiv-linear-solver
│ ├── hooke
@@ -162,7 +159,6 @@ The MFEM source code has the following structure:
│ ├── nurbs
│ ├── parelag
│ ├── performance
│ ├── plasma
│ ├── shifted
│ ├── solvers
│ ├── spde
@@ -193,15 +189,15 @@ respectively.
- The main finite element classes are:
+ [`FiniteElement`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
+ [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElementCollection.html)
+ [`FiniteElementCollection`](https://docs.mfem.org/html/classmfem_1_1FiniteElement.html)
+ [`FiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1FiniteElementSpace.html)
+ [`GridFunction`](https://docs.mfem.org/html/classmfem_1_1GridFunction.html)
+ [`BilinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1BilinearFormIntegrator.html) and [`LinearFormIntegrator`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html)
+ [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
+ [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearFormIntegrator.html), [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`MixedBilinearForm`](https://docs.mfem.org/html/classmfem_1_1MixedBilinearForm.html)
- The main linear algebra classes and sources are
+ [`Operator`](https://docs.mfem.org/html/classmfem_1_1Operator.html) and [`BilinearForm`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html)
+ [`Vector`](https://docs.mfem.org/html/classmfem_1_1Vector.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
+ [`Vector`](https://docs.mfem.org/html/classmfem_1_1BilinearForm.html) and [`LinearForm`](https://docs.mfem.org/html/classmfem_1_1LinearForm.html)
+ [`DenseMatrix`](https://docs.mfem.org/html/classmfem_1_1DenseMatrix.html) and [`SparseMatrix`](https://docs.mfem.org/html/classmfem_1_1SparseMatrix.html)
+ Sparse [smoothers](https://docs.mfem.org/html/sparsesmoothers_8hpp.html) and linear [solvers](https://docs.mfem.org/html/solvers_8hpp.html)
@@ -213,8 +209,8 @@ shared geometric entities between different tasks. The parallel source files
have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
- The main parallel classes are
+ [`ParMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
+ [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParNCMesh.html)
+ [`ParMesh`](https://docs.mfem.org/html/solvers_8hpp.html)
+ [`ParNCMesh`](https://docs.mfem.org/html/classmfem_1_1ParMesh.html)
+ [`ParFiniteElementSpace`](https://docs.mfem.org/html/classmfem_1_1ParFiniteElementSpace.html)
+ [`ParGridFunction`](https://docs.mfem.org/html/classmfem_1_1ParGridFunction.html)
+ [`ParBilinearForm`](https://docs.mfem.org/html/classmfem_1_1ParBilinearForm.html) and [`ParLinearForm`](https://docs.mfem.org/html/classmfem_1_1ParLinearForm.html)
@@ -224,14 +220,14 @@ have a `p` prefix, e.g. `pmesh.cpp` vs. the serial `mesh.cpp`.
#### GPU and general device support
GPU and multi-core CPU support is based on device kernels supporting different
backends (CUDA, HIP, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
backends (CUDA, OCCA, RAJA, OpenMP, etc.) and an internal lightweight
device/host memory manager.
- The main device-relevant classes and sources are:
+ [`Device`](https://docs.mfem.org/html/device_8hpp.html)
+ [`MemoryManager`](https://docs.mfem.org/html/mem_manager_8hpp.html)
+ the [`mfem::forall`](https://docs.mfem.org/html/forall_8hpp.html) function
+ the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html), [`hip.hpp`](https://docs.mfem.org/html/hip_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
+ the [`cuda.hpp`](https://docs.mfem.org/html/cuda_8hpp.html) and [`occa.hpp`](https://docs.mfem.org/html/occa_8hpp.html) files
#### Utilities, building and documentation
- The `general/` directory contains C++ classes that serve as utilities for
@@ -245,8 +241,8 @@ device/host memory manager.
- `examples` and `miniapps` respectively gather simple and more fully-featured
demonstrations of the usage on MFEM. They both rely on `data/` for the
collection of meshes.
- The `tests/` directory contains a unit test suite, additional tests, and
benchmarks.
- The `tests/` directory contains a unit test suite and will later contain more
tests that run example codes.
See also the [code overview](https://mfem.org/code-overview/) section on the MFEM
website.
@@ -280,8 +276,8 @@ Before you can start, you need a GitHub account, here are a few suggestions:
the top of https://github.com/mfem.
- Consider making your membership public by going to https://github.com/orgs/mfem/people
and clicking on the organization visibility drop box next to your name.
- Project discussions and announcements will be posted at https://github.com/orgs/mfem/discussions,
tagging the `@mfem/everyone` team when appropriate.
- Project discussions and announcements will be posted at
https://github.com/orgs/mfem/teams/everyone.
#### Structure
- The MFEM source code is in the [mfem](https://github.com/mfem/mfem)
@@ -341,12 +337,11 @@ Before you can start, you need a GitHub account, here are a few suggestions:
- Well-designed simple code is frequently more general and powerful.
- Lean code base is easier to understand by new collaborators.
- New features should be added only if they are necessary or generally useful.
- Introduction of language constructs not currently used in MFEM should be
- Introduction of language constructions not currently used in MFEM should be
justified and generally avoided (to maintain portability to various systems
and compilers, including early access hardware).
- We prefer basic C++. Use C++17 features judiciously, prioritizing readability,
consistency with existing MFEM code, and portability to different systems,
compilers and device backends.
- We prefer basic C++ and the C++03 standard, to keep the code readable by
a large audience and to make sure it compiles anywhere.
- *Keep the code general and reasonably efficient*
- The main goal is fast prototyping for research and application development.
@@ -389,7 +384,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
- When your branch is ready for other developers to review / comment on
the code, create a pull request towards `mfem:master`.
- Pull requests typically have titles like:
- Pull request typically have titles like:
`Description [new-feature-dev]`
@@ -410,12 +405,12 @@ Before you can start, you need a GitHub account, here are a few suggestions:
- Add a description, appropriate labels and assign yourself to the PR. The MFEM
team will add reviewers as appropriate.
- List outstanding TODO items in the description.
- List outstanding TODO items in the description, see PR #222 for an example.
- When your contribution is fully working and ready to be reviewed, add
or request the `ready-for-review` label.
the `ready-for-review` label.
- PRs are treated similarly to journal submission, with an "editor" assigning
- PRs are treated similarly to journal submission with an "editor" assigning
two reviewers to evaluate the changes. The reviewers have 3 weeks to evaluate
the PR and work with the author to implement improvements and fix issues.
@@ -441,7 +436,7 @@ Before you can start, you need a GitHub account, here are a few suggestions:
checks in GitHub Actions enforce MFEM-specific rules which are explained in
the error messages and the `tests/scripts` directory.
- Also note that the tests `branch-history` and `repo-check` found in GitHub
- Also note that the tests `branch-history` and `repos-checks` found in GitHub
Actions can be triggered automatically before each push using git hooks. See
the [git hooks README](config/githooks/README.md) for a detailed explanation.
@@ -498,15 +493,15 @@ Everyone on the MFEM team can be asked to serve as a reviewer on a PR in their a
3. To ensure the quality of the PR by making sure that the code adheres to the [Developer Guidelines](#developer-guidelines), e.g. all methods, data members, and functions have documentation, including data ownership and lifetime, new examples/miniapps have a corresponding PR in mfem/web, major features have `CHANGELOG` entries, etc.
4. To seek help from the editors in case of difficulties.
3. To seek help from the editors in case of difficulties.
5. To complete the review in a timely manner: 3 weeks from assignment.
4. To complete the review in a timely manner: 3 weeks from assignment.
6. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
5. To test the PR thoroughly before merging in *next*. The PR author is also encouraged to perform testing and inform the reviewers about the results.
7. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
6. To monitor the PR impact on the testing in the *next* branch and alert the editors that the PR is ready for merging in *master*.
8. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
7. The review of bugfixes should be expedited proportional to their importance. The review window can be much less than three weeks in such cases.
#### Responsibilities of Authors
@@ -532,30 +527,30 @@ Before a PR can be merged, it should satisfy the following:
- [ ] Code builds.
- [ ] Code passes `make style`.
- [ ] Update `CHANGELOG`:
- [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
- [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
- [ ] Is this a new feature users need to be aware of? New or updated example or miniapp?
- [ ] Does it make sense to create a new section in the `CHANGELOG` to group with other related features?
- [ ] Update `INSTALL`:
- [ ] Has a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
- [ ] Have the version ranges for any required or optional libraries changed?
- [ ] Does `make` or `cmake` have a new target?
- [ ] Did the requirements or the installation process change? *(rare)*
- [ ] Had a new optional library been added? If so, what range of versions of this library are required? (*Make sure the external library is compatible with our BSD license, e.g. it is not licensed under GPL!*)
- [ ] Have the version ranges for any required or optional libraries changed?
- [ ] Does `make` or `cmake` have a new target?
- [ ] Did the requirements or the installation process change? *(rare)*
- [ ] Update continuous integration server configurations if necessary (e.g. with new version requirements for each of MFEM's dependencies)
- [ ] `.github`
- [ ] `.appveyor.yml`
- [ ] `.github`
- [ ] `.appveyor.yml`
- [ ] Update `.gitignore`:
- [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
- [ ] Add new patterns (just for the new files above) and re-run the above test.
- [ ] Check if `make distclean; git status` shows any files that were generated from the source by the project (not an IDE) but we don't want to track in the repository.
- [ ] Add new patterns (just for the new files above) and re-run the above test.
- [ ] New examples:
- [ ] All sample runs at the top of the example source file work.
- [ ] Update `examples/makefile`:
- [ ] All sample runs at the top of the example source file work.
- [ ] Update `examples/makefile`:
- [ ] Add the example code to the appropriate `SEQ_EXAMPLES` and `PAR_EXAMPLES` variables.
- [ ] Add any files generated by it to the `clean` target.
- [ ] Add the example binary and any files generated by it to the top-level `.gitignore` file.
- [ ] Update `examples/CMakeLists.txt`:
- [ ] Update `examples/CMakeLists.txt`:
- [ ] Add the example code to the `ALL_EXE_SRCS` variable.
- [ ] Make sure `THIS_TEST_OPTIONS` is set correctly for the new example.
- [ ] List the new example in `doc/CodeDocumentation.dox`.
- [ ] If new examples directory (e.g. `examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
- [ ] If new examples directory (e.g.`examples/pumi`), list it in `doc/CodeDocumentation.conf.in`
- [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
- [ ] Update or add example-specific documentation, see e.g. the `src/examples.md`.
- [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
@@ -572,13 +567,13 @@ Before a PR can be merged, it should satisfy the following:
- [ ] Add/update the `CMakeLists.txt` file in the new miniapp directory.
- [ ] Consider adding a new test for the new miniapp.
- [ ] List the new miniapp in `doc/CodeDocumentation.dox`
- [ ] If new miniapps directory (e.g. `miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
- [ ] If new miniapps directory (e.g. `miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
- [ ] If new miniapps directory (e.g.`miniapps/nurbs`), add it to `MINIAPP_SUBDIRS` in the `makefile`.
- [ ] If new miniapps directory (e.g.`miniapps/nurbs`), list it in `doc/CodeDocumentation.conf.in`
- [ ] Companion pull request for documentation in [mfem/web](https://github.com/mfem/web) repo:
- [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
- [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
- [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
- [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
- [ ] Update or add miniapp-specific documentation, see e.g. the `src/meshing.md` and `src/electromagnetics.md` files.
- [ ] Add the description, labels and screenshots in `src/examples.md` and `src/img`.
- [ ] The miniapps go at the end of the page, and are usually listed only under a specific "Application (PDE)" category.
- [ ] Add a short description of the miniapp in the "Extensive Examples" section of `features.md`.
- [ ] New capability:
- [ ] All new public, protected, and private classes, methods, data members, and functions have full Doxygen-style documentation in source comments. Documentation should include descriptions of member data, function arguments and return values, template parameters, and prerequisites for calling new functions.
- [ ] Pointer arguments and return values must specify whether ownership is being transferred or lent with the call.
@@ -680,7 +675,7 @@ MFEM uses a `master`/`next`-branch workflow as described below:
- [ ] Update URL shortlinks:
- [ ] Create a shortlink at [http://bit.ly/](http://bit.ly/) for the release tarball, e.g. https://mfem.github.io/releases/mfem-3.1.tgz.
- [ ] (LLNL only) Add and commit the new shortlink in the `links` and `links-mfem` files of the internal `mfem/downloads` repo.
- [ ] Add the new shortlinks to the MFEM package in `spack`.
- [ ] Add the new shortlinks to the MFEM packages in `spack`, `homebrew/science`, `VisIt`, etc.
- [ ] Update website in `mfem/web` repo:
- Update version and shortlinks in `src/index.md` and `src/download.md`.
- Use [cloc-1.62.pl](http://cloc.sourceforge.net/) and `ls -lh` to estimate the SLOC and the tarball size in `src/download.md`.
@@ -732,24 +727,22 @@ commit or push, see the [README](config/githooks/README.md) in the `config/githo
directory.
### GitHub Actions smoke tests
### Linux and Mac smoke tests
We use GitHub Actions to drive the default tests on the `master` and `next`
branches. See the `.github/workflows` files and the logs at
[https://github.com/mfem/mfem/actions](https://github.com/mfem/mfem/actions).
GitHub Actions testing should be kept lightweight, as there is a time
constraint on jobs. The current workflows cover Linux, macOS, and Windows
configurations.
Testing using GitHub Actions should be kept lightweight, as there is a time
constraint on jobs. Two virtual machines are configured - Mac (OS X) and Linux.
- Tests on the `master` branch are triggered whenever a PR is issued on this branch.
- Tests on the `next` branch are currently scheduled to run each night.
### Additional Windows smoke test
We also use Appveyor to test building with the MS Visual C++ compiler in a Windows
environment, as well as to test the CMake build. See the `.appveyor.yml` file
and the build logs at
### Windows smoke test
We use Appveyor to test building with the MS Visual C++ compiler in a Windows
environment, as well as to test the CMake build. See the `.appveyor` file and the
build logs at
[https://ci.appveyor.com/project/mfem/mfem](https://ci.appveyor.com/project/mfem/mfem).
CMake is used to generate the MSVC Project files and drive the build. A release
+16 -31
View File
@@ -38,13 +38,14 @@ the option MFEM_USE_METIS.
MFEM also includes support for devices such as GPUs, and programming models such
as CUDA, HIP, OCCA, OpenMP and RAJA.
- Starting with version 4.9, MFEM requires a C++17 compiler.
- Starting with version 4.0, MFEM requires a C++11 compiler. We recommend using
a newer compiler, e.g. GCC version 4.9 or higher.
- CUDA support requires an NVIDIA GPU and an installation of the CUDA Toolkit
https://developer.nvidia.com/cuda-toolkit
- HIP support requires an AMD GPU and an installation of the ROCm software stack
https://rocm.docs.amd.com
https://rocmdocs.amd.com
- OCCA support requires the OCCA library
https://libocca.org
@@ -82,9 +83,9 @@ Serial build:
Parallel build:
(download hypre and METIS 4 from above URLs)
(build METIS 4 in ../metis-4.0 relative to mfem/)
(for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
(build hypre in ../hypre relative to mfem/)
make parallel -j 4
(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
CUDA build:
make cuda -j 4
@@ -114,14 +115,14 @@ Serial build:
Parallel build:
(download hypre and METIS 4 from above URLs)
(build METIS 4 in ../metis-4.0 relative to mfem/)
(for METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
(build hypre in ../hypre relative to mfem/)
mkdir <mfem-build-dir> ; cd <mfem-build-dir>
cmake <mfem-source-dir> -DMFEM_USE_MPI=YES
make -j 4
(For METIS 5, see https://mfem.org/building/#parallel-build-using-metis-5)
Parallel build with fetching of hypre and METIS:
mkdir <mfem-build-dir> ; cd <mfem-build-dir>
mkdir <mfem-buil-dir> ; cd <mfem-build-dir>
cmake <mfem-source-dir> -DMFEM_USE_MPI=YES -DMFEM_FETCH_TPLS=YES
make -j 4
@@ -133,8 +134,7 @@ CUDA build:
HIP build:
mkdir <mfem-build-dir> ; cd <mfem-build-dir>
cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 \
-DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
cmake <mfem-source-dir> -DMFEM_USE_HIP=YES -DHIP_ARCH=gfx942 -DCMAKE_CXX_COMPILER=amdclang++ -DCMAKE_HIP_COMPILER=amdclang++
make -j 4
Example codes (serial/parallel, depending on the build):
@@ -269,7 +269,6 @@ Compilers:
CXX - C++ compiler, serial build
MPICXX - MPI C++ compiler, parallel build
CUDA_CXX - The CUDA compiler, 'nvcc' or 'clang++'
HIP_CXX - The HIP compiler, e.g. 'hipcc'
Compiler options:
OPTIM_FLAGS - Options for optimized build
@@ -396,11 +395,6 @@ MFEM_USE_STRUMPACK = YES/NO
classes. When enabled, this option uses the STRUMPACK_* library options, see
below.
MFEM_USE_CUDSS = YES/NO
Enable MFEM functionality based on the cuDSS library. When using cuDSS, CUDA
support must be also enabled in MFEM, i.e. MFEM_USE_CUDA=YES must be set.
When enabled, this option uses the CUDSS_* library options, see below.
MFEM_USE_GINKGO = YES/NO
Enable MFEM functionality based on the Ginkgo library, which provides
iterative linear solvers and preconditioners with OpenMP, CUDA backends, see
@@ -560,13 +554,13 @@ MFEM_USE_RAJA = YES/NO
MFEM_USE_OCCA = YES/NO
Enables support for the OCCA library in MFEM. OCCA is an open-source library
which aims to make it easy to program different types of devices (e.g. CPU,
GPU, FPGA) by providing a unified API for interacting with JIT-compiled
GPU, FPGA) by providing an unified API for interacting with JIT-compiled
backends. In order to use the OCCA CUDA backend, CUDA support must be enabled
in MFEM as well, i.e. MFEM_USE_CUDA=YES must be set.
MFEM_USE_GSLIB = YES/NO
Enables MFEM functionality based on the GSLIB library, and specifically its
FindPoints component, which provides robust algorithms to evaluate finite
FindPoints component, which provides a robust algorithms to evaluate finite
element functions in a collection of points in physical space. When enabled,
the user can use the GSLIB-FindPoints methods as shown in miniapps/gslib.
@@ -725,18 +719,9 @@ The specific libraries and their options are:
Options: STRUMPACK_OPT, STRUMPACK_LIB.
Versions: STRUMPACK >= 3.0.0.
- CUDSS (optional), used when MFEM_USE_CUDSS = YES. Note that CUDSS requires
CUDA 12.x toolkit and the cuDSS libraries. The supported communication backend
is OpenMPI 4.x (default), and OpenMPI 4.x or a later version must be pre-built.
The source files in the cuDSS tarball provide guidance for developing custom
MPI implementations.
URL: https://developer.nvidia.com/cudss
https://docs.nvidia.com/cuda/cudss/advanced_features.html#communication-layer-library-in-cudss
Options: CUDSS_OPT, CUDSS_LIB.
Versions: cuDSS >= 0.6.0.
- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Ginkgo may have additional
requirements and module-specific dependencies; see the webpage below.
- Ginkgo (optional), used when MFEM_USE_GINKGO = YES. Note that Ginkgo needs a
C++ compiler that supports the C++-17 standard. For additional requirements
and dependencies of specific modules, see the Ginkgo webpage below.
URL: https://ginkgo-project.github.io
Options: GINKGO_OPT, GINKGO_LIB, GINKGO_DIR, GINKGO_BUILD_TYPE (Release or
Debug).
@@ -808,7 +793,7 @@ The specific libraries and their options are:
Options: CONDUIT_OPT, CONDUIT_LIB.
Versions: Conduit >= 0.3.1.
- ADIOS2 (optional), used when MFEM_USE_ADIOS2 = YES.
- ADIOS2 (optional) used when MFEM_USE_ADIOS2 = YES.
URL: https://adios2.readthedocs.io/
Versions: ADIOS >= 2.5.0.
@@ -884,7 +869,7 @@ The specific libraries and their options are:
Options: RAJA_DIR, RAJA_OPT, RAJA_LIB.
Versions: RAJA >= 2022.10.3.
- Moonolith (optional), used when MFEM_USE_MOONOLITH = YES.
- Moonolith (optional), use when MFEM_USE_MOONOLITH = YES.
URL: https://bitbucket.org/zulianp/par_moonolith
Options: MOONOLITH_DIR
Versions: MOONOLITH >= 1.1.0.
@@ -972,7 +957,7 @@ CMAKE_BUILD_TYPE which can be set to standard values like "Debug", and "Release"
To use a specific generator use the "-G <generator>" option of cmake:
cmake <mfem-source-dir> -G "Xcode"
cmake <mfem-source-dir> -G "Visual Studio 17 2022"
cmake <mfem-source-dir> -G "Visual Studio 12 2013"
cmake <mfem-source-dir> -G "MinGW Makefiles"
With CMake it is possible to build MFEM as a shared library using the standard
@@ -1217,7 +1202,7 @@ larger problems, there are two options:
Specific options for HIP
========================
MFEM expects the `ROCM_PATH` environment variable to be set to the path of the
ROCm install, as well as having `$ROCM_PATH/bin` in `PATH`.
ROCM install, as well as having `$ROCM_PATH/bin` in `PATH`.
Specific options for RAJA+HIP+MPI
=================================
-1
View File
@@ -28,7 +28,6 @@ license files. These software products and their licenses are as follows:
* AmgXWrapper (linalg/amgxsolver.{hpp,cpp}) -- MIT license
* Catch++ (tests/unit/catch.hpp) -- Boost 1.0 license
* Gecko (general/gecko.{cpp,hpp}) -- BSD 3-clause license
* gslib (fem/gslib.{cpp,hpp}, mesh/bb_grid_map.{cpp,hpp}) -- BSD 3-clause license
* Picojson (fem/picojson.h) -- Custom 2-clause license
* TinyXML2 (general/tinyxml2.{cpp,h}) -- zlib license
* Zstr (general/zstr.hpp) -- MIT license
-9
View File
@@ -35,7 +35,6 @@ set(MFEM_USE_SUITESPARSE @MFEM_USE_SUITESPARSE@)
set(MFEM_USE_SUPERLU @MFEM_USE_SUPERLU@)
set(MFEM_USE_MUMPS @MFEM_USE_MUMPS@)
set(MFEM_USE_STRUMPACK @MFEM_USE_STRUMPACK@)
set(MFEM_USE_CUDSS @MFEM_USE_CUDSS@)
set(MFEM_USE_GINKGO @MFEM_USE_GINKGO@)
set(MFEM_USE_AMGX @MFEM_USE_AMGX@)
set(MFEM_USE_MAGMA @MFEM_USE_MAGMA@)
@@ -110,14 +109,6 @@ if (MFEM_USE_RAJA)
find_dependency(RAJA)
endif()
if (MFEM_USE_CUDSS)
find_dependency(cudss)
endif (MFEM_USE_CUDSS)
if (MFEM_USE_UMPIRE)
find_dependency(umpire)
endif()
if (NOT TARGET mfem)
include(${CMAKE_CURRENT_LIST_DIR}/MFEMTargets.cmake)
endif (NOT TARGET mfem)
-9
View File
@@ -108,15 +108,6 @@
// Enable MFEM functionality based on the STRUMPACK library.
#cmakedefine MFEM_USE_STRUMPACK
// Enable MFEM functionality based on the cuDSS library.
#cmakedefine MFEM_USE_CUDSS
// CUDSS communication layer library path
#cmakedefine MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
// CUDSS threading layer library path
#cmakedefine MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
// Enable functionality based on the Ginkgo library.
#cmakedefine MFEM_USE_GINKGO
-68
View File
@@ -1,68 +0,0 @@
if (NOT cudss_DIR AND CUDSS_DIR)
set(cudss_DIR ${CUDSS_DIR}/lib/cmake/cudss)
endif()
message(STATUS "Looking for CUDSS ...")
message(STATUS " in CUDSS_DIR = ${CUDSS_DIR}")
message(STATUS " cudss_DIR = ${cudss_DIR}")
find_package(cudss)
set(CUDSS_FOUND ${cudss_FOUND})
set(CUDSS_LIBRARIES "cudss")
if (CUDSS_FOUND)
message(STATUS
"Found CUDSS target: ${CUDSS_LIBRARIES} (version: ${cudss_VERSION})")
else()
set(msg STATUS)
if (CUDSS_FIND_REQUIRED)
set(msg FATAL_ERROR)
endif()
message(${msg}
"CUDSS not found. Please set CUDSS_DIR to the install prefix.")
endif()
if(CUDSS_FOUND AND TARGET cudss)
get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION)
if(NOT CUDSS_LIBRARY_LOCATION)
get_target_property(CUDSS_LIBRARY_LOCATION cudss IMPORTED_LOCATION_RELEASE)
endif()
if(CUDSS_LIBRARY_LOCATION)
get_filename_component(CUDSS_LIBRARY_DIR "${CUDSS_LIBRARY_LOCATION}" DIRECTORY)
else()
message(WARNING "Could not determine the location of the cuDSS library.")
endif()
else()
message(WARNING "cuDSS target not available; cannot determine library directory.")
endif()
# Set the full name of the cuDSS threading library if OpenMP is enabled.
# The threading layer library (libcudss_mtlayer_gomp.so) is located under the
# cuDSS library directory by default.
if (MFEM_USE_OPENMP)
find_file(
CUDSS_THREADING_LIB
NAMES libcudss_mtlayer_gomp.so
PATHS ${CUDSS_LIBRARY_DIR}
NO_DEFAULT_PATH
)
if (NOT DEFINED MFEM_CUDSS_THREADING_LIB AND CUDSS_THREADING_LIB)
set(MFEM_CUDSS_THREADING_LIB "${CUDSS_THREADING_LIB}")
endif()
message(STATUS "CUDSS threading layer library: ${MFEM_CUDSS_THREADING_LIB}")
endif()
# Set the full name of the cuDSS communication library if MFEM use OpenMPI.
# The communication layer library (libcudss_commlayer_mpi.so) is located under the
# cuDSS library directory by default.
# The communication layer library is used pre-built communication layers for OpenMPI
# by default.
if (MFEM_USE_MPI)
find_file(
CUDSS_COMM_LIB
NAMES libcudss_commlayer_openmpi.so
PATHS ${CUDSS_LIBRARY_DIR}
NO_DEFAULT_PATH
)
if (NOT DEFINED MFEM_CUDSS_COMM_LIB AND CUDSS_COMM_LIB)
set(MFEM_CUDSS_COMM_LIB "${CUDSS_COMM_LIB}")
endif()
message(STATUS "CUDSS communication layer library: ${MFEM_CUDSS_COMM_LIB}")
endif()
+10 -8
View File
@@ -18,17 +18,19 @@
if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
enable_language(C)
set(GSLIB_FETCH_VERSION 1.0.9)
add_library(GSLIB STATIC IMPORTED)
# set options (technically flags because GSLIB does not use cmake)
string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
set(GSLIB_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
if (BUILD_SHARED_LIBS)
set(GSLIB_FLAGS "${GSLIB_FLAGS} -fPIC")
set(GSLIB_FETCH_VERSION 1.0.9)
set(GSLIB_C_FLAGS ${CMAKE_C_FLAGS_${BUILD_TYPE}})
if (CMAKE_C_FLAGS)
set(GSLIB_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
endif()
if (BUILD_SHARED_LIBS)
set(GSLIB_C_FLAGS "${GSLIB_C_FLAGS} -fPIC")
endif()
add_library(GSLIB STATIC IMPORTED)
# define external project and create future include directory so it is present
# to pass CMake checks at end of MFEM configuration step
message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_FLAGS}")
message(STATUS "Will fetch GSLIB ${GSLIB_FETCH_VERSION} to be built with ${GSLIB_C_FLAGS}")
set(PREFIX ${CMAKE_BINARY_DIR}/fetch/gslib)
include(ExternalProject)
ExternalProject_Add(gslib
@@ -38,7 +40,7 @@ if (MFEM_FETCH_GSLIB OR MFEM_FETCH_TPLS)
UPDATE_DISCONNECTED TRUE
PREFIX ${PREFIX}
CONFIGURE_COMMAND ""
BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS=${GSLIB_FLAGS}"
BUILD_COMMAND cd ${PREFIX}/src/gslib && $(MAKE) clean && $(MAKE) DESTDIR=${PREFIX} MPI=$<BOOL:${MFEM_USE_MPI}> "CFLAGS= ${GSLIB_C_FLAGS}"
INSTALL_COMMAND "")
file(MAKE_DIRECTORY ${PREFIX}/include)
# set imported library target properties
+1 -3
View File
@@ -44,9 +44,6 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
# set options and associated dependencies
set(HYPRE_CMAKE_OPTIONS "")
list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE})
if (BUILD_SHARED_LIBS)
list(APPEND HYPRE_CMAKE_OPTIONS -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON)
endif()
# collect all HYPRE_ENABLE variables and pass them to hypre, assuming they are BOOL.
get_cmake_property(all_vars VARIABLES)
foreach(var ${all_vars})
@@ -98,6 +95,7 @@ if (MFEM_FETCH_HYPRE OR MFEM_FETCH_TPLS)
UPDATE_DISCONNECTED TRUE
SOURCE_SUBDIR src
PREFIX ${HYPRE_INSTALL}
BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j${CMAKE_BUILD_PARALLEL_LEVEL}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${HYPRE_INSTALL} -DCMAKE_INSTALL_LIBDIR:PATH=lib ${HYPRE_CMAKE_OPTIONS})
file(MAKE_DIRECTORY ${HYPRE_INSTALL}/include)
# set imported library target properties
+2 -10
View File
@@ -19,18 +19,10 @@
# - METIS_VERSION_5 (cache variable)
if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
enable_language(C)
set(METIS_FETCH_VERSION 4.0.3)
add_library(METIS STATIC IMPORTED)
# set options (technically flags because METIS does not use cmake)
set(METIS_FLAGS "-Wno-implicit-int -Wno-incompatible-pointer-types")
string(TOUPPER "${CMAKE_BUILD_TYPE}" BUILD_TYPE)
set(METIS_FLAGS "${METIS_FLAGS} ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE}}")
if (BUILD_SHARED_LIBS)
set(METIS_FLAGS "${METIS_FLAGS} -fPIC")
endif()
# define external project
message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with ${METIS_FLAGS}")
message(STATUS "Will fetch METIS ${METIS_FETCH_VERSION} to be built with default options")
set(PREFIX ${CMAKE_BINARY_DIR}/fetch/metis)
include(ExternalProject)
ExternalProject_Add(metis
@@ -40,7 +32,7 @@ if (MFEM_FETCH_METIS OR MFEM_FETCH_TPLS)
UPDATE_DISCONNECTED TRUE
PREFIX ${PREFIX}
CONFIGURE_COMMAND tar -xzf ../metis/metis-${METIS_FETCH_VERSION}-mac.tgz --strip=1
BUILD_COMMAND $(MAKE) clean && $(MAKE) "OPTFLAGS=${METIS_FLAGS}"
BUILD_COMMAND $(MAKE) COPTIONS=-Wno-incompatible-pointer-types
INSTALL_COMMAND mkdir -p ${PREFIX}/lib && cp libmetis.a ${PREFIX}/lib/)
# set imported library target properties
add_dependencies(METIS metis)
+9 -9
View File
@@ -22,15 +22,15 @@ include(MfemCmakeUtilities)
mfem_find_package(SuiteSparse SuiteSparse SuiteSparse_DIR "" "" "" ""
"Paths to headers required by SuiteSparse."
"Libraries required by SuiteSparse."
ADD_COMPONENT "UMFPACK" "include;include/suitesparse;suitesparse" umfpack.h "lib" umfpack
ADD_COMPONENT "KLU" "include;include/suitesparse;suitesparse" klu.h "lib" klu
ADD_COMPONENT "AMD" "include;include/suitesparse;suitesparse" amd.h "lib" amd
ADD_COMPONENT "BTF" "include;include/suitesparse;suitesparse" btf.h "lib" btf
ADD_COMPONENT "CHOLMOD" "include;include/suitesparse;suitesparse" cholmod.h "lib" cholmod
ADD_COMPONENT "COLAMD" "include;include/suitesparse;suitesparse" colamd.h "lib" colamd
ADD_COMPONENT "CAMD" "include;include/suitesparse;suitesparse" camd.h "lib" camd
ADD_COMPONENT "CCOLAMD" "include;include/suitesparse;suitesparse" ccolamd.h "lib" ccolamd
ADD_COMPONENT "config" "include;include/suitesparse;suitesparse" SuiteSparse_config.h "lib"
ADD_COMPONENT "UMFPACK" "include;suitesparse" umfpack.h "lib" umfpack
ADD_COMPONENT "KLU" "include;suitesparse" klu.h "lib" klu
ADD_COMPONENT "AMD" "include;suitesparse" amd.h "lib" amd
ADD_COMPONENT "BTF" "include;suitesparse" btf.h "lib" btf
ADD_COMPONENT "CHOLMOD" "include;suitesparse" cholmod.h "lib" cholmod
ADD_COMPONENT "COLAMD" "include;suitesparse" colamd.h "lib" colamd
ADD_COMPONENT "CAMD" "include;suitesparse" camd.h "lib" camd
ADD_COMPONENT "CCOLAMD" "include;suitesparse" ccolamd.h "lib" ccolamd
ADD_COMPONENT "config" "include;suitesparse" SuiteSparse_config.h "lib"
suitesparseconfig)
if (SuiteSparse_FOUND AND METIS_VERSION_5)
+3 -3
View File
@@ -14,12 +14,12 @@
# - UMPIRE_LIBRARIES
# - UMPIRE_INCLUDE_DIRS
if (NOT umpire_ROOT AND UMPIRE_DIR)
set(umpire_ROOT ${UMPIRE_DIR})
if (NOT umpire_DIR AND UMPIRE_DIR)
set(umpire_DIR ${UMPIRE_DIR}/lib/cmake/umpire)
endif()
message(STATUS "Looking for UMPIRE ...")
message(STATUS " in UMPIRE_DIR = ${UMPIRE_DIR}")
message(STATUS " umpire_ROOT = ${umpire_ROOT}")
message(STATUS " umpire_DIR = ${umpire_DIR}")
find_package(umpire CONFIG)
set(UMPIRE_FOUND ${umpire_FOUND})
set(UMPIRE_LIBRARIES "umpire")
+17 -89
View File
@@ -701,6 +701,7 @@ endfunction(mfem_find_library)
# Extract compile and link options needed by the given target.
#
function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
if (NOT TARGET ${Target})
return()
endif()
@@ -798,12 +799,7 @@ function(mfem_get_target_options Target CompileOptsVar LinkOptsVar)
# message(STATUS "Lib = ${Lib}")
# Filter-out generator expressions
if (NOT ("${Lib}" MATCHES "^\\$"))
if(NOT ("${Lib}" STREQUAL "dl"))
list(APPEND LinkOpts "${Lib}")
else()
# for some reason libdl doesn't include the "-l"
list(APPEND LinkOpts "-ldl")
endif()
list(APPEND LinkOpts "${Lib}")
endif()
else()
mfem_get_target_options(${Lib} COpts LOpts)
@@ -892,18 +888,9 @@ function(mfem_export_mk_files)
set(${var} NO)
endif()
endforeach()
if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
set(MFEM_CXX ${CMAKE_CUDA_COMPILER})
if(MFEM_CUDA_COMPILER_IS_NVCC)
set(MFEM_HOST_CXX ${CMAKE_CUDA_HOST_COMPILER})
else()
set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
endif()
else()
# mfem doesn't use enable_language(HIP)
set(MFEM_CXX ${CMAKE_CXX_COMPILER})
set(MFEM_HOST_CXX ${CMAKE_CXX_COMPILER})
endif()
# TODO: Add support for MFEM_USE_CUDA=YES
set(MFEM_CXX ${CMAKE_CXX_COMPILER})
set(MFEM_HOST_CXX ${MFEM_CXX})
set(MFEM_CPPFLAGS "")
get_target_property(cxx_std mfem CXX_STANDARD)
# For now, we ignore the setting of the CXX_EXTENSIONS property. If this
@@ -913,50 +900,6 @@ function(mfem_export_mk_files)
string(STRIP
"${cxx_std_flag} ${CMAKE_CXX_FLAGS_${BUILD_TYPE}} ${CMAKE_CXX_FLAGS}"
MFEM_CXXFLAGS)
if(MFEM_EXPORT_GPU_CONFIG)
if (MFEM_USE_CUDA)
set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} ${CMAKE_CUDA_FLAGS}")
if (MFEM_CUDA_COMPILER_IS_NVCC)
set(MFEM_CXXFLAGS "-x=cu ${MFEM_CXXFLAGS} -ccbin ${CMAKE_CXX_COMPILER} --forward-unknown-to-host-compiler")
# The following intentionally hides CUDA deprecation warnings
foreach(ENTRY IN LISTS CUDAToolkit_INCLUDE_DIRS)
set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -isystem ${ENTRY}")
endforeach()
if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
# architecture flags not part of CMAKE_CUDA_FLAGS
if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -arch=${CMAKE_CUDA_ARCHITECTURES}")
else()
foreach (ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
set(MFEM_CXXFLAGS
"${MFEM_CXXFLAGS} -gencode arch=compute_${ENTRY},code=sm_${ENTRY}")
endforeach()
endif()
endif()
else()
set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xcuda --cuda-path=${CUDAToolkit_LIBRARY_ROOT}")
if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18.0)
# architecture flags not part of CMAKE_CUDA_FLAGS
if ("all" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
OR "native" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}"
OR "all-major" STREQUAL "${CMAKE_CUDA_ARCHITECTURES}")
# TODO: not supported
else()
foreach(ENTRY IN LISTS CMAKE_CUDA_ARCHITECTURES)
set(MFEM_CXXFLAGS "-cuda-gpu-arch=sm_${ENTRY} ${MFEM_CXXFLAGS}")
endforeach()
endif()
endif()
endif()
elseif (MFEM_USE_HIP)
set(MFEM_CXXFLAGS "${MFEM_CXXFLAGS} -xhip")
foreach(ENTRY IN LISTS CMAKE_HIP_ARCHITECTURES)
set(MFEM_CXXFLAGS "--offload-arch=${ENTRY} ${MFEM_CXXFLAGS}")
endforeach()
endif()
endif()
set(MFEM_TPLFLAGS "")
foreach(dir ${TPL_INCLUDE_DIRS})
set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} -I${dir}")
@@ -987,9 +930,6 @@ function(mfem_export_mk_files)
set(MFEM_SHARED NO)
set(MFEM_STATIC YES)
endif()
if (MFEM_USE_CUDA)
set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} -lcudart")
endif()
set(MFEM_BUILD_TAG "${CMAKE_SYSTEM}")
set(MFEM_PREFIX "${CMAKE_INSTALL_PREFIX}")
# For the next 4 variables, these are the values for the build-tree version of
@@ -998,15 +938,8 @@ function(mfem_export_mk_files)
set(MFEM_LIB_DIR "${PROJECT_BINARY_DIR}")
set(MFEM_TEST_MK "${PROJECT_SOURCE_DIR}/config/test.mk")
set(MFEM_CONFIG_EXTRA "MFEM_BUILD_DIR ?= ${PROJECT_BINARY_DIR}")
if (MFEM_USE_CUDA AND MFEM_EXPORT_GPU_CONFIG)
if (MFEM_CUDA_COMPILER_IS_NVCC)
set(MFEM_XLINKER "-Xlinker=")
else()
set(MFEM_XLINKER "${CMAKE_CUDA_LINKER_WRAPPER_FLAG}")
endif()
else()
set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
endif()
# TODO: CUDA/HIP support:
set(MFEM_XLINKER "${CMAKE_CXX_LINKER_WRAPPER_FLAG}")
set(MFEM_MPIEXEC ${MPIEXEC})
if (NOT MFEM_MPIEXEC)
set(MFEM_MPIEXEC "mpirun")
@@ -1054,21 +987,16 @@ function(mfem_export_mk_files)
# handle interfaces (e.g., SCOREC::apf)
if ("${lib}" MATCHES "SCOREC::.*" OR "${lib}" MATCHES "Ginkgo::.*" OR "${lib}" MATCHES "ParMoonolith::.*")
elseif (TARGET "${lib}")
mfem_get_target_options(${lib} CompileOpts2 LinkOpts2)
# remove generator expressions
string(GENEX_STRIP "${CompileOpts2}" CompileOpts)
string(GENEX_STRIP "${LinkOpts2}" LinkOpts)
mfem_get_target_options(${lib} CompileOpts LinkOpts)
# Removing duplicates may lead to issues:
# list(REMOVE_DUPLICATES CompileOpts)
# list(REMOVE_DUPLICATES LinkOpts)
# message(WARNING "${lib}[LinkOpts]: ${LinkOpts}")
# message(WARNING "${lib}[CompileOpts]: ${CompileOpts}")
foreach(LOpt IN LISTS LinkOpts)
set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpt}")
endforeach()
foreach(COpt IN LISTS CompileOpts)
set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpt}")
endforeach()
string(REPLACE ";" " " COpts "${CompileOpts}")
string(REPLACE ";" " " LOpts "${LinkOpts}")
# message(STATUS "${lib}[COpts]: '${COpts}'")
# message(STATUS "${lib}[LOpts]: '${LOpts}'")
set(MFEM_TPLFLAGS "${MFEM_TPLFLAGS} ${COpts}")
set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${LOpts}")
# message(FATAL_ERROR "***** interface lib found ... exiting *****")
# handle static and shared libs
elseif ("${suffix}" STREQUAL "${CMAKE_SHARED_LIBRARY_SUFFIX}")
@@ -1076,7 +1004,7 @@ function(mfem_export_mk_files)
get_filename_component(fullLibName ${lib} NAME_WE)
string(REGEX REPLACE "^lib" "" libname ${fullLibName})
set(MFEM_EXT_LIBS
"${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
"${MFEM_EXT_LIBS} ${shared_link_flag}${dir} -L${dir} -l${libname}")
else()
set(MFEM_EXT_LIBS "${MFEM_EXT_LIBS} ${lib}")
endif()
@@ -1085,7 +1013,7 @@ function(mfem_export_mk_files)
# Create the build-tree version of 'config.mk'
configure_file(
"${PROJECT_SOURCE_DIR}/config/config.mk.in"
"${PROJECT_BINARY_DIR}/config/config.mk" @ONLY)
"${PROJECT_BINARY_DIR}/config/config.mk")
# Copy 'test.mk' from the source-tree to the build-tree
configure_file(
"${PROJECT_SOURCE_DIR}/config/test.mk"
@@ -1103,7 +1031,7 @@ function(mfem_export_mk_files)
# Create the install-tree version of 'config.mk'
configure_file(
"${PROJECT_SOURCE_DIR}/config/config.mk.in"
"${PROJECT_BINARY_DIR}/config/config-install.mk" @ONLY)
"${PROJECT_BINARY_DIR}/config/config-install.mk")
# Install rules for 'config.mk' and 'test.mk'
install(FILES ${PROJECT_SOURCE_DIR}/config/test.mk
+2 -8
View File
@@ -18,9 +18,9 @@
#define MFEM_CONFIG_HPP
#ifdef MFEM_CONFIG_FILE
#include MFEM_CONFIG_FILE // IWYU pragma: export
#include MFEM_CONFIG_FILE
#else
#include "_config.hpp" // IWYU pragma: export
#include "_config.hpp"
#endif
#include <cstdint>
@@ -157,10 +157,4 @@ constexpr real_t operator""_r(unsigned long long v)
#endif
#endif // MFEM_USE_MPI not defined
#ifndef MFEM_USE_CUDA
#ifdef MFEM_USE_CUDSS
#error Building with cuDSS (MFEM_USE_CUDSS=YES) requires CUDA (MFEM_USE_CUDA=YES)
#endif
#endif // MFEM_USE_CUDSS not defined
#endif // MFEM_CONFIG_HPP
-9
View File
@@ -108,15 +108,6 @@
// Enable MFEM functionality based on the STRUMPACK library.
// #define MFEM_USE_STRUMPACK
// Enable MFEM functionality based on the cuDSS library.
// #define MFEM_USE_CUDSS
// CUDSS communication layer library path
// #define MFEM_CUDSS_COMM_LIB "@MFEM_CUDSS_COMM_LIB@"
// CUDSS threading layer library path
// #define MFEM_CUDSS_THREADING_LIB "@MFEM_CUDSS_THREADING_LIB@"
// Enable MFEM features based on the Ginkgo library.
// #define MFEM_USE_GINKGO
-3
View File
@@ -36,9 +36,6 @@ MFEM_USE_SUPERLU = @MFEM_USE_SUPERLU@
MFEM_USE_SUPERLU5 = @MFEM_USE_SUPERLU5@
MFEM_USE_MUMPS = @MFEM_USE_MUMPS@
MFEM_USE_STRUMPACK = @MFEM_USE_STRUMPACK@
MFEM_USE_CUDSS = @MFEM_USE_CUDSS@
MFEM_CUDSS_COMM_LIB = @MFEM_CUDSS_COMM_LIB@
MFEM_CUDSS_THREADING_LIB = @MFEM_CUDSS_THREADING_LIB@
MFEM_USE_GINKGO = @MFEM_USE_GINKGO@
MFEM_USE_AMGX = @MFEM_USE_AMGX@
MFEM_USE_MAGMA = @MFEM_USE_MAGMA@
-1
View File
@@ -38,7 +38,6 @@ option(MFEM_USE_SUPERLU "Enable SuperLU_DIST usage" OFF)
option(MFEM_USE_SUPERLU5 "Use the old SuperLU_DIST 5.1 version" OFF)
option(MFEM_USE_MUMPS "Enable MUMPS usage" OFF)
option(MFEM_USE_STRUMPACK "Enable STRUMPACK usage" OFF)
option(MFEM_USE_CUDSS "Enable cuDSS usage" OFF)
option(MFEM_USE_GINKGO "Enable Ginkgo usage" OFF)
option(MFEM_USE_AMGX "Enable AmgX usage" OFF)
option(MFEM_USE_MAGMA "Enable MAGMA usage" OFF)
+1 -15
View File
@@ -153,7 +153,6 @@ MFEM_USE_SUPERLU = NO
MFEM_USE_SUPERLU5 = NO
MFEM_USE_MUMPS = NO
MFEM_USE_STRUMPACK = NO
MFEM_USE_CUDSS = NO
MFEM_USE_GINKGO = NO
MFEM_USE_AMGX = NO
MFEM_USE_MAGMA = NO
@@ -369,19 +368,6 @@ STRUMPACK_OPT = -I$(STRUMPACK_DIR)/include $(SCOTCH_OPT)
STRUMPACK_LIB = -L$(STRUMPACK_DIR)/lib -lstrumpack $(MPI_FORTRAN_LIB)\
$(SCOTCH_LIB) $(SCALAPACK_LIB)
# CUDSS library configuration
CUDSS_DIR = @MFEM_DIR@/../cudss
CUDSS_INCLUDE_DIR = $(CUDSS_DIR)/include
CUDSS_LIBRARY_DIR = $(CUDSS_DIR)/lib
CUDSS_OPT = -I$(CUDSS_INCLUDE_DIR)
CUDSS_LIB = \
$(XLINKER)-rpath,$(CUDSS_LIBRARY_DIR) -L$(CUDSS_LIBRARY_DIR) -lcudss
# The cuDSS communication and threading libraries.
MFEM_CUDSS_COMM_LIB = $(abspath $(wildcard $(or $(CUDSS_COMM_LIB),\
$(subst @MFEM_DIR@,$(MFEM_DIR), $(CUDSS_LIBRARY_DIR)/libcudss_commlayer_openmpi.so))))
MFEM_CUDSS_THREADING_LIB = $(abspath $(wildcard $(or $(CUDSS_THREADING_LIB),\
$(subst @MFEM_DIR@,$(MFEM_DIR),$(CUDSS_LIBRARY_DIR)/libcudss_mtlayer_gomp.so))))
# Ginkgo library configuration
GINKGO_DIR = @MFEM_DIR@/../ginkgo/install
GINKGO_SEARCH_DIR = $(subst @MFEM_DIR@,$(MFEM_DIR),$(GINKGO_DIR))
@@ -635,7 +621,7 @@ PARELAG_LIB = -L$(PARELAG_DIR)/build/src -lParELAG
AXOM_DIR = @MFEM_DIR@/../axom
TRIBOL_DIR = @MFEM_DIR@/../tribol
TRIBOL_OPT = -I$(TRIBOL_DIR)/include -I$(AXOM_DIR)/include
TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -ltribol_shared -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
TRIBOL_LIB = -L$(TRIBOL_DIR)/lib -ltribol -lredecomp -L$(AXOM_DIR)/lib -laxom_mint\
-laxom_slam -laxom_slic -laxom_core
# Enzyme configuration
+1 -2
View File
@@ -47,7 +47,6 @@ list(APPEND ALL_EXE_SRCS
ex39.cpp
ex40.cpp
ex41.cpp
# jitplayground.cpp
)
if (MFEM_USE_MPI)
@@ -216,7 +215,7 @@ if (MFEM_ENABLE_TESTING)
add_test(NAME ex1p_ceed_np=${MFEM_MPI_NP}
COMMAND ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MFEM_MPI_NP}
${MPIEXEC_PREFLAGS}
$<TARGET_FILE:ex1p> "-no-vis" "-d" "ceed-cpu" "-pa" "-a"
$<TARGET_FILE:ex1p> "-no-vis" "-d ceed-cpu" "-pa" "-a"
${MPIEXEC_POSTFLAGS})
endif()
endif()
+1 -1
View File
@@ -64,7 +64,7 @@ PARALLEL_NAME := Parallel AMGX example
$(MFEM_LIB_FILE):
$(error The MFEM library is not build)
clean: clean-build clean-exec
clean: clean-build
clean-build:
rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
+3 -3
View File
@@ -64,12 +64,12 @@ ex1p-test-par: ex1p
$(MFEM_LIB_FILE):
$(error The MFEM library is not built)
clean: clean-build clean-exec
clean: clean-build clean-exec $(SUBDIRS_CLEAN)
clean-build:
rm -f *.o *~ $(SEQ_EXAMPLES) $(PAR_EXAMPLES)
rm -rf *.dSYM *.TVD.*breakpoints
clean-exec:
@rm -f refined.mesh mesh.*
@rm -f sol.*
@rm -f refined.mesh displaced.mesh mesh.* ex5.mesh
@rm -f sphere_refined.* sol.* sol_u.* sol_p.* sol_r.* sol_i.*
+21 -34
View File
@@ -50,10 +50,6 @@
// ex1 -m ../data/beam-tet.mesh -pa -d ceed-cpu
// ex1 -m ../data/beam-tet.mesh -pa -d ceed-cuda:/gpu/cuda/ref
//
// Device simplices sample runs:
// ex1 -pa -d gpu -m ../data/inline-tet.mesh
// ex1 -pa -d gpu -m ../data/inline-tri.mesh
//
// Description: This example code demonstrates the use of MFEM to define a
// simple finite element discretization of the Poisson problem
// -Delta u = 1 with homogeneous Dirichlet boundary conditions.
@@ -142,25 +138,25 @@ int main(int argc, char *argv[])
}
// 5. Define a finite element space on the mesh. Here we use continuous
// Lagrange finite elements of the specified order.
// - If order < 1, we instead use an isoparametric/isogeometric space.
// - If the mesh is simplicial and partial assembly is requested,
// we use the positive basis, which supports device execution.
// Lagrange finite elements of the specified order. If order < 1, we
// instead use an isoparametric/isogeometric space.
FiniteElementCollection *fec;
auto basis_type = (pa && mesh.IsSimplexMesh()) ?
BasisType::Positive : BasisType::GaussLobatto;
bool delete_fec;
if (order > 0)
{
fec = new H1_FECollection(order, dim, basis_type);
fec = new H1_FECollection(order, dim);
delete_fec = true;
}
else if (mesh.GetNodes())
{
fec = mesh.GetNodes()->OwnFEC();
delete_fec = false;
cout << "Using isoparametric FEs: " << fec->Name() << endl;
}
else
{
fec = new H1_FECollection(order = 1, dim, basis_type);
fec = new H1_FECollection(order = 1, dim);
delete_fec = true;
}
FiniteElementSpace fespace(&mesh, fec);
cout << "Number of finite element unknowns: "
@@ -228,29 +224,17 @@ int main(int argc, char *argv[])
// 11. Solve the linear system A X = B.
if (!pa)
{
#ifdef MFEM_USE_CUDSS
if (Device::Allows(Backend::CUDA_MASK))
{
// Use cuDSS to solve the system.
CuDSSSolver cudss_solver;
cudss_solver.SetOperator(*A);
cudss_solver.Mult(B, X);
}
else
#endif
{
#ifndef MFEM_USE_SUITESPARSE
// Use a simple symmetric Gauss-Seidel preconditioner with PCG.
GSSmoother M((SparseMatrix&)(*A));
PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
// Use a simple symmetric Gauss-Seidel preconditioner with PCG.
GSSmoother M((SparseMatrix&)(*A));
PCG(*A, M, B, X, 1, 200, 1e-12, 0.0);
#else
// If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
UMFPackSolver umf_solver;
umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
umf_solver.SetOperator(*A);
umf_solver.Mult(B, X);
// If MFEM was compiled with SuiteSparse, use UMFPACK to solve the system.
UMFPackSolver umf_solver;
umf_solver.Control[UMFPACK_ORDERING] = UMFPACK_ORDERING_METIS;
umf_solver.SetOperator(*A);
umf_solver.Mult(B, X);
#endif
}
}
else
{
@@ -289,14 +273,17 @@ int main(int argc, char *argv[])
if (visualization)
{
char vishost[] = "localhost";
int visport = 19916;
int visport = 19916;
socketstream sol_sock(vishost, visport);
sol_sock.precision(8);
sol_sock << "solution\n" << mesh << x << flush;
}
// 15. Free the used memory.
if (order > 0) { delete fec; }
if (delete_fec)
{
delete fec;
}
return 0;
}
+2 -2
View File
@@ -5,9 +5,9 @@
// Sample runs:
// mpirun -np 4 ex12p -m ../data/beam-tri.mesh
// mpirun -np 4 ex12p -m ../data/beam-quad.mesh
// mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 464 -n 10 -o 2 -elast
// mpirun -np 4 ex12p -m ../data/beam-tet.mesh -s 462 -n 10 -o 2 -elast
// mpirun -np 4 ex12p -m ../data/beam-hex.mesh -s 3878
// mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 82
// mpirun -np 4 ex12p -m ../data/beam-wedge.mesh -s 81
// mpirun -np 4 ex12p -m ../data/beam-tri.mesh -s 3877 -o 2 -sys
// mpirun -np 4 ex12p -m ../data/beam-quad.mesh -s 4544 -n 6 -o 3 -elast
// mpirun -np 4 ex12p -m ../data/beam-quad-nurbs.mesh
+34 -60
View File
@@ -42,11 +42,7 @@
// mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared
// mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/square-mixed.mesh
// mpirun -np 4 ex1p -pa -d ceed-cuda:/gpu/cuda/shared -m ../data/fichera-mixed.mesh
// mpirun -np 4 ex1p -pa -d ceed-cpu -m ../data/beam-tet.mesh
//
// Device simplices sample runs:
// mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tet.mesh
// mpirun -np 4 ex1p -pa -d gpu -m ../data/inline-tri.mesh
// mpirun -np 4 ex1p -m ../data/beam-tet.mesh -pa -d ceed-cpu
//
// Description: This example code demonstrates the use of MFEM to define a
// simple finite element discretization of the Poisson problem
@@ -87,9 +83,6 @@ int main(int argc, char *argv[])
const char *device_config = "cpu";
bool visualization = true;
bool algebraic_ceed = false;
#ifdef MFEM_USE_CUDSS
bool cudss_solver = false;
#endif
OptionsParser args(argc, argv);
args.AddOption(&mesh_file, "-m", "--mesh",
@@ -109,10 +102,6 @@ int main(int argc, char *argv[])
args.AddOption(&algebraic_ceed, "-a", "--algebraic",
"-no-a", "--no-algebraic",
"Use algebraic Ceed solver");
#endif
#ifdef MFEM_USE_CUDSS
args.AddOption(&cudss_solver, "-cudss", "--cudss-solver", "-no-cudss",
"--no-cudss-solver", "Use the cuDSS Solver.");
#endif
args.AddOption(&visualization, "-vis", "--visualization", "-no-vis",
"--no-visualization",
@@ -169,20 +158,19 @@ int main(int argc, char *argv[])
}
// 7. Define a parallel finite element space on the parallel mesh. Here we
// use continuous Lagrange finite elements of the specified order.
// - If order < 1, we instead use an isoparametric/isogeometric space.
// - If the mesh is simplicial and partial assembly is requested,
// we use the positive basis, which supports device execution.
// use continuous Lagrange finite elements of the specified order. If
// order < 1, we instead use an isoparametric/isogeometric space.
FiniteElementCollection *fec;
auto basis_type = (pa && pmesh.IsSimplexMesh()) ?
BasisType::Positive : BasisType::GaussLobatto;
bool delete_fec;
if (order > 0)
{
fec = new H1_FECollection(order, dim, basis_type);
fec = new H1_FECollection(order, dim);
delete_fec = true;
}
else if (pmesh.GetNodes())
{
fec = pmesh.GetNodes()->OwnFEC();
delete_fec = false;
if (myid == 0)
{
cout << "Using isoparametric FEs: " << fec->Name() << endl;
@@ -190,7 +178,8 @@ int main(int argc, char *argv[])
}
else
{
fec = new H1_FECollection(order = 1, dim, basis_type);
fec = new H1_FECollection(order = 1, dim);
delete_fec = true;
}
ParFiniteElementSpace fespace(&pmesh, fec);
HYPRE_BigInt size = fespace.GlobalTrueVSize();
@@ -259,51 +248,33 @@ int main(int argc, char *argv[])
// 13. Solve the linear system A X = B.
// * With full assembly, use the BoomerAMG preconditioner from hypre.
// * With partial assembly, use Jacobi smoothing, for now.
#ifdef MFEM_USE_CUDSS
if (!pa && (Device::Allows(Backend::CUDA_MASK) && cudss_solver))
Solver *prec = NULL;
if (pa)
{
// Solve using a direct solver with cuDSS
CuDSSSolver cudss_solver(MPI_COMM_WORLD);
cudss_solver.SetMatrixSymType(
CuDSSSolver::SYMMETRIC_POSITIVE_DEFINITE);
cudss_solver.SetMatrixViewType(CuDSSSolver::UPPER);
cudss_solver.SetOperator(*A);
cudss_solver.Mult(B, X);
}
else
#endif
{
Solver *prec = NULL;
if (pa)
if (UsesTensorBasis(fespace))
{
if (UsesTensorBasis(fespace))
if (algebraic_ceed)
{
if (algebraic_ceed)
{
prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
}
else
{
prec = new OperatorJacobiSmoother(a, ess_tdof_list);
}
prec = new ceed::AlgebraicSolver(a, ess_tdof_list);
}
else
{
prec = new OperatorJacobiSmoother(a, ess_tdof_list);
}
}
else
{
prec = new HypreBoomerAMG;
}
CGSolver cg(MPI_COMM_WORLD);
cg.SetRelTol(1e-12);
cg.SetMaxIter(2000);
cg.SetPrintLevel(1);
if (prec)
{
cg.SetPreconditioner(*prec);
}
cg.SetOperator(*A);
cg.Mult(B, X);
delete prec;
}
else
{
prec = new HypreBoomerAMG;
}
CGSolver cg(MPI_COMM_WORLD);
cg.SetRelTol(1e-12);
cg.SetMaxIter(2000);
cg.SetPrintLevel(1);
if (prec) { cg.SetPreconditioner(*prec); }
cg.SetOperator(*A);
cg.Mult(B, X);
delete prec;
// 14. Recover the parallel grid function corresponding to X. This is the
// local finite element solution on each processor.
@@ -337,7 +308,10 @@ int main(int argc, char *argv[])
}
// 17. Free the used memory.
if (order > 0) { delete fec; }
if (delete_fec)
{
delete fec;
}
return 0;
}
+9 -27
View File
@@ -302,21 +302,15 @@ int main(int argc, char *argv[])
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock_r(vishost, visport);
socketstream sol_sock_i(vishost, visport);
sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
sol_sock_r.precision(8);
sol_sock_i.precision(8);
sol_sock_r << "solution\n" << *pmesh << u_exact->real()
<< "window_title 'Exact: Real Part'" << flush;
// Make sure all ranks have sent their real solution before initiating
// another set of GLVis connections (one from each rank):
MPI_Barrier(pmesh->GetComm());
socketstream sol_sock_i(vishost, visport);
sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
sol_sock_i.precision(8);
sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
<< "window_title 'Exact: Imaginary Part'" << flush;
// Make sure all ranks have sent their imaginary solution before initiating
// another set of GLVis connections (one from each rank):
MPI_Barrier(pmesh->GetComm());
}
// 11. Set up the parallel sesquilinear form a(.,.) on the finite element
@@ -540,21 +534,15 @@ int main(int argc, char *argv[])
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock_r(vishost, visport);
socketstream sol_sock_i(vishost, visport);
sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
sol_sock_r.precision(8);
sol_sock_i.precision(8);
sol_sock_r << "solution\n" << *pmesh << u.real()
<< "window_title 'Solution: Real Part'" << flush;
// Make sure all ranks have sent their real solution before initiating
// another set of GLVis connections (one from each rank):
MPI_Barrier(pmesh->GetComm());
socketstream sol_sock_i(vishost, visport);
sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
sol_sock_i.precision(8);
sol_sock_i << "solution\n" << *pmesh << u.imag()
<< "window_title 'Solution: Imaginary Part'" << flush;
// Make sure all ranks have sent their imaginary solution before initiating
// another set of GLVis connections (one from each rank):
MPI_Barrier(pmesh->GetComm());
}
if (visualization && exact_sol)
{
@@ -563,21 +551,15 @@ int main(int argc, char *argv[])
char vishost[] = "localhost";
int visport = 19916;
socketstream sol_sock_r(vishost, visport);
socketstream sol_sock_i(vishost, visport);
sol_sock_r << "parallel " << num_procs << " " << myid << "\n";
sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
sol_sock_r.precision(8);
sol_sock_i.precision(8);
sol_sock_r << "solution\n" << *pmesh << u_exact->real()
<< "window_title 'Error: Real Part'" << flush;
// Make sure all ranks have sent their real solution before initiating
// another set of GLVis connections (one from each rank):
MPI_Barrier(pmesh->GetComm());
socketstream sol_sock_i(vishost, visport);
sol_sock_i << "parallel " << num_procs << " " << myid << "\n";
sol_sock_i.precision(8);
sol_sock_i << "solution\n" << *pmesh << u_exact->imag()
<< "window_title 'Error: Imaginary Part'" << flush;
// Make sure all ranks have sent their imaginary solution before initiating
// another set of GLVis connections (one from each rank):
MPI_Barrier(pmesh->GetComm());
}
if (visualization)
{
-9
View File
@@ -95,15 +95,6 @@ int main(int argc, char *argv[])
args.PrintOptions(cout);
}
if (amg_elast && !static_cond && reorder_space)
{
if (myid == 0)
cerr << "\nThe AMG elasticity solver requires ordering byVDIM! "
<< "Ignoring the specified option -nodes/--by-nodes.\n"
<< endl;
reorder_space = false;
}
// 3. Enable hardware devices such as GPUs, and programming models such as
// CUDA, OCCA, RAJA and OpenMP based on command line options.
Device device(device_config);
+52 -11
View File
@@ -5,8 +5,8 @@
// Sample runs:
// ex37 -alpha 10
// ex37 -alpha 10 -pv
// ex37 -lambda 0.1 -mu 0.1 -growth 1
// ex37 -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
// ex37 -lambda 0.1 -mu 0.1
// ex37 -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
// ex37 -r 6 -o 1 -alpha 25.0 -epsilon 0.02 -mi 50 -ntol 1e-5
//
// Description: This example code demonstrates the use of MFEM to solve a
@@ -55,6 +55,53 @@
using namespace std;
using namespace mfem;
/**
* @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
* ∫_Ω ρ dx = θ vol(Ω) as follows:
*
* 1. Compute the root of the R → R function
* f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
* 2. Set ψ ← ψ + c.
*
* @param psi a GridFunction to be updated
* @param target_volume θ vol(Ω)
* @param tol Newton iteration tolerance
* @param max_its Newton maximum iteration number
* @return real_t Final volume, ∫_Ω sigmoid(ψ)
*/
real_t proj(GridFunction &psi, real_t target_volume, real_t tol=1e-12,
int max_its=10)
{
MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
LinearForm int_sigmoid_psi(psi.FESpace());
int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
LinearForm int_der_sigmoid_psi(psi.FESpace());
int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
der_sigmoid_psi));
bool done = false;
for (int k=0; k<max_its; k++) // Newton iteration
{
int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
const real_t f = int_sigmoid_psi.Sum() - target_volume;
int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
const real_t df = int_der_sigmoid_psi.Sum();
const real_t dc = -f/df;
psi += dc;
if (abs(dc) < tol) { done = true; break; }
}
if (!done)
{
mfem_warning("Projection reached maximum iteration without converging. "
"Result may not be accurate.");
}
int_sigmoid_psi.Assemble();
return int_sigmoid_psi.Sum();
}
/*
* ---------------------------------------------------------------
* ALGORITHM PREAMBLE
@@ -133,11 +180,10 @@ int main(int argc, char *argv[])
int ref_levels = 5;
int order = 2;
real_t alpha = 1.0;
real_t growth = 2;
real_t epsilon = 0.01;
real_t vol_fraction = 0.5;
int max_it = 1e3;
real_t itol = 1e-2;
real_t itol = 1e-1;
real_t ntol = 1e-4;
real_t rho_min = 1e-6;
real_t lambda = 1.0;
@@ -152,8 +198,6 @@ int main(int argc, char *argv[])
"Order (degree) of the finite elements.");
args.AddOption(&alpha, "-alpha", "--alpha-step-length",
"Step length for gradient descent.");
args.AddOption(&growth, "-growth", "--alpha-growth-rate",
"Growth rate of step length for gradient descent.");
args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
"Length scale for ρ.");
args.AddOption(&max_it, "-mi", "--max-it",
@@ -288,7 +332,6 @@ int main(int argc, char *argv[])
}
FilterSolver->SetEssentialBoundary(ess_bdr_filter);
FilterSolver->SetupFEM();
FilterSolver->AssembleDiffusionBilinear();
BilinearForm mass(&control_fes);
mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -342,7 +385,7 @@ int main(int argc, char *argv[])
// 11. Iterate:
for (int k = 1; k <= max_it; k++)
{
if (k > 1) { alpha = std::pow((real_t) k,growth); }
if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }
mfem::out << "\nStep = " << k << std::endl;
@@ -379,9 +422,7 @@ int main(int argc, char *argv[])
// Step 5 - Update design variable ψ ← proj(ψ - αG)
psi.Add(-alpha, grad);
GridFunction alpha_grad(grad);
alpha_grad *= alpha;
const real_t material_volume = proj(psi, alpha_grad, target_volume);
const real_t material_volume = proj(psi, target_volume);
// Compute ||ρ - ρ_old|| in control fes.
real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
+23 -183
View File
@@ -137,7 +137,7 @@ public:
exponent(exponent_), rho_min(rho_min_)
{
MFEM_ASSERT(rho_min_ >= 0.0, "rho_min must be >= 0");
MFEM_ASSERT(rho_min_ < 1.0, "rho_min must be < 1");
MFEM_ASSERT(rho_min_ < 1.0, "rho_min must be > 1");
MFEM_ASSERT(u, "displacement field is not set");
MFEM_ASSERT(rho_filter, "density field is not set");
}
@@ -231,12 +231,9 @@ private:
FiniteElementCollection * fec = nullptr;
FiniteElementSpace * fes = nullptr;
Array<int> ess_bdr;
Array<int> ess_tdof_list;
Array<int> neumann_bdr;
GridFunction * u = nullptr;
LinearForm * b = nullptr;
BilinearForm * a = nullptr;
OperatorPtr A;
bool parallel;
#ifdef MFEM_USE_MPI
ParMesh * pmesh = nullptr;
@@ -270,8 +267,6 @@ public:
void ResetFEM();
void SetupFEM();
void UpdateEssentialTDofs();
void AssembleDiffusionBilinear(bool update_ess_tdofs=true);
void Solve();
GridFunction * GetFEMSolution();
LinearForm * GetLinearForm() {return b;}
@@ -376,130 +371,6 @@ public:
};
/**
* @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
* ∫_Ω ρ dx = θ vol(Ω) as follows:
*
* 1. Compute the root of the R → R function
* f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
* using the Illinois method
* 2. Set ψ ← ψ + c.
*
* @param psi a GridFunction to be updated
* @param alpha_grad alpha multiplied by gradient
* @param target_volume θ vol(Ω)
* @param tol Illinois iteration tolerance
* @param max_its Illinois maximum iteration number
* @return real_t Final volume (∫_Ω sigmoid(ψ) dx)
*/
real_t proj(GridFunction &psi, GridFunction &alpha_grad, real_t target_volume,
real_t tol = 1e-12, int max_its = 100)
{
#ifdef MFEM_USE_MPI
FiniteElementSpace *fes = psi.FESpace();
ParFiniteElementSpace *pfes = dynamic_cast<ParFiniteElementSpace*>(fes);
#endif
ConstantCoefficient zero_cf(0.0);
real_t a = -alpha_grad.ComputeMaxError(zero_cf);
real_t b = -a;
real_t y = 0.0;
MappedGridFunctionCoefficient sigmoid_psi(
&psi, [&y](const real_t x) { return sigmoid(x + y); });
std::unique_ptr<LinearForm> int_sigmoid_psi;
#ifdef MFEM_USE_MPI
ParGridFunction *par_psi = dynamic_cast<ParGridFunction *>(&psi);
if (par_psi)
{
int_sigmoid_psi.reset(new ParLinearForm(par_psi->ParFESpace()));
}
else
{
int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
}
#else
int_sigmoid_psi.reset(new LinearForm(psi.FESpace()));
#endif
int_sigmoid_psi->AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
y = a;
int_sigmoid_psi->Assemble();
real_t f_a = int_sigmoid_psi->Sum(); // f_a := f(a) + θ vol(Ω)
y = b;
int_sigmoid_psi->Assemble();
real_t f_b = int_sigmoid_psi->Sum(); // f_b := f(b) + θ vol(Ω)
#ifdef MFEM_USE_MPI
if (pfes)
{
MPI_Allreduce(MPI_IN_PLACE, &f_a, 1, MPITypeMap<real_t>::mpi_type,
MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, &f_b, 1, MPITypeMap<real_t>::mpi_type,
MPI_SUM, MPI_COMM_WORLD);
}
#endif
f_a -= target_volume; // f_a := f(a)
f_b -= target_volume; // f_b := f(b)
real_t c = 0.0;
real_t f_c = 0.0;
int side = 0;
bool done = false;
for (int k=0; k < max_its; k++)
{
c = (f_a * b - f_b * a) / (f_a - f_b);
if (abs(b - a) < tol * abs(b + a)) { done = true; break; }
y = c;
int_sigmoid_psi->Assemble();
f_c = int_sigmoid_psi->Sum(); // f_c := f(c) + θ vol(Ω)
#ifdef MFEM_USE_MPI
if (pfes)
{
MPI_Allreduce(MPI_IN_PLACE, &f_c, 1, MPITypeMap<real_t>::mpi_type,
MPI_SUM, MPI_COMM_WORLD);
}
#endif
f_c -= target_volume; // f_c := f(c)
if (f_c * f_b > 0)
{
b = c;
f_b = f_c;
if (side == -1) { f_a /= 2.0; }
side = -1;
}
else if (f_c * f_a > 0)
{
a = c;
f_a = f_c;
if (side == 1) { f_b /= 2.0; }
side = 1;
}
else
{
done = true; break;
}
}
if (!done)
{
mfem_warning("Projection reached maximum iteration without converging. "
"Result may not be accurate.");
}
y = 0.0;
psi += c;
int_sigmoid_psi->Assemble();
real_t material_volume = int_sigmoid_psi->Sum();
#ifdef MFEM_USE_MPI
if (pfes)
{
MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
}
#endif
return material_volume;
}
// Poisson solver
@@ -551,8 +422,12 @@ void DiffusionSolver::SetupFEM()
}
}
void DiffusionSolver::UpdateEssentialTDofs()
void DiffusionSolver::Solve()
{
OperatorPtr A;
Vector B, X;
Array<int> ess_tdof_list;
#ifdef MFEM_USE_MPI
if (parallel)
{
@@ -565,39 +440,7 @@ void DiffusionSolver::UpdateEssentialTDofs()
#else
fes->GetEssentialTrueDofs(ess_bdr,ess_tdof_list);
#endif
}
void DiffusionSolver::AssembleDiffusionBilinear(bool update_ess_tdofs)
{
if (update_ess_tdofs)
{
UpdateEssentialTDofs();
}
#ifdef MFEM_USE_MPI
if (parallel)
{
a = new ParBilinearForm(pfes);
}
else
{
a = new BilinearForm(fes);
}
#else
a = new BilinearForm(fes);
#endif
a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
if (masscf)
{
a->AddDomainIntegrator(new MassIntegrator(*masscf));
}
a->Assemble();
a->FormSystemMatrix(ess_tdof_list, A);
}
void DiffusionSolver::Solve()
{
Vector B, X;
*u=0.0;
if (b)
{
delete b;
@@ -632,33 +475,31 @@ void DiffusionSolver::Solve()
b->Assemble();
*u=0.0;
if (essbdr_cf)
{
u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
}
BilinearForm * a = nullptr;
#ifdef MFEM_USE_MPI
if (parallel)
{
X.SetSize(pfes->TrueVSize());
B.SetSize(pfes->TrueVSize());
dynamic_cast<ParGridFunction*>(u)->ParallelAssemble(X);
dynamic_cast<ParLinearForm*>(b)->ParallelAssemble(B);
dynamic_cast<ParBilinearForm*>(a)->ParallelEliminateTDofsInRHS(
ess_tdof_list, X, B);
a = new ParBilinearForm(pfes);
}
else
{
X.NewDataAndSize(u->GetData(), u->Size());
B.NewDataAndSize(b->GetData(), b->Size());
a->EliminateVDofsInRHS(ess_tdof_list, X, B);
a = new BilinearForm(fes);
}
#else
X.NewDataAndSize(u->GetData(), u->Size());
B.NewDataAndSize(b->GetData(), b->Size());
a->EliminateVDofsInRHS(ess_tdof_list, X, B);
a = new BilinearForm(fes);
#endif
a->AddDomainIntegrator(new DiffusionIntegrator(*diffcf));
if (masscf)
{
a->AddDomainIntegrator(new MassIntegrator(*masscf));
}
a->Assemble();
if (essbdr_cf)
{
u->ProjectBdrCoefficient(*essbdr_cf,ess_bdr);
}
a->FormLinearSystem(ess_tdof_list, *u, *b, A, X, B);
CGSolver * cg = nullptr;
Solver * M = nullptr;
@@ -687,6 +528,7 @@ void DiffusionSolver::Solve()
delete M;
delete cg;
a->RecoverFEMSolution(X, *b, *u);
delete a;
}
GridFunction * DiffusionSolver::GetFEMSolution()
@@ -718,8 +560,6 @@ DiffusionSolver::~DiffusionSolver()
#endif
delete fec; fec = nullptr;
delete b;
A.Clear();
delete a;
}
+60 -11
View File
@@ -4,8 +4,8 @@
//
// Sample runs:
// mpirun -np 4 ex37p -alpha 10 -pv
// mpirun -np 4 ex37p -lambda 0.1 -mu 0.1 -growth 1
// mpirun -np 4 ex37p -o 2 -alpha 10.0 -mi 50 -vf 0.4 -ntol 1e-5 -growth 1.5
// mpirun -np 4 ex37p -lambda 0.1 -mu 0.1
// mpirun -np 4 ex37p -o 2 -alpha 5.0 -mi 50 -vf 0.4 -ntol 1e-5
// mpirun -np 4 ex37p -r 6 -o 2 -alpha 10.0 -epsilon 0.02 -mi 50 -ntol 1e-5
//
// Description: This example code demonstrates the use of MFEM to solve a
@@ -54,6 +54,61 @@
using namespace std;
using namespace mfem;
/**
* @brief Bregman projection of ρ = sigmoid(ψ) onto the subspace
* ∫_Ω ρ dx = θ vol(Ω) as follows:
*
* 1. Compute the root of the R → R function
* f(c) = ∫_Ω sigmoid(ψ + c) dx - θ vol(Ω)
* 2. Set ψ ← ψ + c.
*
* @param psi a GridFunction to be updated
* @param target_volume θ vol(Ω)
* @param tol Newton iteration tolerance
* @param max_its Newton maximum iteration number
* @return real_t Final volume, ∫_Ω sigmoid(ψ)
*/
real_t proj(ParGridFunction &psi, real_t target_volume, real_t tol=1e-12,
int max_its=10)
{
MappedGridFunctionCoefficient sigmoid_psi(&psi, sigmoid);
MappedGridFunctionCoefficient der_sigmoid_psi(&psi, der_sigmoid);
ParLinearForm int_sigmoid_psi(psi.ParFESpace());
int_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(sigmoid_psi));
ParLinearForm int_der_sigmoid_psi(psi.ParFESpace());
int_der_sigmoid_psi.AddDomainIntegrator(new DomainLFIntegrator(
der_sigmoid_psi));
bool done = false;
for (int k=0; k<max_its; k++) // Newton iteration
{
int_sigmoid_psi.Assemble(); // Recompute f(c) with updated ψ
real_t f = int_sigmoid_psi.Sum();
MPI_Allreduce(MPI_IN_PLACE, &f, 1, MPITypeMap<real_t>::mpi_type,
MPI_SUM, MPI_COMM_WORLD);
f -= target_volume;
int_der_sigmoid_psi.Assemble(); // Recompute df(c) with updated ψ
real_t df = int_der_sigmoid_psi.Sum();
MPI_Allreduce(MPI_IN_PLACE, &df, 1, MPITypeMap<real_t>::mpi_type,
MPI_SUM, MPI_COMM_WORLD);
const real_t dc = -f/df;
psi += dc;
if (abs(dc) < tol) { done = true; break; }
}
if (!done)
{
mfem_warning("Projection reached maximum iteration without converging. "
"Result may not be accurate.");
}
int_sigmoid_psi.Assemble();
real_t material_volume = int_sigmoid_psi.Sum();
MPI_Allreduce(MPI_IN_PLACE, &material_volume, 1,
MPITypeMap<real_t>::mpi_type, MPI_SUM, MPI_COMM_WORLD);
return material_volume;
}
/*
* ---------------------------------------------------------------
* ALGORITHM PREAMBLE
@@ -138,11 +193,10 @@ int main(int argc, char *argv[])
int ref_levels = 5;
int order = 2;
real_t alpha = 1.0;
real_t growth = 2;
real_t epsilon = 0.01;
real_t vol_fraction = 0.5;
int max_it = 1e3;
real_t itol = 1e-2;
real_t itol = 1e-1;
real_t ntol = 1e-4;
real_t rho_min = 1e-6;
real_t lambda = 1.0;
@@ -157,8 +211,6 @@ int main(int argc, char *argv[])
"Order (degree) of the finite elements.");
args.AddOption(&alpha, "-alpha", "--alpha-step-length",
"Step length for gradient descent.");
args.AddOption(&growth, "-growth", "--alpha-growth-rate",
"Growth rate of step length for gradient descent.");
args.AddOption(&epsilon, "-epsilon", "--epsilon-thickness",
"Length scale for ρ.");
args.AddOption(&max_it, "-mi", "--max-it",
@@ -307,7 +359,6 @@ int main(int argc, char *argv[])
}
FilterSolver->SetEssentialBoundary(ess_bdr_filter);
FilterSolver->SetupFEM();
FilterSolver->AssembleDiffusionBilinear();
ParBilinearForm mass(&control_fes);
mass.AddDomainIntegrator(new InverseIntegrator(new MassIntegrator(one)));
@@ -361,7 +412,7 @@ int main(int argc, char *argv[])
// 11. Iterate:
for (int k = 1; k <= max_it; k++)
{
if (k > 1) { alpha = std::pow((real_t) k,growth); }
if (k > 1) { alpha *= ((real_t) k) / ((real_t) k-1); }
if (myid == 0)
{
@@ -401,9 +452,7 @@ int main(int argc, char *argv[])
// Step 5 - Update design variable ψ ← proj(ψ - αG)
psi.Add(-alpha, grad);
ParGridFunction alpha_grad(grad);
alpha_grad *= alpha;
const real_t material_volume = proj(psi, alpha_grad, target_volume);
const real_t material_volume = proj(psi, target_volume);
// Compute ||ρ - ρ_old|| in control fes.
real_t norm_increment = zerogf.ComputeL1Error(succ_diff_rho);
+1 -1
View File
@@ -76,4 +76,4 @@ clean-build:
rm -rf *.dSYM *.TVD.*breakpoints
clean-exec:
@rm -f refined.mesh sol.gf mesh.* sol.*
@rm -f refined.mesh sol.gf
-548
View File
@@ -1,548 +0,0 @@
#include <mfem.hpp>
#ifdef MFEM_USE_PROTEUS
#include "../fem/dfem/util.hpp"
#include <proteus/CppJitModule.h>
#include "jitplayground.hpp"
#include <algorithm>
#include <array>
#include <cctype>
#include <cmath>
#include <fstream>
#include <initializer_list>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
namespace util
{
constexpr std::string_view Dirname(std::string_view path)
{
const size_t last_sep = path.find_last_of("/\\");
if (last_sep == std::string_view::npos) { return {}; }
return path.substr(0, last_sep);
}
constexpr std::string_view thisFileDir = Dirname(__FILE__);
}
template <typename T>
static std::string TypeNameString()
{
return std::string(mfem::future::get_type_name<T>());
}
template <typename Tuple, size_t... Is>
static auto ParamTypeStringsImpl(std::index_sequence<Is...>)
{
return std::array<std::string, sizeof...(Is)>
{
TypeNameString<std::remove_reference_t<decltype(mfem::future::get<Is>(std::declval<Tuple&>()))>>()...
};
}
template <typename Tuple>
static auto ParamTypeStrings()
{
return ParamTypeStringsImpl<Tuple>(
std::make_index_sequence<mfem::future::tuple_size<Tuple>::value> {});
}
static std::string_view Trim(std::string_view s)
{
size_t begin = 0;
while (begin < s.size() && std::isspace(static_cast<unsigned char>(s[begin])))
{
++begin;
}
size_t end = s.size();
while (end > begin &&
std::isspace(static_cast<unsigned char>(s[end - 1])))
{
--end;
}
return s.substr(begin, end - begin);
}
static bool IsValidIdentifier(std::string_view s)
{
if (s.empty()) { return false; }
const unsigned char c0 = static_cast<unsigned char>(s[0]);
if (!(std::isalpha(c0) || c0 == '_')) { return false; }
for (size_t i = 1; i < s.size(); ++i)
{
const unsigned char c = static_cast<unsigned char>(s[i]);
if (!(std::isalnum(c) || c == '_')) { return false; }
}
return true;
}
static bool ParseJitDirective(std::string_view line,
std::string &type,
std::string &var,
std::string &kind)
{
const size_t jit_pos = line.find("$JIT");
if (jit_pos == std::string_view::npos) { return false; }
const size_t open = line.find('[', jit_pos);
const size_t close = line.find(']', jit_pos);
MFEM_VERIFY(open != std::string_view::npos &&
close != std::string_view::npos &&
close > open,
"malformed $JIT directive (expected brackets): " << line);
const std::string_view payload = line.substr(open + 1, close - open - 1);
const size_t comma1 = payload.find(',');
const size_t comma2 = (comma1 == std::string_view::npos)
? std::string_view::npos
: payload.find(',', comma1 + 1);
MFEM_VERIFY(comma1 != std::string_view::npos &&
comma2 != std::string_view::npos,
"malformed $JIT directive (expected 3 comma-separated fields): "
<< line);
const std::string_view f0 = Trim(payload.substr(0, comma1));
const std::string_view f1 = Trim(payload.substr(comma1 + 1,
comma2 - comma1 - 1));
const std::string_view f2 = Trim(payload.substr(comma2 + 1));
MFEM_VERIFY(!f0.empty() && !f1.empty() && !f2.empty(),
"malformed $JIT directive (empty field): " << line);
type.assign(f0);
var.assign(f1);
kind.assign(f2);
return true;
}
static std::string ReadFileOrEmpty(const std::string &fn)
{
std::ifstream file(fn);
if (!file.is_open())
{
std::cerr << "could not open file " << fn << "\n";
return {};
}
std::stringstream buffer;
buffer << file.rdbuf();
return buffer.str();
}
static std::vector<std::string> ExtractJitVarNames(const std::string
&kernel_code)
{
std::stringstream ss(kernel_code);
std::string line;
std::vector<std::string> var_names;
std::unordered_set<std::string> seen_vars;
while (std::getline(ss, line))
{
std::string type, var, kind;
if (ParseJitDirective(line, type, var, kind))
{
MFEM_VERIFY(IsValidIdentifier(var),
"$JIT variable must be a valid identifier: " << var);
MFEM_VERIFY(seen_vars.insert(var).second,
"duplicate $JIT variable name: " << var);
var_names.push_back(var);
}
}
return var_names;
}
static std::string RewriteKernelForJit(std::string kernel_code,
const std::vector<std::string> &jit_values)
{
std::stringstream ss(kernel_code);
std::string line;
std::string out;
out.reserve(kernel_code.size() + 128);
bool have_pending = false;
size_t pending_index = 0;
std::string pending_type;
std::string pending_var;
std::unordered_set<std::string> seen_vars;
while (std::getline(ss, line))
{
line.push_back('\n');
if (have_pending)
{
MFEM_VERIFY(pending_index < jit_values.size(),
"not enough JIT values provided");
const size_t indent_end = line.find_first_not_of(" \t");
const std::string indent =
(indent_end == std::string::npos) ? std::string() :
line.substr(0, indent_end);
out += indent + "const " + pending_type + " " + pending_var + " = " +
jit_values[pending_index] + ";\n";
have_pending = false;
++pending_index;
continue;
}
std::string type, var, kind;
if (ParseJitDirective(line, type, var, kind))
{
MFEM_VERIFY(IsValidIdentifier(var),
"$JIT variable must be a valid identifier: " << var);
MFEM_VERIFY(kind == "generic",
"unsupported $JIT kind: " << kind);
MFEM_VERIFY(seen_vars.insert(var).second,
"duplicate $JIT variable name: " << var);
pending_type = std::move(type);
pending_var = std::move(var);
have_pending = true;
continue; // drop directive line
}
out += line;
}
MFEM_VERIFY(!have_pending,
"$JIT directive must annotate a following line");
MFEM_VERIFY(jit_values.size() == pending_index,
"JIT value count must match number of $JIT directives");
return out;
}
static std::string GeneratedOutputPath(std::string_view original_path)
{
const size_t last_sep = original_path.find_last_of("/\\");
const size_t dot = original_path.find_last_of('.');
const bool dot_in_filename =
(dot != std::string_view::npos) &&
(last_sep == std::string_view::npos || dot > last_sep);
const std::string_view base =
dot_in_filename ? original_path.substr(0, dot) : original_path;
return std::string(base) + "_generated.hpp";
}
static void WriteFileOrWarn(const std::string &path,
const std::string &contents)
{
std::ofstream out(path);
if (!out.is_open())
{
std::cerr << "could not write generated file " << path << "\n";
return;
}
out << contents;
}
class JitQFunction
{
public:
template <typename ImplT, size_t N>
JitQFunction(ImplT, const std::string &fn,
const std::array<bool, N> &activity_map)
{
using qf_signature = typename
mfem::future::get_function_signature<
decltype(&ImplT::operator())>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
constexpr size_t nparams = mfem::future::tuple_size<qf_param_ts>::value;
static_assert(N == nparams, "activity_map size must match qfunc arity");
this->fn = fn;
this->nparams = nparams;
this->activity_map.reserve(N);
for (size_t i = 0; i < N; ++i)
{
this->activity_map.push_back(activity_map[i]);
}
{
const auto param_types_arr = ParamTypeStrings<qf_param_ts>();
this->param_types.assign(param_types_arr.begin(), param_types_arr.end());
}
this->return_type = TypeNameString<typename qf_signature::return_t>();
this->return_is_void = std::is_same_v<typename qf_signature::return_t, void>;
this->impl_type_name = TypeNameString<ImplT>();
this->jit_var_names = ExtractJitVarNames(ReadFileOrEmpty(fn));
}
template <typename ReturnT, typename... Args>
ReturnT run(std::string_view name,
std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
Args&&... args)
{
auto ordered_values = MatchJitValues(jit_values);
auto &mod = GetOrCreateModule(ordered_values);
auto &instance = mod.instantiate(std::string(name), std::string());
return instance.template run<ReturnT>(std::forward<Args>(args)...);
}
template <typename ReturnT, typename... Args>
ReturnT run_primal(
std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
Args&&... args)
{
return run<ReturnT>(qfunc_name, jit_values,
std::forward<Args>(args)...);
}
template <typename ReturnT, typename... Args>
ReturnT run_derivative(
std::initializer_list<std::pair<std::string_view, std::string_view>> jit_values,
Args&&... args)
{
return run<ReturnT>(qfunc_name + "_fwddiff", jit_values,
std::forward<Args>(args)...);
}
private:
std::vector<std::string_view> MatchJitValues(
std::initializer_list<std::pair<std::string_view, std::string_view>>
named_values) const
{
std::unordered_map<std::string_view, std::string_view> value_map;
for (const auto &[name, value] : named_values)
{
value_map[name] = value;
}
std::vector<std::string_view> ordered_values;
ordered_values.reserve(jit_var_names.size());
for (const auto &var_name : jit_var_names)
{
auto it = value_map.find(var_name);
MFEM_VERIFY(it != value_map.end(),
"missing JIT value for variable: " << var_name);
ordered_values.push_back(it->second);
}
MFEM_VERIFY(ordered_values.size() == named_values.size(),
"provided " << named_values.size() << " JIT values but expected "
<< jit_var_names.size());
return ordered_values;
}
std::string BuildModuleCode(const std::vector<std::string> &jit_values) const
{
std::string module_code =
RewriteKernelForJit(ReadFileOrEmpty(fn), jit_values);
module_code += "\n\n";
module_code += "// --- generated ---\n";
module_code +=
"template <typename return_type, typename... Args>\n"
"return_type __enzyme_fwddiff(Args...);\n"
"\n"
"extern int enzyme_const;\n"
"extern int enzyme_dup;\n"
"\n";
// Generate a primal wrapper with the requested symbol name, so the kernel
// header can just define the qfunc as a functor.
//
// Note: Proteus instantiates entrypoints via `qfunc_wrapper<>(...)` even
// when there are no user template args, so keep the wrapper itself a
// template (with a default parameter) while still doing literal `$JIT`
// replacements in the kernel code.
module_code += "template <typename = void>\n";
module_code += return_type + " " +
std::string(qfunc_name) + "(";
bool first = true;
for (size_t i = 0; i < nparams; ++i)
{
if (!first) { module_code += ", "; }
first = false;
module_code += param_types[i] + " Arg" + std::to_string(i);
}
module_code += ")\n";
module_code += "{\n";
module_code += " " + impl_type_name + " qf;\n";
if (return_is_void)
{
module_code += " ";
}
else
{
module_code += " return ";
}
module_code += "qf(";
for (size_t i = 0; i < nparams; ++i)
{
if (i) { module_code += ", "; }
module_code += "Arg" + std::to_string(i);
}
module_code += ");\n";
module_code += "}\n\n";
module_code += "template <typename = void>\n";
module_code += return_type + " " +
std::string(qfunc_name) + "_fwddiff(";
first = true;
for (size_t i = 0; i < nparams; ++i)
{
if (!first) { module_code += ", "; }
first = false;
module_code += param_types[i] + " Arg" + std::to_string(i);
if (activity_map[i])
{
module_code += ", " + param_types[i] + " dArg" + std::to_string(i);
}
}
module_code += ")\n";
module_code += "{\n";
if (return_is_void)
{
module_code += " __enzyme_fwddiff<void>(\n";
}
else
{
module_code += " return __enzyme_fwddiff<" +
return_type + ">(\n";
}
module_code += " (void*)" + std::string(qfunc_name) + "<>";
module_code += ",\n";
for (size_t i = 0; i < nparams; ++i)
{
if (activity_map[i])
{
module_code += " enzyme_dup, Arg" + std::to_string(i) +
", dArg" + std::to_string(i);
}
else
{
module_code += " enzyme_const, Arg" + std::to_string(i);
}
module_code += (i + 1 == nparams) ? ");\n" : ",\n";
}
module_code += "}\n";
WriteFileOrWarn(GeneratedOutputPath(fn), module_code);
return module_code;
}
proteus::CppJitModule &GetOrCreateModule(
const std::vector<std::string_view> &jit_values)
{
std::string key;
for (const auto &val : jit_values)
{
if (!key.empty()) { key += ","; }
key += val;
}
auto it = modules.find(key);
if (it != modules.end())
{
return *it->second;
}
std::vector<std::string> values(jit_values.begin(), jit_values.end());
std::string code = BuildModuleCode(values);
auto mod = std::make_unique<proteus::CppJitModule>("host", code,
DefaultExtraArgs());
auto [inserted, ok] = modules.emplace(key, std::move(mod));
MFEM_VERIFY(ok, "failed to cache JIT module");
return *inserted->second;
}
static std::vector<std::string> DefaultExtraArgs()
{
return {"-fplugin=/Users/andrej1/local/enzyme/lib/ClangEnzyme-20.dylib"};
}
std::string qfunc_name = "qfunc_wrapper";
std::string fn;
size_t nparams = 0;
std::vector<bool> activity_map;
std::vector<std::string> param_types;
std::string return_type;
bool return_is_void = false;
std::string impl_type_name;
std::vector<std::string> jit_var_names;
std::unordered_map<std::string, std::unique_ptr<proteus::CppJitModule>> modules;
};
int main()
{
const size_t N = 4;
const size_t M = 5;
const double A = 123.4;
std::vector<double> X(N);
std::vector<double> Y(N);
for (size_t i = 0; i < N; ++i)
{
X[i] = static_cast<double>(i + 1);
Y[i] = static_cast<double>(N - i);
}
// // >>> user interface calls
// const std::string kernel_path = std::string(util::thisFileDir) +
// "/jitplayground.hpp";
// JitQFunction qf(daxpy_op{}, kernel_path, std::array{false, true, false});
// // <<< user interface calls
// // this will happen internally in dFEM
daxpy_op op;
printf("\n\nfunction call\n");
op(&A, X.data(), Y.data(), &N);
// reset X for the derivative test
for (size_t i = 0; i < N; ++i)
{
X[i] = static_cast<double>(i + 1);
Y[i] = static_cast<double>(N - i);
}
std::vector<double> dX(N, 1.0);
printf("\n\nforward diff call\n");
daxpy_op_fwddiff(&A, X.data(), dX.data(), Y.data(), &N);
std::vector<double> dX_manual(N, A);
printf("\n\nderivative checks\n");
std::cout << "dX: ";
for (size_t i = 0; i < N; ++i)
{
std::cout << dX[i] << (i + 1 == N ? '\n' : ' ');
}
std::cout << "dX_manual: ";
for (size_t i = 0; i < N; ++i)
{
std::cout << dX_manual[i] << (i + 1 == N ? '\n' : ' ');
}
double max_abs_err = 0.0;
for (size_t i = 0; i < N; ++i)
{
max_abs_err = std::max(max_abs_err, std::abs(dX[i] - dX_manual[i]));
}
std::cout << "max |dX - dX_manual| = " << max_abs_err << "\n";
return 0;
}
#else
int main()
{
std::cerr << "Proteus is required to run this example.\n";
return EXIT_FAILURE;
}
#endif // MFEM_USE_PROTEUS
-58
View File
@@ -1,58 +0,0 @@
#pragma once
#include <cstddef>
#include <vector>
#include <type_traits>
#include "proteus/JitInterface.h"
struct daxpy_op
{
void operator()(
const double *a,
double *x,
const double *y,
const size_t *N) const
{
const size_t n = *N;
auto lam = [=, n = proteus::jit_variable(n)]
() __attribute__((annotate("jit")))
{
printf("N = %zu\n", n);
for (size_t i = 0; i < n; ++i)
{
printf("x[%zu] = %f, y[%zu] = %f\n", i, x[i], i, y[i]);
x[i] = *a * x[i] + y[i];
printf("updated x[%zu] = %f\n", i, x[i]);
}
};
proteus::register_lambda(lam);
lam();
}
};
template <typename return_type, typename... Args>
return_type __enzyme_fwddiff(Args...);
extern int enzyme_const;
extern int enzyme_dup;
void daxpy_op_wrapper(const double * Arg0, double * Arg1,
const double * Arg2, const size_t *Arg3)
{
daxpy_op qf;
qf(Arg0, Arg1, Arg2, Arg3);
}
void daxpy_op_fwddiff(const double * Arg0, double * Arg1,
double * dArg1, const double * Arg2, const size_t *Arg3)
{
__enzyme_fwddiff<void>(
(void*)daxpy_op_wrapper,
enzyme_const, Arg0,
enzyme_dup, Arg1, dArg1,
enzyme_const, Arg2,
enzyme_const, Arg3);
}
+2 -7
View File
@@ -71,7 +71,6 @@ endif
SUBDIRS_ALL = $(addsuffix /all,$(SUBDIRS))
SUBDIRS_TEST = $(addsuffix /test,$(SUBDIRS))
SUBDIRS_TEST_NOCLEAN = $(addsuffix /test-noclean,$(SUBDIRS))
SUBDIRS_CLEAN = $(addsuffix /clean,$(SUBDIRS))
SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
@@ -88,9 +87,8 @@ SUBDIRS_TPRINT = $(addsuffix /test-print,$(SUBDIRS))
all: $(EXAMPLES) $(SUBDIRS_ALL)
.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) \
$(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_TEST_NOCLEAN) $(SUBDIRS_CLEAN):
.PHONY: $(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN) $(SUBDIRS_TPRINT)
$(SUBDIRS_ALL) $(SUBDIRS_TEST) $(SUBDIRS_CLEAN):
$(MAKE) -C $(@D) $(@F)
$(SUBDIRS_TPRINT):
@$(MAKE) -C $(@D) $(@F)
@@ -109,7 +107,6 @@ endif
MFEM_TESTS = EXAMPLES
include $(MFEM_TEST_MK)
test: $(SUBDIRS_TEST)
test-noclean: $(SUBDIRS_TEST_NOCLEAN)
test-print: $(SUBDIRS_TPRINT)
# Testing: Parallel vs. serial runs
@@ -160,8 +157,6 @@ ex37-test-seq: ex37
@$(call mfem-test,$<,, Serial example,-mi 3)
ex37p-test-par: ex37p
@$(call mfem-test,$<, $(RUN_MPI), Parallel example,-mi 3)
ex39-test-seq: ex39
@$(call mfem-test,$<,, Serial example,-m ../data/compass.mesh)
ex41-test-seq: ex41
@$(call mfem-test,$<,, Serial example,-tf 1.0)
ex41p-test-par: ex41p
+16 -49
View File
@@ -65,7 +65,6 @@ set(SRCS
dgmassinv.cpp
doftrans.cpp
dfem/doperator.cpp
dfem/backends/local_qf/kernels.cpp
eltrans.cpp
batchitrans.cpp
estimators.cpp
@@ -122,11 +121,6 @@ set(SRCS
qinterp/eval_hdiv.cpp
qinterp/grad_by_nodes.cpp
qinterp/grad_by_vdim.cpp
qinterp/grad_transpose.cpp
qinterp/grad_transpose_by_nodes.cpp
qinterp/grad_transpose_by_vdim.cpp
qinterp/eval_transpose.cpp
qinterp/eval_transpose_by_vdim.cpp
qspace.cpp
quadinterpolator.cpp
quadinterpolator_face.cpp
@@ -139,7 +133,7 @@ set(SRCS
tmop/assemble/diag2.cpp
tmop/assemble/grad2_limit.cpp
tmop/assemble/grad2.cpp
tmop/assemble/diag3_limit.cpp
tmop/assemble/diag3_limit.cpp
tmop/assemble/diag3.cpp
tmop/assemble/grad3_limit.cpp
tmop/assemble/grad3.cpp
@@ -177,12 +171,8 @@ set(SRCS
tmop_tools.cpp
tmop_amr.cpp
gslib.cpp
gslib/findptsedge_local_2.cpp
gslib/findptsedge_local_3.cpp
gslib/findptssurf_local_3.cpp
gslib/findpts_local_2.cpp
gslib/findpts_local_3.cpp
gslib/interpolate_local_1.cpp
gslib/interpolate_local_2.cpp
gslib/interpolate_local_3.cpp
transfer.cpp
@@ -201,14 +191,12 @@ set(HDRS
integ/bilininteg_dgtrace_kernels.hpp
integ/bilininteg_vecdiffusion_kernels.hpp
integ/bilininteg_convection_kernels.hpp
integ/bilininteg_diffusion_pa_simplices.hpp
integ/bilininteg_diffusion_kernels.hpp
integ/bilininteg_elasticity_kernels.hpp
integ/bilininteg_hcurl_kernels.hpp
integ/bilininteg_hdiv_kernels.hpp
integ/bilininteg_hcurlhdiv_kernels.hpp
integ/bilininteg_mass_kernels.hpp
integ/bilininteg_mass_pa_simplices.hpp
integ/bilininteg_vecdiffusion_pa.hpp
integ/bilininteg_vecmass_pa.hpp
coefficient.hpp
@@ -219,31 +207,13 @@ set(HDRS
dgmassinv_kernels.hpp
doftrans.hpp
dfem/doperator.hpp
dfem/fielddescriptor.hpp
dfem/fieldoperator.hpp
dfem/integrator_ctx.hpp
dfem/integrate.hpp
dfem/parameterspace.hpp
dfem/qfunction_apply.hpp
dfem/qfunction_transform.hpp
dfem/tuple.hpp
dfem/util.hpp
dfem/backends/util.hpp
dfem/backends/global_qf/action.hpp
dfem/backends/global_qf/derivative_action.hpp
dfem/backends/global_qf/derivative_apply.hpp
dfem/backends/global_qf/derivative_apply_transpose.hpp
dfem/backends/global_qf/derivative_setup.hpp
dfem/backends/global_qf/prelude.hpp
dfem/backends/local_qf/action.hpp
dfem/backends/local_qf/derivative_action.hpp
dfem/backends/local_qf/derivative_apply.hpp
dfem/backends/local_qf/derivative_apply_transpose.hpp
dfem/backends/local_qf/derivative_assemble.hpp
dfem/backends/local_qf/derivative_assemble_diagonal.hpp
dfem/backends/local_qf/derivative_setup.hpp
dfem/backends/local_qf/kernels.hpp
dfem/backends/local_qf/kernels_ho.hpp
dfem/backends/local_qf/kernels_lo.hpp
dfem/backends/local_qf/prelude.hpp
dfem/backends/local_qf/util.hpp
eltrans.hpp
estimators.hpp
fe.hpp
@@ -308,10 +278,8 @@ set(HDRS
qfunction.hpp
qinterp/det.hpp
qinterp/eval.hpp
qinterp/eval_transpose.hpp
qinterp/eval_hdiv.hpp
qinterp/grad.hpp
qinterp/grad_transpose.hpp
qspace.hpp
quadinterpolator.hpp
quadinterpolator_face.hpp
@@ -337,7 +305,6 @@ set(HDRS
tmop_tools.hpp
tmop_amr.hpp
gslib.hpp
gslib/gslib_kernel_helpers.hpp
transfer.hpp
hyperbolic.hpp
integrator.hpp
@@ -346,36 +313,36 @@ set(HDRS
)
if (MFEM_USE_SIDRE)
list(APPEND SRCS sidredatacollection.cpp)
list(APPEND HDRS sidredatacollection.hpp)
list(APPEND SRCS sidredatacollection.cpp)
list(APPEND HDRS sidredatacollection.hpp)
endif()
if (MFEM_USE_CONDUIT)
list(APPEND SRCS conduitdatacollection.cpp)
list(APPEND HDRS conduitdatacollection.hpp)
list(APPEND SRCS conduitdatacollection.cpp)
list(APPEND HDRS conduitdatacollection.hpp)
endif()
if (MFEM_USE_ADIOS2)
list(APPEND SRCS adios2datacollection.cpp)
list(APPEND HDRS adios2datacollection.hpp)
list(APPEND SRCS adios2datacollection.cpp)
list(APPEND HDRS adios2datacollection.hpp)
endif()
if (MFEM_USE_FMS)
list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
list(APPEND SRCS fmsdatacollection.cpp fmsconvert.cpp)
list(APPEND HDRS fmsdatacollection.hpp fmsconvert.hpp)
endif()
if (MFEM_USE_MPI)
list(APPEND SRCS
list(APPEND SRCS
pbilinearform.cpp
pfespace.cpp
pgridfunc.cpp
plinearform.cpp
pnonlinearform.cpp
prestriction.cpp)
# If this list (HDRS -> HEADERS) is used for install, we probably want the
# headers added all the time.
list(APPEND HDRS
# If this list (HDRS -> HEADERS) is used for install, we probably want the
# headers added all the time.
list(APPEND HDRS
pbilinearform.hpp
pfespace.hpp
pgridfunc.hpp
+3 -7
View File
@@ -729,8 +729,7 @@ void BilinearForm::Assemble(int skip_zeros)
tr = mesh -> GetBdrFaceTransformations (i);
if (tr != NULL)
{
mfem::DofTransformation doftrans;
fes -> GetElementVDofs (tr -> Elem1No, vdofs, doftrans);
fes -> GetElementVDofs (tr -> Elem1No, vdofs);
fe1 = fes -> GetFE (tr -> Elem1No);
// The fe2 object is really a dummy and not used on the boundaries,
// but we can't dereference a NULL pointer, and we don't want to
@@ -744,7 +743,6 @@ void BilinearForm::Assemble(int skip_zeros)
boundary_face_integs[k] -> AssembleFaceMatrix (*fe1, *fe2, *tr,
elemmat);
doftrans.TransformDual(elemmat);
mat -> AddSubMatrix (vdofs, vdofs, elemmat, skip_zeros);
}
}
@@ -1725,7 +1723,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
}
}
DofTransformation dom_dof_trans, ran_dof_trans;
for (int i = 0; i < trial_fes -> GetNBE(); i++)
{
const int bdr_attr = mesh->GetBdrAttribute(i);
@@ -1734,8 +1731,8 @@ void MixedBilinearForm::Assemble(int skip_zeros)
ftr = mesh -> GetBdrFaceTransformations (i);
if (ftr != NULL)
{
trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs, dom_dof_trans);
test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs, ran_dof_trans);
trial_fes->GetElementVDofs(ftr->Elem1No, trial_vdofs);
test_fes->GetElementVDofs(ftr->Elem1No, test_vdofs);
trial_fe1 = trial_fes->GetFE(ftr->Elem1No);
test_fe1 = test_fes->GetFE(ftr->Elem1No);
// The test_fe2 object is really a dummy and not used on the
@@ -1751,7 +1748,6 @@ void MixedBilinearForm::Assemble(int skip_zeros)
boundary_face_integs[k]->AssembleFaceMatrix(*trial_fe1, *test_fe1, *trial_fe2,
*test_fe2,
*ftr, elemmat);
TransformDual(ran_dof_trans, dom_dof_trans, elemmat);
mat->AddSubMatrix(test_vdofs, trial_vdofs, elemmat, skip_zeros);
}
}
+4 -22
View File
@@ -1345,8 +1345,7 @@ real_t DiffusionIntegrator::ComputeFluxEnergy
}
const IntegrationRule &DiffusionIntegrator::GetRule(
const FiniteElement &trial_fe, const FiniteElement &test_fe,
const bool stroud)
const FiniteElement &trial_fe, const FiniteElement &test_fe)
{
int order;
if (trial_fe.Space() == FunctionSpace::Pk)
@@ -1363,15 +1362,7 @@ const IntegrationRule &DiffusionIntegrator::GetRule(
{
return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
}
if (stroud)
{
return StroudIntRules.Get(trial_fe.GetGeomType(), order);
}
else
{
return IntRules.Get(trial_fe.GetGeomType(), order);
}
return IntRules.Get(trial_fe.GetGeomType(), order);
}
MassIntegrator::MassIntegrator(const IntegrationRule *ir)
@@ -1458,8 +1449,7 @@ void MassIntegrator::AssembleElementMatrix2(
const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
const FiniteElement &test_fe,
const ElementTransformation &Trans,
const bool stroud)
const ElementTransformation &Trans)
{
// int order = trial_fe.GetOrder() + test_fe.GetOrder();
const int order = trial_fe.GetOrder() + test_fe.GetOrder() + Trans.OrderW();
@@ -1468,15 +1458,7 @@ const IntegrationRule &MassIntegrator::GetRule(const FiniteElement &trial_fe,
{
return RefinedIntRules.Get(trial_fe.GetGeomType(), order);
}
if (stroud)
{
return StroudIntRules.Get(trial_fe.GetGeomType(), order);
}
else
{
return IntRules.Get(trial_fe.GetGeomType(), order);
}
return IntRules.Get(trial_fe.GetGeomType(), order);
}
+3 -41
View File
@@ -2184,22 +2184,11 @@ public:
const Vector&, const Vector&,
Vector&, const int, const int);
using ApplySimplexKernelType = void(*)(const int, const bool, const Array<int>&,
const Array<int>&,
const Array<int>&, const Array<int>&, const Array<int>&,
const Array<real_t>&, const Array<real_t>&,
const Array<real_t>&, const Array<real_t>&,
const Array<real_t>&, const Array<real_t>&,
const Vector&, const Vector&,
Vector&, const int, const int);
using DiagonalKernelType = void(*)(const int, const bool, const Array<real_t>&,
const Array<real_t>&, const Vector&, Vector&,
const int, const int);
MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
int));
MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
struct Kernels { Kernels(); };
@@ -2352,8 +2341,7 @@ public:
void AddMultPatchPA(const int patch, const Vector &x, Vector &y) const;
static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
const FiniteElement &test_fe,
const bool stroud = false);
const FiniteElement &test_fe);
bool SupportsCeed() const override { return DeviceCanUseCeed(); }
@@ -2364,13 +2352,6 @@ public:
{
ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
AddSimplexSpecialization<DIM,D1D,Q1D>();
}
template <int DIM, int D1D, int Q1D>
static void AddSimplexSpecialization()
{
ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
}
protected:
const IntegrationRule* GetDefaultIntegrationRule(
@@ -2407,22 +2388,11 @@ public:
const Array<real_t>&, const Vector&,
const Vector&, Vector&, const int, const int);
using ApplySimplexKernelType = void(*)(const int, const Array<int>&,
const Array<int>&,
const Array<int>&, const Array<int>&, const Array<int>&,
const Array<real_t>&, const Array<real_t>&,
const Array<real_t>&, const Array<real_t>&,
const Array<real_t>&, const Array<real_t>&,
const Vector&, const Vector&, Vector&,
const int, const int);
using DiagonalKernelType = void(*)(const int, const Array<real_t>&,
const Vector&, Vector&, const int,
const int);
MFEM_REGISTER_KERNELS(ApplyPAKernels, ApplyKernelType, (int, int, int));
MFEM_REGISTER_KERNELS(ApplySimplexPAKernels, ApplySimplexKernelType, (int, int,
int));
MFEM_REGISTER_KERNELS(DiagonalPAKernels, DiagonalKernelType, (int, int, int));
struct Kernels { Kernels(); };
@@ -2471,8 +2441,7 @@ public:
static const IntegrationRule &GetRule(const FiniteElement &trial_fe,
const FiniteElement &test_fe,
const ElementTransformation &Trans,
const bool stroud = false);
const ElementTransformation &Trans);
bool SupportsCeed() const override { return DeviceCanUseCeed(); }
@@ -2483,13 +2452,6 @@ public:
{
ApplyPAKernels::Specialization<DIM,D1D,Q1D>::Add();
DiagonalPAKernels::Specialization<DIM,D1D,Q1D>::Add();
AddSimplexSpecialization<DIM,D1D,Q1D>();
}
template <int DIM, int D1D, int Q1D>
static void AddSimplexSpecialization()
{
ApplySimplexPAKernels::Specialization<DIM,D1D,Q1D>::Add();
}
protected:
@@ -2748,7 +2710,7 @@ public:
/** Integrator for $(-Q u, \nabla v)$ for Nedelec ($u$) and $H^1$ ($v$) elements.
This is equivalent to a weak divergence of the $H(curl)$ basis functions. */
This is equivalent to a weak divergence of the $H(curl$ basis functions. */
class VectorFEWeakDivergenceIntegrator: public BilinearFormIntegrator
{
protected:
-6
View File
@@ -54,8 +54,6 @@ void Coefficient::Project(QuadratureFunction &qf)
QuadratureSpaceBase &qspace = *qf.GetSpace();
const int ne = qspace.GetNE();
Vector values;
// GetValues makes a reference, but we need it to be valid on Host
qf.HostWrite();
for (int iel = 0; iel < ne; ++iel)
{
qf.GetValues(iel, values);
@@ -329,8 +327,6 @@ void VectorCoefficient::Project(QuadratureFunction &qf)
const int ne = qspace.GetNE();
DenseMatrix values;
Vector col;
// GetValues makes a reference, but we need it to be valid on Host
qf.HostWrite();
for (int iel = 0; iel < ne; ++iel)
{
qf.GetValues(iel, values);
@@ -699,8 +695,6 @@ void MatrixCoefficient::Project(QuadratureFunction &qf, bool transpose)
QuadratureSpaceBase &qspace = *qf.GetSpace();
const int ne = qspace.GetNE();
DenseMatrix values, matrix;
// GetValues makes a reference, but we need it to be valid on Host
qf.HostWrite();
for (int iel = 0; iel < ne; ++iel)
{
qf.GetValues(iel, values);
-3
View File
@@ -52,9 +52,6 @@ public:
/// Get the time for time dependent coefficients
real_t GetTime() { return time; }
/// Returns dimension of the vector.
int GetVDim() { return 1; }
/** @brief Evaluate the coefficient in the element described by @a T at the
point @a ip. */
/** @note When this method is called, the caller must make sure that the
+26 -13
View File
@@ -830,9 +830,15 @@ ParComplexGridFunction::ParComplexGridFunction(ParMesh *m, std::istream &input)
int vsize = pfes->GetVSize();
Vector::Load(input, 2*vsize);
real_t *h_data = HostReadWrite();
pfes->ApplyDofSigns(h_data);
pfes->ApplyDofSigns(h_data + vsize);
real_t *data_ = const_cast<real_t*>(HostRead());
for (int i = 0; i < vsize; i++)
{
if (pfes->GetDofSign(i) < 0)
{
data_[i] = -data_[i];
data_[i+vsize] = -data_[i+vsize];
}
}
// if the mesh is a legacy (v1.1) NC mesh, it has old vertex ordering
@@ -1045,14 +1051,15 @@ void ParComplexGridFunction::Save(std::ostream &os) const
os << '\n';
int vsize = pfes->GetVSize();
// We use const_cast + HostRead (instead of HostReadWrite) because we only
// need to change the host data temporarily and this way we do not invalidate
// the data if it is on device. If we use HostReadWrite here, later calls to
// Read or ReadWrite will need to copy the data from host to device. With the
// approach used here, the host-to-device copy is avoided.
real_t *h_data = const_cast<real_t*>(HostRead());
pfes->ApplyDofSigns(h_data);
pfes->ApplyDofSigns(h_data + vsize);
real_t *data_ = const_cast<real_t*>(HostRead());
for (int i = 0; i < vsize; i++)
{
if (pfes->GetDofSign(i) < 0)
{
data_[i] = -data_[i];
data_[i+vsize] = -data_[i+vsize];
}
}
if (pfes->GetOrdering() == Ordering::byNODES)
{
@@ -1063,8 +1070,14 @@ void ParComplexGridFunction::Save(std::ostream &os) const
Vector::Print(os, pfes->GetVDim());
}
pfes->ApplyDofSigns(h_data);
pfes->ApplyDofSigns(h_data + vsize);
for (int i = 0; i < vsize; i++)
{
if (pfes->GetDofSign(i) < 0)
{
data_[i] = -data_[i];
data_[i+vsize] = -data_[i+vsize];
}
}
os.flush();
}
-19
View File
@@ -82,25 +82,6 @@ public:
/// underlying #fes
int VectorDim() const;
/// Copy assignment. Only the data of the base class Vector is copied.
/** It is assumed that this object and @a rhs use FiniteElementSpace%s that
have the same size.
@note Defining this method overwrites the implicitly defined copy
assignment operator. */
ComplexGridFunction &operator=(const ComplexGridFunction &rhs)
{ return operator=((const Vector &)rhs); }
/// Copy the data from @a v.
/** The size of @a v must be equal to double of the size of the associated
FiniteElementSpace #fes. */
ComplexGridFunction &operator=(const Vector &v)
{
MFEM_ASSERT(fes && v.Size() == 2*fes->GetVSize(), "");
Vector::operator=(v);
return *this;
}
/// Assign constant values to the ComplexGridFunction data.
ComplexGridFunction &operator=(const std::complex<real_t> & value)
{ *gfr = value.real(); *gfi = value.imag(); return *this; }
-4
View File
@@ -114,10 +114,6 @@ void ConduitDataCollection::Save()
n_mesh["fields"][name]);
}
// TODO: in parallel, we need to call ParFiniteElementSpace::ApplyDofSigns
// for all ParGridFunction objects before and after saving, see
// ParGridFunction::Save.
// save mesh data
SaveMeshAndFields(myid,
n_mesh,
+5 -18
View File
@@ -492,8 +492,6 @@ void VisItDataCollection::SaveRootFile()
to_padded_string(cycle, pad_digits_cycle) +
".mfem_root";
std::ofstream root_file(root_name);
MFEM_VERIFY(root_file.is_open(),
"Failed to open ofstream " << root_name);
root_file << GetVisItRootString();
if (!root_file)
{
@@ -979,10 +977,7 @@ void ParaViewDataCollection::Save()
// Save the local part of the mesh and grid functions fields to the local
// VTU file. Also save coefficient fields.
{
std::string os_str = vtu_prefix + GenerateVTUFileName("proc", myid);
std::ofstream os(os_str);
MFEM_VERIFY(os.is_open(),
"Failed to open ofstream " << os_str);
std::ofstream os(vtu_prefix + GenerateVTUFileName("proc", myid));
os.precision(precision);
SaveDataVTU(os, levels_of_detail);
}
@@ -994,10 +989,7 @@ void ParaViewDataCollection::Save()
"QuadratureFunction output is not supported for "
"ParaViewDataCollection on domain boundary!");
const std::string &field_name = qfield.first;
std::string os_str = vtu_prefix + GenerateVTUFileName(field_name, myid);
std::ofstream os(os_str);
MFEM_VERIFY(os.is_open(),
"Failed to open ofstream " << os_str);
std::ofstream os(vtu_prefix + GenerateVTUFileName(field_name, myid));
qfield.second->SaveVTU(os, pv_data_format, GetCompressionLevel(), field_name);
}
@@ -1008,10 +1000,7 @@ void ParaViewDataCollection::Save()
{
// Create the main PVTU file
{
std::string os_str = vtu_prefix + GeneratePVTUFileName("data");
std::ofstream pvtu_out(os_str);
MFEM_VERIFY(pvtu_out.is_open(),
"Failed to open ofstream " << os_str);
std::ofstream pvtu_out(vtu_prefix + GeneratePVTUFileName("data"));
WritePVTUHeader(pvtu_out);
// Grid function fields and coefficient fields
@@ -1066,10 +1055,8 @@ void ParaViewDataCollection::Save()
const std::string &q_field_name = q_field.first;
std::string q_fname = GeneratePVTUPath() + "/"
+ GeneratePVTUFileName(q_field_name);
std::string os_str = col_path + "/" + q_fname;
std::ofstream pvtu_out(os_str);
MFEM_VERIFY(pvtu_out.is_open(),
"Failed to open ofstream " << os_str);
std::ofstream pvtu_out(col_path + "/" + q_fname);
WritePVTUHeader(pvtu_out);
int vec_dim = q_field.second->GetVDim();
pvtu_out << "<PPointData>\n";
+403
View File
@@ -0,0 +1,403 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "util.hpp"
namespace mfem::future
{
/// @brief Assemble element matrix for three dimensional data.
///
/// Note: In the below layouts, total_trial_op_dim is > 1 if
/// there are more than one inputs dependent on the derivative variable.
///
/// @param A Memory for one element matrix with layout
/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
/// @param fhat Memory to hold the residual computation with layout
/// [test_vdim, test_op_dim, nqp].
/// @param qpdc The quadrature point data cache with data layout
/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
/// @param itod Input Trial Operator Dimension array. If the trial
/// operator is not dependent, the dimension is 0 to indicate that.
/// @param inputs The input field operator types.
/// @param output The output field operator types.
/// @param input_dtqmaps The input DofToQuad maps.
/// @param output_dtqmap The output DofToQuad maps.
/// @param scratch_shmem Scratch shared memory for computations.
/// @param q1d The number of quadrature points in one dimension.
/// @param td1d The number of trial dofs in one dimension.
template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
MFEM_HOST_DEVICE void assemble_element_mat_t3d(
const DeviceTensor<4, real_t>& A,
const DeviceTensor<3, real_t>& fhat,
const DeviceTensor<5, const real_t>& qpdc,
const DeviceTensor<1, const real_t>& itod,
const input_fop_ts& inputs,
const output_fop_t& output,
const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
const DofToQuadMap& output_dtqmap,
std::array<DeviceTensor<1>, 6>& scratch_shmem,
const int& q1d,
const int& td1d)
{
constexpr int dimension = 3;
// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
const int test_vdim = qpdc.GetShape()[0];
const int test_op_dim = qpdc.GetShape()[1];
const int trial_vdim = qpdc.GetShape()[2];
// [num_test_dof, ...]
const auto num_test_dof = A.GetShape()[0];
for (int Jx = 0; Jx < td1d; Jx++)
{
for (int Jy = 0; Jy < td1d; Jy++)
{
for (int Jz = 0; Jz < td1d; Jz++)
{
const int J = Jx + td1d * (Jy + td1d * Jz);
for (int j = 0; j < trial_vdim; j++)
{
for (int tv = 0; tv < test_vdim; tv++)
{
for (int tod = 0; tod < test_op_dim; tod++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
fhat(tv, tod, q) = 0.0;
}
}
}
}
}
// MSVC lambda capture workaround
[[maybe_unused]] const auto& inputs_ref = inputs;
int m_offset = 0;
for_constexpr<num_inputs>([&](auto s)
{
using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
if (trial_op_dim == 0)
{
// This is inside a lambda so we have to return
// instead of idiomatic 'continue'.
return;
}
auto& B = input_dtqmaps[s].B;
auto& G = input_dtqmaps[s].G;
if constexpr (is_value_fop<fop_t>::value)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
for (int m = 0; m < trial_op_dim; m++)
{
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t f = qpdc(i, k, j, m + m_offset, q);
fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
}
}
}
}
}
}
}
else if constexpr (is_gradient_fop<fop_t>::value)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
for (int m = 0; m < trial_op_dim; m++)
{
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t f = qpdc(i, k, j, m + m_offset, q);
if (m == 0)
{
fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy) * B(qz, 0, Jz);
}
else if (m == 1)
{
fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy) * B(qz, 0, Jz);
}
else if (m == 2)
{
fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy) * G(qz, 0, Jz);
}
}
}
}
}
}
}
}
else
{
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
MFEM_ABORT("sum factorized sparse matrix assemble routine "
"not implemented for field operator");
#endif
}
MFEM_SYNC_THREAD;
m_offset += trial_op_dim;
});
auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
scratch_shmem, dimension, true);
}
}
}
}
}
/// @brief Assemble element matrix for two dimensional data.
///
/// Note: In the below layouts, total_trial_op_dim is > 1 if
/// there are more than one inputs dependent on the derivative variable.
///
/// @param A Memory for one element matrix with layout
/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
/// @param fhat Memory to hold the residual computation with layout
/// [test_vdim, test_op_dim, nqp].
/// @param qpdc The quadrature point data cache with data layout
/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
/// @param itod Input Trial Operator Dimension array. If the trial
/// operator is not dependent, the dimension is 0 to indicate that.
/// @param inputs The input field operator types.
/// @param output The output field operator types.
/// @param input_dtqmaps The input DofToQuad maps.
/// @param output_dtqmap The output DofToQuad maps.
/// @param scratch_shmem Scratch shared memory for computations.
/// @param q1d The number of quadrature points in one dimension.
/// @param td1d The number of trial dofs in one dimension.
template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
MFEM_HOST_DEVICE void assemble_element_mat_t2d(
const DeviceTensor<4, real_t>& A,
const DeviceTensor<3, real_t>& fhat,
const DeviceTensor<5, const real_t>& qpdc,
const DeviceTensor<1, const real_t>& itod,
const input_fop_ts& inputs,
const output_fop_t& output,
const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
const DofToQuadMap& output_dtqmap,
std::array<DeviceTensor<1>, 6>& scratch_shmem,
const int& q1d,
const int& td1d)
{
constexpr int dimension = 2;
// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, num_qp]
const int test_vdim = qpdc.GetShape()[0];
const int test_op_dim = qpdc.GetShape()[1];
const int trial_vdim = qpdc.GetShape()[2];
// [num_test_dof, ...]
const auto num_test_dof = A.GetShape()[0];
for (int Jx = 0; Jx < td1d; Jx++)
{
for (int Jy = 0; Jy < td1d; Jy++)
{
const int J = Jy + Jx * td1d;
for (int j = 0; j < trial_vdim; j++)
{
for (int tv = 0; tv < test_vdim; tv++)
{
for (int tod = 0; tod < test_op_dim; tod++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
const int q = qy + qx * q1d;
fhat(tv, tod, q) = 0.0;
}
}
}
}
// MSVC lambda capture workaround
[[maybe_unused]] const auto& inputs_ref = inputs;
int m_offset = 0;
for_constexpr<num_inputs>([&](auto s)
{
using fop_t = std::decay_t<decltype(get<s>(inputs_ref))>;
const int trial_op_dim = static_cast<int>(itod(static_cast<int>(s)));
if (trial_op_dim == 0)
{
// This is inside a lambda so we have to return
// instead of idiomatic 'continue'.
return;
}
auto& B = input_dtqmaps[s].B;
auto& G = input_dtqmaps[s].G;
if constexpr (is_value_fop<fop_t>::value)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
const int q = qy + qx * q1d;
for (int m = 0; m < trial_op_dim; m++)
{
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t f = qpdc(i, k, j, m + m_offset, q);
fhat(i, k, q) += f * B(qx, 0, Jx) * B(qy, 0, Jy);
}
}
}
}
}
}
else if constexpr (is_gradient_fop<fop_t>::value)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
const int q = qy + qx * q1d;
for (int m = 0; m < trial_op_dim; m++)
{
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t f = qpdc(i, k, j, m + m_offset, q);
if (m == 0)
{
fhat(i, k, q) += f * B(qx, 0, Jx) * G(qy, 0, Jy);
}
else
{
fhat(i, k, q) += f * G(qx, 0, Jx) * B(qy, 0, Jy);
}
}
}
}
}
}
}
else
{
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
MFEM_ABORT("sum factorized sparse matrix assemble routine "
"not implemented for field operator");
#endif
}
MFEM_SYNC_THREAD;
m_offset += trial_op_dim;
});
auto bvtfhat = Reshape(&A(0, 0, J, j), num_test_dof, test_vdim);
map_quadrature_data_to_fields(bvtfhat, fhat, output, output_dtqmap,
scratch_shmem, dimension, true);
}
}
}
}
/// @brief Assemble element matrix for two or three dimensional data.
///
/// Note: In the below layouts, total_trial_op_dim is > 1 if
/// there are more than one inputs dependent on the derivative variable.
///
/// @param A Memory for one element matrix with layout
/// [test_ndof, test_vdim, trial_ndof, trial_vdim].
/// @param fhat Memory to hold the residual computation with layout
/// [test_vdim, test_op_dim, nqp].
/// @param qpdc The quadrature point data cache with data layout
/// [test_vdim, test_op_dim, trial_vdim, total_trial_op_dim, nqp].
/// @param itod Input Trial Operator Dimension array. If the trial
/// operator is not dependent, the dimension is 0 to indicate that.
/// @param inputs The input field operator types.
/// @param output The output field operator types.
/// @param input_dtqmaps The input DofToQuad maps.
/// @param output_dtqmap The output DofToQuad maps.
/// @param scratch_shmem Scratch shared memory for computations.
/// @param dimension The spatial dimension.
/// @param q1d The number of quadrature points in one dimension.
/// @param td1d The number of trial dofs in one dimension.
/// @param use_sum_factorization Indicator if sum factorization is used.
template <typename input_fop_ts, size_t num_inputs, typename output_fop_t>
MFEM_HOST_DEVICE void assemble_element_mat_naive(
const DeviceTensor<4, real_t>& A,
const DeviceTensor<3, real_t>& fhat,
const DeviceTensor<5, const real_t>& qpdc,
const DeviceTensor<1, const real_t>& itod,
const input_fop_ts& inputs,
const output_fop_t& output,
const std::array<DofToQuadMap, num_inputs>& input_dtqmaps,
const DofToQuadMap& output_dtqmap,
std::array<DeviceTensor<1>, 6>& scratch_shmem,
const int& dimension,
const int& q1d,
const int& td1d,
const bool& use_sum_factorization)
{
if (use_sum_factorization)
{
if (dimension == 2)
{
assemble_element_mat_t2d(A, fhat, qpdc, itod, inputs, output,
input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
}
else if (dimension == 3)
{
assemble_element_mat_t3d(A, fhat, qpdc, itod, inputs, output,
input_dtqmaps, output_dtqmap, scratch_shmem, q1d, td1d);
}
}
else
{
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
MFEM_ABORT("element matrix assemble not implemented for non tensor "
"product basis");
#endif
}
}
} // namespace mfem::future
-122
View File
@@ -1,122 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../util.hpp"
#include "../../integrator_ctx.hpp"
#include <utility>
namespace mfem::future
{
namespace GlobalQFImpl
{
template<
typename qfunc_t,
typename inputs_t,
typename outputs_t,
size_t ninputs = tuple_size<inputs_t>::value,
size_t noutputs = tuple_size<outputs_t>::value>
struct Action
{
Action(
IntegratorContext ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs) :
ctx(ctx),
qfunc(qfunc),
inputs(inputs),
outputs(outputs)
{
create_fop_to_fd(inputs, ctx.infds, input_to_infd);
create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
check_consistency(inputs, input_to_infd, ctx.infds);
check_consistency(outputs, output_to_outfd, ctx.outfds);
create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
const int nqp = ctx.ir.GetNPoints();
gnqp = nqp * ctx.nentities;
// prepare xq and yq BlockVectors
xq_offsets.SetSize(ninputs + 1);
xq_offsets[0] = 0;
constexpr_for<0, ninputs>([&](auto i)
{
const auto input = get<i>(inputs);
xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
});
xq_offsets.PartialSum();
InitBlockVector(xq, xq_offsets);
yq_offsets.SetSize(noutputs + 1);
yq_offsets[0] = 0;
constexpr_for<0, noutputs>([&](auto i)
{
const auto output = get<i>(outputs);
yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
});
yq_offsets.PartialSum();
InitBlockVector(yq, yq_offsets);
}
void operator()(
const std::vector<Vector *> &xe,
std::vector<Vector *> &ye) const
{
if (ctx.attr.Size() == 0) { return; }
// E -> Q
interpolate(input_to_infd, input_bases, xe, xq);
// Q -> Q
static_assert(
detail::supports_tensor_array_qfunc<qfunc_t, inputs_t, outputs_t>::value,
"qfunc signature not supported by default backend Action");
detail::call_qfunc(
qfunc, xq, yq, gnqp, input_qlayouts, output_qlayouts,
std::make_index_sequence<ninputs> {},
std::make_index_sequence<noutputs> {});
// Q -> E
integrate(output_to_outfd, output_bases, yq, ye);
}
IntegratorContext ctx;
qfunc_t qfunc;
inputs_t inputs;
outputs_t outputs;
std::array<size_t, ninputs> input_to_infd;
std::array<size_t, noutputs> output_to_outfd;
std::array<FieldBasis, ninputs> input_bases;
std::array<FieldBasis, noutputs> output_bases;
std::array<std::vector<int>, ninputs> input_qlayouts;
std::array<std::vector<int>, noutputs> output_qlayouts;
int gnqp = 0;
Array<int> xq_offsets, yq_offsets;
mutable BlockVector xq, yq;
};
}
}
@@ -1,180 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "../util.hpp"
#include <utility>
namespace mfem::future::GlobalQFImpl
{
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t,
size_t ninputs = tuple_size<inputs_t>::value,
size_t noutputs = tuple_size<outputs_t>::value>
struct DerivativeAction
{
using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
DerivativeAction(
IntegratorContext ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs):
ctx(ctx), qfunc(qfunc),
qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
inputs(inputs), outputs(outputs)
{
create_fop_to_fd(inputs, ctx.infds, input_to_infd);
create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
check_consistency(inputs, input_to_infd, ctx.infds);
check_consistency(outputs, output_to_outfd, ctx.outfds);
create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
create_fieldbases(
outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
const int nqp = ctx.ir.GetNPoints();
gnqp = nqp * ctx.nentities;
xq_offsets.SetSize(ninputs + 1);
xq_offsets[0] = 0;
constexpr_for<0, ninputs>([&](auto i)
{
const auto input = get<i>(inputs);
xq_offsets[i + 1] = nqp * input.size_on_qp * ctx.nentities;
});
xq_offsets.PartialSum();
InitBlockVector(xq, xq_offsets);
yq_offsets.SetSize(noutputs + 1);
yq_offsets[0] = 0;
constexpr_for<0, noutputs>([&](auto i)
{
const auto output = get<i>(outputs);
yq_offsets[i + 1] = nqp * output.size_on_qp * ctx.nentities;
});
yq_offsets.PartialSum();
InitBlockVector(yq, yq_offsets);
// Shadow blocks use the same offsets as xq so tensor_array views
shadow_xq_offsets.SetSize(xq_offsets.Size());
shadow_xq_offsets = xq_offsets;
InitBlockVector(shadow_xq, shadow_xq_offsets);
dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
const size_t direction_fd_idx = FindIdx(derivative_id, ctx.infds);
MFEM_ASSERT(direction_fd_idx != SIZE_MAX,
"derivative direction field not found in infds");
direction_fd = ctx.infds[direction_fd_idx];
}
void operator()(
const std::vector<Vector *> &xe,
const Vector *de,
std::vector<Vector *> &ye)
{
if (ctx.attr.Size() == 0) { return; }
// E -> Q
interpolate(input_to_infd, input_bases, xe, xq);
constexpr auto input_active =
detail::make_activity_map<derivative_id>(inputs_t{});
MFEM_ASSERT(de != nullptr, "derivative direction vector is null");
restriction(direction_fd, direction_rcache, *de, direction_e,
dof_ordering);
shadow_xq = 0.0;
shadow_xq.SyncToBlocks();
constexpr_for<0, ninputs>([&](auto i)
{
if (!input_active[i]) { return; }
input_bases[i].forward(direction_e, shadow_xq.GetBlock(i));
});
static_assert(detail::supports_tensor_array_qfunc<qfunc_t,
inputs_t,
outputs_t>::value,
"qfunc signature not supported by default backend Action");
// Q -> Q
yq = 0.0;
yq.SyncToBlocks();
if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
{
detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t, inputs_t,
outputs_t>(
qfunc,
qfunc_shadow,
xq,
shadow_xq,
yq,
gnqp,
input_qlayouts,
output_qlayouts,
std::make_index_sequence<ninputs> {},
std::make_index_sequence<noutputs> {});
}
else
{
detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
qfunc,
xq,
shadow_xq,
yq,
gnqp,
input_qlayouts,
output_qlayouts,
std::make_index_sequence<ninputs> {},
std::make_index_sequence<noutputs> {});
}
// Q -> E
integrate(output_to_outfd, output_bases, yq, ye);
}
IntegratorContext ctx;
qfunc_t qfunc;
qfunc_shadow_t qfunc_shadow;
inputs_t inputs;
outputs_t outputs;
std::array<size_t, ninputs> input_to_infd;
std::array<size_t, noutputs> output_to_outfd;
std::array<FieldBasis, ninputs> input_bases;
std::array<FieldBasis, noutputs> output_bases;
std::array<std::vector<int>, ninputs> input_qlayouts;
std::array<std::vector<int>, noutputs> output_qlayouts;
int gnqp = 0;
Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
mutable BlockVector xq, shadow_xq, yq;
FieldDescriptor direction_fd;
ElementDofOrdering dof_ordering = ElementDofOrdering::LEXICOGRAPHIC;
mutable Vector direction_e;
mutable RestrictionCache<Entity::Element> direction_rcache;
};
} // namespace mfem::future::GlobalQFImpl
@@ -1,242 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "../util.hpp"
#include <array>
#include <utility>
#include <vector>
namespace mfem::future::GlobalQFImpl
{
// Q-function-shape-agnostic cached forward apply (J·v)
template<
int derivative_id,
typename inputs_t,
typename outputs_t>
struct DerivativeApply
{
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
DerivativeApply(
IntegratorContext ctx,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache):
ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
qp_cache(qp_cache)
{
create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
check_consistency(this->inputs, input_to_infd, ctx.infds);
check_consistency(this->outputs, output_to_outfd, ctx.outfds);
create_fieldbases(
this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
create_fieldbases(
this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
const int nqp = ctx.ir.GetNPoints();
const int ne = ctx.nentities;
num_qp = nqp;
gnqp = nqp * ne;
// Precompute Q-space BlockVector layouts
dir_q_offsets.SetSize(n_inputs + 1);
dir_q_offsets[0] = 0;
constexpr_for<0, n_inputs>([&](auto i)
{
dir_q_offsets[i + 1] =
dir_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
});
InitBlockVector(dir_q_local, dir_q_offsets);
result_q_offsets.SetSize(n_outputs + 1);
result_q_offsets[0] = 0;
constexpr_for<0, n_outputs>([&](auto i)
{
result_q_offsets[i + 1] =
result_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
});
InitBlockVector(result_q_local, result_q_offsets);
// Cache layout metadata (must match DerivativeSetup)
residual_size_on_qp = 0;
trial_vdim = 0;
total_trial_op_dim = 0;
constexpr auto activity =
detail::make_activity_map<derivative_id>(inputs_t{});
constexpr_for<0, n_inputs>([&](auto i)
{
if (!activity[i]) { return; }
const auto &fop = get<i>(this->inputs);
trial_vdim = fop.vdim;
total_trial_op_dim += fop.size_on_qp / fop.vdim;
});
constexpr_for<0, n_outputs>([&](auto i)
{ residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
residual_size_on_qp *= trial_vdim * total_trial_op_dim;
}
void operator()(
const std::vector<Vector *> & /*xe*/,
const Vector *direction_l,
std::vector<Vector *> &ye) const
{
if (ctx.attr.Size() == 0) { return; }
MFEM_ASSERT(direction_l != nullptr,
"Global DerivativeApply: direction vector is null");
// Re-zero pre-allocated Q temporaries
dir_q_local = 0.0;
dir_q_local.SyncToBlocks();
result_q_local = 0.0;
result_q_local.SyncToBlocks();
// Restrict trial direction from the derivative field
size_t in_fd = SIZE_MAX;
constexpr_for<0, n_inputs>([&](auto i)
{
if (get<i>(inputs).GetFieldId() == derivative_id)
{
in_fd = input_to_infd[i.value];
}
});
MFEM_ASSERT(in_fd != SIZE_MAX,
"DerivativeApply: derivative field not found among inputs");
const auto &fd = ctx.infds[in_fd];
Vector dir_e;
restriction(fd, direction_rcache, *direction_l, dir_e,
ElementDofOrdering::LEXICOGRAPHIC);
// Forward the trial direction into active input Q block
constexpr_for<0, n_inputs>([&](auto s)
{
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
input_bases[s.value].forward(dir_e, dir_q_local.GetBlock(s.value));
});
const real_t *cache_ptr = qp_cache.Read();
const int res_sz = residual_size_on_qp;
const int gnqp_local = gnqp;
const int num_qp_local = num_qp;
const int trial_vdim_local = trial_vdim;
const int total_trial_op_dim_local = total_trial_op_dim;
constexpr_for<0, n_outputs>([&](auto o)
{
const int tv_o = get<o>(outputs).vdim;
const int to_o = get<o>(outputs).size_on_qp / tv_o;
const int out_base = [&]
{
int off = 0;
constexpr_for<0, o.value>([&](auto prev)
{ off += get<prev>(outputs).size_on_qp; });
return off;
}();
real_t *res_o = result_q_local.GetBlock(o.value).ReadWrite();
int m_offset = 0;
constexpr_for<0, n_inputs>([&](auto s)
{
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
const int tv = get<s>(inputs).vdim;
const int to = get<s>(inputs).size_on_qp / tv;
const real_t *dir_s = dir_q_local.GetBlock(s.value).Read();
mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
{
// Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
// read adjacent addresses for a fixed cache_idx.
const int cache_base =
(gq % num_qp_local) +
num_qp_local * res_sz * (gq / num_qp_local);
for (int j = 0; j < tv; ++j)
{
for (int m = 0; m < to; ++m)
{
const real_t v = dir_s[(j * to + m) + (tv * to) * gq];
const int m_global = m + m_offset;
for (int i = 0; i < tv_o; ++i)
{
for (int k = 0; k < to_o; ++k)
{
const int out_comp = out_base + i * to_o + k;
const int cache_idx =
out_comp * trial_vdim_local * total_trial_op_dim_local +
j * total_trial_op_dim_local + m_global;
const real_t c =
cache_ptr[cache_base + num_qp_local * cache_idx];
res_o[(i * to_o + k) + (tv_o * to_o) * gq] += c * v;
}
}
}
}
});
m_offset += to;
});
});
result_q_local.SyncToBlocks();
// Map result Q back to output fields
constexpr_for<0, n_outputs>([&](auto o)
{
const size_t out_fd = output_to_outfd[o.value];
output_bases[o.value].transpose(result_q_local.GetBlock(o.value),
*ye[out_fd]);
});
}
private:
IntegratorContext ctx;
inputs_t inputs;
outputs_t outputs;
const Vector &qp_cache;
std::array<size_t, n_inputs> input_to_infd;
std::array<size_t, n_outputs> output_to_outfd;
std::array<FieldBasis, n_inputs> input_bases;
std::array<FieldBasis, n_outputs> output_bases;
int gnqp = 0;
int num_qp = 0;
Array<int> dir_q_offsets;
Array<int> result_q_offsets;
mutable BlockVector dir_q_local;
mutable BlockVector result_q_local;
mutable RestrictionCache<Entity::Element> direction_rcache;
int residual_size_on_qp = 0;
int trial_vdim = 0;
int total_trial_op_dim = 0;
};
} // namespace mfem::future::GlobalQFImpl
@@ -1,260 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "../util.hpp"
#include <array>
#include <utility>
#include <vector>
namespace mfem::future::GlobalQFImpl
{
// Q-function-shape-agnostic cached transpose apply (Jᵀ·w)
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
struct DerivativeApplyTranspose
{
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
DerivativeApplyTranspose(
IntegratorContext ctx,
qfunc_t /*qfunc*/,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache):
ctx(ctx), inputs(std::move(inputs)), outputs(std::move(outputs)),
qp_cache(qp_cache), dir_out_l(n_outputs), dir_out_e(n_outputs)
{
create_fop_to_fd(this->inputs, ctx.infds, input_to_infd);
create_fop_to_fd(this->outputs, ctx.outfds, output_to_outfd);
check_consistency(this->inputs, input_to_infd, ctx.infds);
check_consistency(this->outputs, output_to_outfd, ctx.outfds);
create_fieldbases(
this->inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
create_fieldbases(
this->outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
const int nqp = ctx.ir.GetNPoints();
const int ne = ctx.nentities;
num_qp = nqp;
gnqp = nqp * ne;
// Precompute Q-space BlockVector layouts
dir_q_offsets.SetSize(n_outputs + 1);
dir_q_offsets[0] = 0;
constexpr_for<0, n_outputs>([&](auto i)
{
dir_q_offsets[i + 1] =
dir_q_offsets[i] + get<i>(this->outputs).size_on_qp * nqp * ne;
});
InitBlockVector(dir_q_local, dir_q_offsets);
result_q_offsets.SetSize(n_inputs + 1);
result_q_offsets[0] = 0;
constexpr_for<0, n_inputs>([&](auto i)
{
result_q_offsets[i + 1] =
result_q_offsets[i] + get<i>(this->inputs).size_on_qp * nqp * ne;
});
InitBlockVector(result_q_local, result_q_offsets);
// Cache layout metadata
residual_size_on_qp = 0;
trial_vdim = 0;
total_trial_op_dim = 0;
constexpr auto activity =
detail::make_activity_map<derivative_id>(inputs_t{});
constexpr_for<0, n_inputs>([&](auto i)
{
if (!activity[i]) { return; }
const auto &fop = get<i>(this->inputs);
trial_vdim = fop.vdim;
total_trial_op_dim += fop.size_on_qp / fop.vdim;
});
constexpr_for<0, n_outputs>([&](auto i)
{ residual_size_on_qp += get<i>(this->outputs).size_on_qp; });
residual_size_on_qp *= trial_vdim * total_trial_op_dim;
}
void operator()(
const std::vector<Vector *> & /*xe*/,
const Vector *direction_l,
std::vector<Vector *> &ye) const
{
if (ctx.attr.Size() == 0) { return; }
MFEM_ASSERT(direction_l != nullptr,
"Global DerivativeApplyTranspose: direction vector is null");
// Re-zero the pre-allocated Q temporaries
dir_q_local = 0.0;
result_q_local = 0.0;
dir_q_local.SyncToBlocks();
result_q_local.SyncToBlocks();
// Bring test cotangent to quadrature points
pull_output_cotangents_to_q(direction_l, dir_q_local);
// Contract qp_cache with test directions at quadrature points
const real_t *cache_ptr = qp_cache.Read();
const int res_sz = residual_size_on_qp;
const int gnqp_local = gnqp;
const int num_qp_local = num_qp;
const int trial_vdim_local = trial_vdim;
const int total_trial_op_dim_local = total_trial_op_dim;
constexpr_for<0, n_outputs>([&](auto o)
{
const int tv_o = get<o>(outputs).vdim;
const int to_o = get<o>(outputs).size_on_qp / tv_o;
const int out_base = [&]
{
int off = 0;
constexpr_for<0, o.value>([&](auto prev)
{ off += get<prev>(outputs).size_on_qp; });
return off;
}();
const int size_o = get<o>(outputs).size_on_qp;
const real_t *dir_o = dir_q_local.GetBlock(o.value).Read();
int m_offset = 0;
constexpr_for<0, n_inputs>([&](auto s)
{
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
const int size_s = get<s>(inputs).size_on_qp;
const int to_s = size_s / trial_vdim_local;
real_t *res_s = result_q_local.GetBlock(s.value).ReadWrite();
mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
{
// Cache is (q, cache_idx, e): adjacent threads (adjacent gq)
// read adjacent addresses for a fixed cache_idx.
const int cache_base =
(gq % num_qp_local) +
num_qp_local * res_sz * (gq / num_qp_local);
for (int i = 0; i < tv_o; ++i)
{
for (int k = 0; k < to_o; ++k)
{
const int out_comp = out_base + i * to_o + k;
const real_t w = dir_o[(i * to_o + k) + size_o * gq];
for (int j = 0; j < trial_vdim_local; ++j)
{
for (int m = 0; m < to_s; ++m)
{
const int m_global = m + m_offset;
const int cache_idx =
out_comp * trial_vdim_local * total_trial_op_dim_local +
j * total_trial_op_dim_local + m_global;
const real_t c =
cache_ptr[cache_base + num_qp_local * cache_idx];
res_s[(j * to_s + m) + size_s * gq] += c * w;
}
}
}
}
});
m_offset += to_s;
});
});
// Map result Q back to the trial (input) fields
constexpr_for<0, n_inputs>([&](auto s)
{
if (get<s>(inputs).GetFieldId() != derivative_id) { return; }
const size_t in_fd = input_to_infd[s.value];
input_bases[s.value].transpose(
result_q_local.GetBlock(s.value), *ye[in_fd]);
});
}
private:
IntegratorContext ctx;
inputs_t inputs;
outputs_t outputs;
const Vector &qp_cache;
std::array<size_t, n_inputs> input_to_infd;
std::array<size_t, n_outputs> output_to_outfd;
std::array<FieldBasis, n_inputs> input_bases;
std::array<FieldBasis, n_outputs> output_bases;
int gnqp = 0;
int num_qp = 0;
// Pre-allocated Q-space temporaries
Array<int> dir_q_offsets;
Array<int> result_q_offsets;
mutable BlockVector dir_q_local;
mutable BlockVector result_q_local;
// Pre-allocated owning storage for output cotangent temporaries
mutable std::array<Vector, n_outputs> dir_out_l_owned;
mutable std::array<Vector, n_outputs> dir_out_e_owned;
mutable std::vector<Vector *> dir_out_l;
mutable std::vector<Vector *> dir_out_e;
mutable RestrictionCache<Entity::Element> out_rcache;
int residual_size_on_qp = 0;
int trial_vdim = 0;
int total_trial_op_dim = 0;
/// Pull output cotangents from L-space into the pre-allocated Q BlockVector
void pull_output_cotangents_to_q(const Vector *direction_l,
BlockVector &dir_q) const
{
int l_offset = 0;
constexpr_for<0, n_outputs>([&](auto i)
{
const size_t outfd = output_to_outfd[i];
const auto &fd = ctx.outfds[outfd];
const int l_size = GetVSize(fd);
dir_out_l_owned[i] =
Vector(*const_cast<Vector *>(direction_l), l_offset, l_size);
dir_out_e_owned[i].SetSize(0);
dir_out_e_owned[i].UseDevice(true);
dir_out_l[i] = &dir_out_l_owned[i];
dir_out_e[i] = &dir_out_e_owned[i];
l_offset += l_size;
});
restriction(ctx.outfds, out_rcache, dir_out_l, dir_out_e);
constexpr_for<0, n_outputs>([&](auto i)
{
output_bases[i.value].forward(*dir_out_e[i], dir_q.GetBlock(i.value));
});
dir_q.SyncToBlocks();
}
};
} // namespace mfem::future::GlobalQFImpl
@@ -1,245 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "../util.hpp"
#include <utility>
namespace mfem::future::GlobalQFImpl
{
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t,
size_t ninputs = tuple_size<inputs_t>::value,
size_t noutputs = tuple_size<outputs_t>::value>
struct DerivativeSetup
{
using qfunc_shadow_t = detail::qfunc_shadow_t<qfunc_t>;
DerivativeSetup(
IntegratorContext ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
Vector &qp_cache) :
ctx(ctx),
qfunc(qfunc),
qfunc_shadow(detail::MakeQFunctionShadowStorage(this->qfunc)),
inputs(inputs),
outputs(outputs),
qp_cache(qp_cache)
{
create_fop_to_fd(inputs, ctx.infds, input_to_infd);
create_fop_to_fd(outputs, ctx.outfds, output_to_outfd);
check_consistency(inputs, input_to_infd, ctx.infds);
check_consistency(outputs, output_to_outfd, ctx.outfds);
create_fieldbases(inputs, input_to_infd, ctx.infds, ctx.ir, input_bases);
create_fieldbases(outputs, output_to_outfd, ctx.outfds, ctx.ir, output_bases);
create_qlayouts(inputs, ctx.in_qlayouts, input_qlayouts);
create_qlayouts(outputs, ctx.out_qlayouts, output_qlayouts);
const int nqp = ctx.ir.GetNPoints();
num_qp = nqp;
nentities = ctx.nentities;
gnqp = nqp * nentities;
xq_offsets.SetSize(ninputs + 1);
xq_offsets[0] = 0;
constexpr_for<0, ninputs>([&](auto i)
{
xq_offsets[i + 1] = nqp * get<i>(inputs).size_on_qp * nentities;
});
xq_offsets.PartialSum();
InitBlockVector(xq, xq_offsets);
shadow_xq_offsets.SetSize(xq_offsets.Size());
shadow_xq_offsets = xq_offsets;
InitBlockVector(shadow_xq, shadow_xq_offsets);
yq_offsets.SetSize(noutputs + 1);
yq_offsets[0] = 0;
constexpr_for<0, noutputs>([&](auto o)
{
yq_offsets[o + 1] = nqp * get<o>(outputs).size_on_qp * nentities;
});
yq_offsets.PartialSum();
InitBlockVector(yq, yq_offsets);
total_out_size_on_qp = 0;
constexpr_for<0, noutputs>([&](auto o)
{
total_out_size_on_qp += get<o>(outputs).size_on_qp;
out_vdim[o] = get<o>(outputs).vdim;
out_op_dim[o] = get<o>(outputs).size_on_qp / get<o>(outputs).vdim;
});
activity_map = detail::make_activity_map<derivative_id>(inputs_t {});
trial_vdim = 0;
total_trial_op_dim = 0;
constexpr_for<0, ninputs>([&](auto i)
{
if (!activity_map[i]) { return; }
const auto inp = get<i>(inputs);
trial_vdim = inp.vdim;
total_trial_op_dim += inp.size_on_qp / inp.vdim;
});
constexpr_for<0, ninputs>([&](auto i)
{
input_size_on_qp_arr[i] = get<i>(inputs).size_on_qp;
});
residual_size_on_qp = total_out_size_on_qp * trial_vdim * total_trial_op_dim;
qp_cache.SetSize(residual_size_on_qp * num_qp * nentities);
qp_cache.UseDevice(true);
}
void operator()(const std::vector<Vector *> &xe)
{
if (ctx.attr.Size() == 0) { return; }
interpolate(input_to_infd, input_bases, xe, xq);
const int gnqp_local = gnqp;
const int num_qp_local = num_qp;
const int trial_vdim_local = trial_vdim;
const int total_trial_op_dim_local = total_trial_op_dim;
const int residual_size_local = residual_size_on_qp;
for (int j = 0; j < trial_vdim; j++)
{
int m_offset = 0;
constexpr_for<0, ninputs>([&](auto s)
{
if (!activity_map[s]) { return; }
const int input_vdim_s = get<s>(inputs).vdim;
const int input_size_s = input_size_on_qp_arr[s];
const int trial_op_dim_s = input_size_s / input_vdim_s;
for (int m = 0; m < trial_op_dim_s; m++)
{
shadow_xq = 0.0;
shadow_xq.SyncToBlocks();
// Set component (j + input_vdim_s * m) to 1 at all QPs
const int c_shadow = j + input_vdim_s * m;
real_t *shadow_ptr = shadow_xq.GetBlock(s.value).ReadWrite();
mfem::forall(gnqp_local, [=] MFEM_HOST_DEVICE(int gq)
{
shadow_ptr[c_shadow + input_size_s * gq] = 1.0;
});
yq = 0.0;
yq.SyncToBlocks();
if constexpr (detail::qfunc_uses_scratch_v<qfunc_t>)
{
detail::fwddiff<derivative_id, qfunc_t, qfunc_shadow_t,
inputs_t, outputs_t>(
qfunc, qfunc_shadow, xq, shadow_xq, yq, gnqp,
input_qlayouts, output_qlayouts,
std::make_index_sequence<ninputs> {},
std::make_index_sequence<noutputs> {});
}
else
{
detail::fwddiff<derivative_id, qfunc_t, inputs_t, outputs_t>(
qfunc, xq, shadow_xq, yq, gnqp,
input_qlayouts, output_qlayouts,
std::make_index_sequence<ninputs> {},
std::make_index_sequence<noutputs> {});
}
yq.SyncToBlocks();
real_t *cache_d = qp_cache.ReadWrite();
// Write yq into the cache column
const int m_global = m + m_offset;
const int j_cur = j;
int out_offset = 0;
constexpr_for<0, noutputs>([&](auto o)
{
const int test_vdim_o = out_vdim[o];
const int test_op_dim_o = out_op_dim[o];
const int yq_out_size = test_vdim_o * test_op_dim_o;
const int out_offset_o = out_offset;
const real_t *yq_d = yq.GetBlock(o.value).Read();
// The cache is (q, cache_idx, e) with the quadrature index
// fastest, so gq is the fastest-varying thread index to keep
// the stores coalesced.
mfem::forall(gnqp_local * yq_out_size, [=] MFEM_HOST_DEVICE(int idx)
{
const int gq = idx % gnqp_local;
const int c_out = idx / gnqp_local;
const int q = gq % num_qp_local;
const int entity = gq / num_qp_local;
const int out_comp = out_offset_o + c_out;
const int cache_idx =
out_comp * trial_vdim_local * total_trial_op_dim_local +
j_cur * total_trial_op_dim_local +
m_global;
cache_d[q + num_qp_local *
(cache_idx + residual_size_local * entity)] =
yq_d[c_out + yq_out_size * gq];
});
out_offset += yq_out_size;
});
}
m_offset += trial_op_dim_s;
});
}
}
IntegratorContext ctx;
qfunc_t qfunc;
qfunc_shadow_t qfunc_shadow;
inputs_t inputs;
outputs_t outputs;
Vector &qp_cache;
std::array<size_t, ninputs> input_to_infd;
std::array<size_t, noutputs> output_to_outfd;
std::array<FieldBasis, ninputs> input_bases;
std::array<FieldBasis, noutputs> output_bases;
std::array<std::vector<int>, ninputs> input_qlayouts;
std::array<std::vector<int>, noutputs> output_qlayouts;
int gnqp = 0;
int num_qp = 0;
int nentities = 0;
Array<int> xq_offsets, shadow_xq_offsets, yq_offsets;
mutable BlockVector xq, shadow_xq, yq;
int total_out_size_on_qp = 0;
int trial_vdim = 0;
int total_trial_op_dim = 0;
int residual_size_on_qp = 0;
std::array<int, noutputs> out_vdim {};
std::array<int, noutputs> out_op_dim {};
std::array<int, ninputs> input_size_on_qp_arr {};
std::array<bool, ninputs> activity_map {};
};
} // namespace mfem::future::GlobalQFImpl
-219
View File
@@ -1,219 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "action.hpp"
#include "derivative_action.hpp"
#include "derivative_setup.hpp"
#include "derivative_apply_transpose.hpp"
#include "../local_qf/derivative_apply.hpp"
#include "../local_qf/derivative_assemble.hpp"
#include "../local_qf/derivative_assemble_diagonal.hpp"
#include "../scratch_bank.hpp"
namespace mfem::future
{
namespace detail
{
template <typename T>
struct LocalQFShapeArg
{
using type = std::remove_const_t<T>&;
};
template <typename scalar_t, int ndims, int... tensor_sizes>
struct LocalQFShapeArg<tensor_ndarray<scalar_t, ndims, tensor_sizes...>>
{
using scalar_type = std::remove_const_t<scalar_t>;
using type = std::conditional_t<
sizeof...(tensor_sizes) == 0,
scalar_type,
tensor<scalar_type, tensor_sizes...>>&;
};
template <typename scalar_t, int... tensor_sizes>
struct LocalQFShapeArg<tensor<scalar_t, tensor_sizes...>>
{
using scalar_type = std::remove_const_t<scalar_t>;
using type = std::conditional_t<
sizeof...(tensor_sizes) == 0,
scalar_type,
tensor<scalar_type, tensor_sizes...>>&;
};
template <typename qf_param_ts>
struct LocalQFShapeFunction;
template <typename... qf_param_ts>
struct LocalQFShapeFunction<tuple<qf_param_ts...>>
{
void operator()(
typename LocalQFShapeArg<qf_param_decay_t<qf_param_ts>>::type...) const;
};
template <typename qfunc_t>
using LocalQFShapeFunctionFor = LocalQFShapeFunction<
typename get_function_signature<qfunc_t>::type::parameter_ts>;
} // namespace detail
struct GlobalQFBackend
{
/**
* @brief Make an action for a global Q-function.
*
* @param ctx The integrator context.
* @param args The arguments to the action.
* @return The action.
*/
template<
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeAction(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs)
{
return GlobalQFImpl::Action(ctx, qfunc, inputs, outputs);
}
/**
* @brief Make a derivative action for a global Q-function.
*
* @tparam derivative_id The id of the derivative.
* @param ctx The integrator context.
* @param args The arguments to the derivative action.
* @return The derivative action.
*/
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeDerivativeAction(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs)
{
return GlobalQFImpl::DerivativeAction<
derivative_id, qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs);
}
/**
* @brief Make a derivative setup for a global Q-function.
*
* @tparam derivative_id The id of the derivative.
* @param ctx The integrator context.
* @param args The arguments to the derivative setup.
* @return The derivative setup.
*/
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeDerivativeSetup(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
Vector &qp_cache)
{
return GlobalQFImpl::DerivativeSetup<
derivative_id, qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeDerivativeApply(
const IntegratorContext &ctx,
const qfunc_t & /*qfunc*/,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeApply<
derivative_id,
detail::LocalQFShapeFunctionFor<qfunc_t>,
inputs_t,
outputs_t>(ctx,
detail::LocalQFShapeFunctionFor<qfunc_t> {},
inputs,
outputs,
qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeDerivativeApplyTranspose(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return GlobalQFImpl::DerivativeApplyTranspose<
derivative_id, qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeDerivativeAssemble(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeAssemble<
derivative_id, qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
auto static MakeDerivativeAssembleDiagonal(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeAssembleDiagonal<
derivative_id, qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs, qp_cache);
}
};
} // namespace mfem::future
-514
View File
@@ -1,514 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "fem/dfem/fieldoperator.hpp"
#include "kernels.hpp"
#include "util.hpp"
#include "../../integrator_ctx.hpp"
#include "../util.hpp"
namespace mfem::future::LocalQFImpl
{
template<typename qfunc_t, typename inputs_t, typename outputs_t>
class Action
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
using qf_signature = typename get_function_signature<qfunc_t>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
using args_tuple_t = decay_tuple<qf_param_ts>;
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
"LocalQF: q-function arity must match inputs + outputs");
const qfunc_t qfunc;
const inputs_t inputs;
const outputs_t outputs;
const IntegratorContext ctx;
const std::vector<const DofToQuad *> dtqs;
// inputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_inputs> input_dtq;
const std::array<size_t, n_inputs> input_idx; // input to field
const std::array<const real_t *, n_inputs> input_B, input_G;
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
// outputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_outputs> output_dtq;
const std::array<size_t, n_outputs> output_idx; // output to field
const std::array<const real_t *, n_outputs> output_B, output_G;
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
// other constants
const int dim, ne, nq, q1d;
public:
////////////////////////////////////////////////////////
Action() = delete;
Action(IntegratorContext ctx,
qfunc_t qfunc,
inputs_t inputs,
outputs_t outputs):
qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
dtqs(make_dtqs(ctx)),
// inputs: dtq, idx, B, G, d1d, q1d, vdim
input_dtq(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx, inputs),
ctx.unionfds,
ctx.ir)),
input_idx(create_input_vector_map(ctx, inputs)),
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
input_vdim(get_vdim(inputs)),
// outputs: dtq, idx, B, G, d1d, q1d, vdim
output_dtq(create_dtq_maps<Entity::Element>(
outputs,
dtqs,
create_union_field_map_for_dtq(ctx, outputs),
ctx.unionfds,
ctx.ir)),
output_idx(create_output_vector_map(ctx, outputs)),
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
output_vdim(get_vdim(outputs)),
// other constants
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
q1d(tensor_1d_size(nq, dim))
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
}
template<typename Backend>
void run_kernels(const std::vector<Vector *> &xe,
std::vector<Vector *> &ye) const
{
Backend::Run(dim,
q1d,
// arguments
ctx,
qfunc,
// inputs
input_idx,
input_B,
input_G,
input_vdim,
input_d1d,
input_q1d,
// outputs
output_idx,
output_B,
output_G,
output_vdim,
output_d1d,
output_q1d,
// input and output vectors
xe,
ye,
// fallback arguments
dim,
q1d);
}
void operator()(const std::vector<Vector *> &xe,
std::vector<Vector *> &ye) const
{
if (q1d <= LocalQFLOBackendMQ1())
{
run_kernels<ActionLO>(xe, ye);
}
else if (q1d <= LocalQFHOBackendMQ1())
{
run_kernels<ActionHO>(xe, ye);
}
else
{
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
}
}
////////////////////////////////////////////////////////
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
static void
action_callback(const IntegratorContext &ctx,
const qfunc_t &qfunc,
// inputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_inputs> &in_idx,
const std::array<const real_t *, n_inputs> in_B,
const std::array<const real_t *, n_inputs> in_G,
const std::array<int, n_inputs> &in_vdim,
const std::array<int, n_inputs> &in_d1d,
const std::array<int, n_inputs> &in_q1d,
// outputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_outputs> &out_idx,
const std::array<const real_t *, n_outputs> out_B,
const std::array<const real_t *, n_outputs> out_G,
const std::array<int, n_outputs> &out_vdim,
const std::array<int, n_outputs> &out_d1d,
const std::array<int, n_outputs> &out_q1d,
const std::vector<Vector *> &xe,
std::vector<Vector *> &ye,
// fallback arguments
const int dim,
const int q1d)
{
if (ctx.attr.Size() == 0) { return; }
MFEM_CONTRACT_VAR(dim);
MFEM_ASSERT(dim == ctx.mesh.Dimension(), "Dimension mismatch");
static constexpr auto B2D = backend_t::DIM == 2;
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::Q1D;
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
const int ne = ctx.nentities;
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
// --------------------------------------------------
// INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
// --------------------------------------------------
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = in_idx[i];
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP>)
{
MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
}
else if constexpr (is_weight_fop_v<FOP>)
{
MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
"tensor-product IR expected");
in_XE[i] = Reshape(
ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
}
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------------------
// OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
// --------------------------------------------------
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = out_idx[i];
const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
using FOP = tuple_element_t<i, outputs_t>;
if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
{
MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP> || is_functionalvalue_fop_v<FOP>)
{
MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
}
else
{
static_assert(false, "Unsupported FieldOperator");
}
});
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
dfem::forall<MTPB>(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
// -----------------------------------------------
// Inputs and outputs argument registers
// -----------------------------------------------
args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
// -----------------------------------------------
// Shared memory
// -----------------------------------------------
MFEM_SHARED typename backend_t::Shared smem;
// -----------------------------------------------
// Load inputs
// -----------------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const auto &XE = in_XE[i];
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
;
const real_t *B = in_B[i], *G = in_G[i];
auto &rarg = get<i>(rargs);
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop<FOP>::value)
{
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
using XE_t = decltype(XE);
using rarg_t = decltype(rarg);
using qf_param_t =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
backend_t::template LoadGradient<RNK, rarg_t, XE_t, qf_param_t>(
smem, e, d, q, q1d, B, G, XE, rarg);
}
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
{
// qp values are read directly from in_XE / IR
}
else
{
static_assert(false, "Unsupported");
}
});
// -----------------------------------------------
// Evaluate the quadrature function
// Warning: no 'DIRECT' on the 'Z' direction,
// as one backend may need to iterate over it.
// -----------------------------------------------
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
args_tuple_t qargs;
// --------------------------------------
// Pulling arguments from registers to qargs tuple
// --------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
auto &qarg = get<i>(qargs);
const auto &XE = in_XE[i];
using FOP = tuple_element_t<i, inputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
using DT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
if constexpr (qf_param_uses_dual_v<DT>)
{
qarg = backend_t::template identity_qp_pull_dual<DT>(
false, XE, XE, qx, qy, qz, e);
}
else
{
qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
}
}
else if constexpr (is_weight_fop_v<FOP>)
{
qarg = XE(qx, qy, qz, 0, 0);
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
qarg = backend_t::template qp_pull<ARG>(
get<i>(rargs), qx, qy, qz);
}
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------
// Call the quadrature function
// --------------------------------------
call_qfunc_no_move(qfunc, qargs);
// --------------------------------------
// Pushing arguments from qargs tuple to registers
// --------------------------------------
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value, o = n_inputs + i;
const auto &qarg = get<o>(qargs);
const auto &YE = out_YE[i];
using FOP = tuple_element_t<i, outputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP> ||
is_functionalvalue_fop_v<FOP>)
{
using DT =
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
if constexpr (qf_param_uses_dual_v<DT>)
{
backend_t::identity_qp_write_value(
YE, qx, qy, qz, e, qarg);
}
else
{
as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
}
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
auto &rarg = get<o>(rargs);
backend_t::template qp_push<ARG>(
rarg, qx, qy, qz, qarg);
}
else
{
static_assert(false, "Unsupported");
}
});
}
}
}
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Integrate outputs
// -----------------------------------------------
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value, o = n_inputs + i;
const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
const auto B = out_B[i], G = out_G[i];
const auto &YE = out_YE[i];
auto &rarg = get<o>(rargs);
using FOP = tuple_element_t<i, outputs_t>;
if constexpr (is_value_fop_v<FOP>)
{
backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
using YE_t = decltype(YE);
using rarg_t = decltype(rarg);
// Both the rank and the extents have to come from the output
// parameter slot o, not from the output index i.
using qf_param_t =
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
smem, e, d, q, Q1D, B, G, YE, rarg);
}
else if constexpr (is_identity_fop_v<FOP> ||
is_functionalvalue_fop_v<FOP>)
{
// nothing to do
}
else
{
static_assert(false, "Unsupported");
}
});
},
ne,
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
q1d, in_d1d, out_d1d)),
0,
nullptr);
}
using KernelType = decltype(&Action::action_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionLO, KernelType, (int, int) );
MFEM_REGISTER_KERNELS_HEADER_ONLY(ActionHO, KernelType, (int, int) );
};
// Low Order kernels
template<typename qfunc_t, typename inputs_t, typename outputs_t>
template<int DIM, int Q1D>
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Kernel()
{
static_assert(Q1D <= LocalQFLOBackend<DIM>::MQ1);
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
return action_t::template action_callback<LocalQFLOBackend<DIM, Q1D>>;
}
// Low Order fallback
template<typename qfunc_t, typename inputs_t, typename outputs_t>
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
Action<qfunc_t, inputs_t, outputs_t>::ActionLO::Fallback(int dim, int q1d)
{
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
using ActionLO = typename action_t::ActionLO;
if (dim == 2)
{
return DispatchLOKernelByQ1D<ActionLO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchLOKernelByQ1D<ActionLO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
// High Order kernels
template<typename qfunc_t, typename inputs_t, typename outputs_t>
template<int DIM, int Q1D>
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Kernel()
{
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
return action_t::template action_callback<LocalQFHOBackend<DIM>, Q1D>;
}
// High Order fallback
template<typename qfunc_t, typename inputs_t, typename outputs_t>
inline typename Action<qfunc_t, inputs_t, outputs_t>::KernelType
Action<qfunc_t, inputs_t, outputs_t>::ActionHO::Fallback(int dim, int q1d)
{
using action_t = Action<qfunc_t, inputs_t, outputs_t>;
using ActionHO = typename action_t::ActionHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<ActionHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<ActionHO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,763 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "kernels.hpp"
#include "util.hpp"
namespace mfem::future::LocalQFImpl
{
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
class DerivativeAction
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
using qf_signature = typename get_function_signature<qfunc_t>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
using args_tuple_t = decay_tuple<qf_param_ts>;
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
"LocalQF: q-function arity must match inputs + outputs");
qfunc_t qfunc;
const inputs_t inputs;
const outputs_t outputs;
const IntegratorContext ctx;
const std::vector<const DofToQuad *> dtqs;
// inputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_inputs> input_dtq;
const std::array<size_t, n_inputs> input_idx;
const std::array<const real_t *, n_inputs> input_B, input_G;
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
// outputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_outputs> output_dtq;
const std::array<size_t, n_outputs> output_idx;
const std::array<const real_t *, n_outputs> output_B, output_G;
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
// other constants
const int dim, ne, nq, q1d;
std::array<bool, n_inputs> input_is_dependent;
FieldDescriptor direction_fd;
mutable Vector direction_e;
mutable RestrictionCache<Entity::Element> direction_rcache;
public:
//////////////////////////////////////////////////////////////////
DerivativeAction() = delete;
DerivativeAction(IntegratorContext ctx,
qfunc_t qfunc,
inputs_t inputs,
outputs_t outputs):
qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
dtqs(make_dtqs(ctx)),
// inputs: dtq, idx, B, G, d1d, q1d, vdim
input_dtq(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx, inputs),
ctx.unionfds,
ctx.ir)),
input_idx(create_input_vector_map(ctx, inputs)),
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
input_vdim(get_vdim(inputs)),
// outputs: dtq, idx, B, G, d1d, q1d, vdim
output_dtq(create_dtq_maps<Entity::Element>(
outputs,
dtqs,
create_union_field_map_for_dtq(ctx, outputs),
ctx.unionfds,
ctx.ir)),
output_idx(create_output_vector_map(ctx, outputs)),
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
output_vdim(get_vdim(outputs)),
// other constants
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
q1d(tensor_1d_size(nq, dim))
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
// Determine which inputs are dependent on the derivative direction
auto dependency_map = make_dependency_map(inputs);
auto it = dependency_map.find(derivative_id);
MFEM_ASSERT(it != dependency_map.end(),
"Derivative ID not found in dependency map");
input_is_dependent = it->second;
// Find direction field index
int direction_field_idx = -1;
for (size_t uf = 0; uf < nfields; uf++)
{
if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
{
direction_field_idx = static_cast<int>(uf);
break;
}
}
MFEM_ASSERT(
direction_field_idx != -1,
"LocalQFBackend: derivative direction field not found in unionfds");
direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
}
//////////////////////////////////////////////////////////////////
template<typename Backend>
void run_kernels(const std::vector<Vector *> &xe,
std::vector<Vector *> &ye)
{
Backend::Run(dim,
q1d,
// arguments
ctx,
qfunc,
// inputs
input_idx,
input_B,
input_G,
input_vdim,
input_d1d,
input_q1d,
// outputs
output_idx,
output_B,
output_G,
output_vdim,
output_d1d,
output_q1d,
// input and output vectors
xe,
ye,
input_is_dependent,
direction_e,
// fallback arguments
dim,
q1d);
}
//////////////////////////////////////////////////////////////////
void operator()(const std::vector<Vector *> &xe,
const Vector *direction_l,
std::vector<Vector *> &ye)
{
if (ctx.attr.Size() == 0) { return; }
MFEM_ASSERT(direction_l != nullptr,
"LocalQF DerivativeAction: direction vector is null");
restriction(direction_fd,
direction_rcache,
*direction_l,
direction_e,
ElementDofOrdering::LEXICOGRAPHIC);
if (q1d <= LocalQFLOBackendMQ1())
{
run_kernels<DerivativeActionLO>(xe, ye);
}
else if (q1d <= LocalQFHOBackendMQ1())
{
run_kernels<DerivativeActionHO>(xe, ye);
}
else
{
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
}
}
//////////////////////////////////////////////////////////////////
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
static void
derivative_action_callback(const IntegratorContext &ctx,
qfunc_t &qfunc,
// inputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_inputs> &in_idx,
const std::array<const real_t *, n_inputs> in_B,
const std::array<const real_t *, n_inputs> in_G,
const std::array<int, n_inputs> &in_vdim,
const std::array<int, n_inputs> &in_d1d,
const std::array<int, n_inputs> &in_q1d,
// outputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_outputs> &out_idx,
const std::array<const real_t *, n_outputs> out_B,
const std::array<const real_t *, n_outputs> out_G,
const std::array<int, n_outputs> &out_vdim,
const std::array<int, n_outputs> &out_d1d,
const std::array<int, n_outputs> &out_q1d,
const std::vector<Vector *> &xe,
std::vector<Vector *> &ye,
const std::array<bool, n_inputs> &input_dep,
const Vector &direction_e,
// fallback arguments
const int dim,
const int q1d)
{
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
if (ctx.attr.Size() == 0) { return; }
static constexpr auto B2D = backend_t::DIM == 2;
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
const int ne = ctx.nentities;
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
// --------------------------------------------------
// INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
// --------------------------------------------------
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = in_idx[i];
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP>)
{
MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
}
else if constexpr (is_weight_fop_v<FOP>)
{
MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
"tensor-product IR expected");
in_XE[i] = Reshape(
ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
}
else
{
static_assert(false, "Unsupported");
}
});
const auto d_direction = direction_e.Read();
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = in_idx[i];
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
if (input_dep[i])
{
MFEM_ASSERT(direction_e.Size() == xe[k]->Size(),
"direction E-vector size mismatch for input " << i);
in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
}
else
{
in_XE_dir[i] = in_XE[i];
}
}
else if constexpr (is_identity_fop_v<FOP>)
{
if (input_dep[i])
{
MFEM_VERIFY(direction_e.Size() == xe[k]->Size(),
"direction E-vector size mismatch (identity input) "
<< i);
in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
}
else
{
in_XE_dir[i] = in_XE[i];
}
}
else if constexpr (is_weight_fop_v<FOP>) { in_XE_dir[i] = in_XE[i]; }
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------------------
// OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
// --------------------------------------------------
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = out_idx[i];
const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
using FOP = tuple_element_t<i, outputs_t>;
if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
{
MFEM_ASSERT(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP>)
{
MFEM_ASSERT(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
}
else
{
static_assert(false, "Unsupported FieldOperator");
}
});
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
dfem::forall<MTPB>(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
// -----------------------------------------------
// Inputs and outputs argument registers
// -----------------------------------------------
args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1>
sargs; // shadow
// -----------------------------------------------
// Shared memory
// -----------------------------------------------
MFEM_SHARED typename backend_t::Shared smem;
// -----------------------------------------------
// Load primal inputs (rargs)
// -----------------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const auto &XE = in_XE[i];
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
const real_t *B = in_B[i], *G = in_G[i];
auto &rarg = get<i>(rargs);
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop<FOP>::value)
{
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, rarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
using FieldParamT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
backend_t::template LoadGradient<RNK,
decltype(rarg),
decltype(XE),
FieldParamT>(
smem, e, d, q, Q1D, B, G, XE, rarg);
}
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
{
// qp values are read directly from in_XE / IR
}
else
{
static_assert(false, "Unsupported");
}
});
// -----------------------------------------------
// Load tangent directions (sargs)
// -----------------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
if (!input_dep[i]) { return; }
const auto &XE = in_XE_dir[i];
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
const real_t *B = in_B[i], *G = in_G[i];
auto &sarg = get<i>(sargs); // shadow argument register
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop<FOP>::value)
{
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
using FieldParamT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
backend_t::template LoadGradient<RNK,
decltype(sarg),
decltype(XE),
FieldParamT>(
smem, e, d, q, Q1D, B, G, XE, sarg);
}
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
{
}
else
{
static_assert(false, "Unsupported");
}
});
// -----------------------------------------------
// Evaluate the quadrature function
// Warning: no 'DIRECT' on the 'Z' direction,
// as one backend may need to iterate over it.
// -----------------------------------------------
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
#ifdef MFEM_USE_ENZYME
args_tuple_t primal_args {}, shadow_args {};
// --------------------------------------
// Pulling arguments from registers to primal and shadow
// tuples
// --------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
auto &parg = get<i>(primal_args);
auto &targ = get<i>(shadow_args);
const auto &XE = in_XE[i];
const auto &XEd = in_XE_dir[i];
using FOP = tuple_element_t<i, inputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
if (input_dep[i])
{
targ = as_tensor<ARG>(&XEd(0, qx, qy, qz, e));
}
else
{
targ = ARG{};
}
}
else if constexpr (is_weight_fop_v<FOP>)
{
parg = XE(qx, qy, qz, 0, 0);
targ = real_t(0.0);
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
parg = backend_t::template qp_pull<ARG>(
get<i>(rargs), qx, qy, qz);
if (input_dep[i])
{
targ = backend_t::template qp_pull<ARG>(
get<i>(sargs), qx, qy, qz);
}
else
{
targ = ARG{};
}
}
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------
// Call the quadrature function
// --------------------------------------
call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
// --------------------------------------
// Pushing arguments from enzyme_shadow tuple to registers
// --------------------------------------
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value, o = n_inputs + i;
const auto &qout = get<o>(shadow_args);
auto &YE = out_YE[i];
using FOP = tuple_element_t<i, outputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qout;
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
auto &rarg = get<o>(rargs);
backend_t::template qp_push_tangent<ARG>(
rarg, qx, qy, qz, qout);
}
else
{
static_assert(false, "Unsupported");
}
});
#else // MFEM_USE_ENZYME
args_tuple_t qargs;
// --------------------------------------
// Pulling arguments from registers to qargs tuple
// --------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
auto &qarg = get<i>(qargs);
const auto &XE = in_XE[i];
const auto &XEd = in_XE_dir[i];
using FOP = tuple_element_t<i, inputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
using DT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
if constexpr (qf_param_uses_dual_v<DT>)
{
qarg = backend_t::template identity_qp_pull_dual<DT>(
input_dep[i], XE, XEd, qx, qy, qz, e);
}
else
{
qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
}
}
else if constexpr (is_weight_fop_v<FOP>)
{
qarg = XE(qx, qy, qz, 0, 0);
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
qarg = backend_t::template qp_pull_directional<ARG>(
get<i>(rargs),
get<i>(sargs),
qx,
qy,
qz,
input_dep[i]);
}
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------
// Call the quadrature function
// --------------------------------------
call_qfunc_no_move(qfunc, qargs);
// --------------------------------------
// Pushing arguments from qargs tuple to registers
// --------------------------------------
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value, o = n_inputs + i;
const auto &qarg = get<o>(qargs);
const auto &YE = out_YE[i];
using FOP = tuple_element_t<i, outputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, o>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
using DT =
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
if constexpr (qf_param_uses_dual_v<DT>)
{
backend_t::identity_qp_write_tangent(
YE, qx, qy, qz, e, qarg);
}
else
{
as_tensor<ARG>(&YE(0, qx, qy, qz, e)) = qarg;
}
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
auto &rarg = get<o>(rargs);
backend_t::template qp_push_tangent<ARG>(
rarg, qx, qy, qz, qarg);
}
else
{
static_assert(false, "Unsupported");
}
});
#endif // MFEM_USE_ENZYME
}
}
}
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Integrate outputs
// -----------------------------------------------
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value, o = n_inputs + i;
const int d = out_d1d[i], q = out_q1d[i];
const auto B = out_B[i], G = out_G[i];
auto &YE = out_YE[i];
auto &rarg = get<o>(rargs);
using FOP = tuple_element_t<i, outputs_t>;
if constexpr (is_value_fop_v<FOP>)
{
backend_t::WriteValue(smem, e, d, q, q1d, B, YE, rarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
using YE_t = decltype(YE);
using rarg_t = decltype(rarg);
using qf_param_t =
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
smem, e, d, q, q1d, B, G, YE, rarg);
}
else if constexpr (is_identity_fop_v<FOP>)
{
// nothing to do
}
else
{
static_assert(false, "Unsupported");
}
});
},
ne,
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
q1d, in_d1d, out_d1d)),
0,
nullptr);
}
using DerivativeKernelType =
decltype(&DerivativeAction::derivative_action_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionLO,
DerivativeKernelType,
(int, int) );
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeActionHO,
DerivativeKernelType,
(int, int) );
};
// Low Order kernels
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeKernelType
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeActionLO::Kernel()
{
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
using derivative_action_t =
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
return derivative_action_t::template derivative_action_callback<
LocalQFLOBackend<DIM, Q1D>>;
}
// Low Order fallback
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeKernelType
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeActionLO::Fallback(int dim, int q1d)
{
using derivative_action_t =
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeActionLO =
typename derivative_action_t::DerivativeActionLO;
if (dim == 2)
{
return DispatchLOKernelByQ1D<DerivativeActionLO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchLOKernelByQ1D<DerivativeActionLO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
// High Order kernels
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeKernelType
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeActionHO::Kernel()
{
using derivative_action_t =
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
return derivative_action_t::
template derivative_action_callback<LocalQFHOBackend<DIM>, Q1D>;
}
// High Order fallback
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeKernelType
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeActionHO::Fallback(int dim, int q1d)
{
using derivative_action_t =
DerivativeAction<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeActionHO = typename derivative_action_t::DerivativeActionHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<DerivativeActionHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<DerivativeActionHO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,738 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "kernels.hpp"
#include "util.hpp"
#include <array>
namespace mfem::future::LocalQFImpl
{
// Cached Jacobian apply: J·v from qp_cache filled by DerivativeSetup
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
class DerivativeApply
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
using qf_signature = typename get_function_signature<qfunc_t>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
using args_tuple_t = decay_tuple<qf_param_ts>;
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
"LocalQF: q-function arity must match inputs + outputs");
const inputs_t inputs;
const outputs_t outputs;
const IntegratorContext ctx;
const Vector &qp_cache;
const std::vector<const DofToQuad *> dtqs;
// inputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_inputs> input_dtq;
const std::array<size_t, n_inputs> input_idx;
const std::array<const real_t *, n_inputs> input_B, input_G;
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
// outputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_outputs> output_dtq;
const std::array<size_t, n_outputs> output_idx;
const std::array<const real_t *, n_outputs> output_B, output_G;
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
// Jacobian cache metadata
const std::array<bool, n_inputs> input_is_dependent;
const std::array<int, n_inputs> input_size_on_qp;
const std::array<int, n_outputs> out_op_dim;
const std::array<int, n_outputs> out_offsets;
const int output_size_on_qp;
const int trial_vdim;
const int total_trial_op_dim;
const int residual_size_on_qp;
// other constants
const int dim, ne, nq, q1d;
FieldDescriptor direction_fd;
mutable Vector direction_e;
mutable RestrictionCache<Entity::Element> direction_rcache;
template <std::size_t slot>
static constexpr int ParamRank()
{
using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
return qf_param_shape<param_t>::rank;
}
template <std::size_t slot, int dim_idx>
static constexpr int ParamExtent()
{
using param_t = typename qf_param_slot<qfunc_t, slot>::qf_decay_param_t;
return qf_param_shape<param_t>::extents[dim_idx];
}
template <typename fop_t, std::size_t slot>
static constexpr int StaticVDim()
{
constexpr int rank = ParamRank<slot>();
if constexpr (is_gradient_fop_v<fop_t>)
{
if constexpr (rank <= 1) { return 1; }
else { return ParamExtent<slot, 0>(); }
}
else
{
if constexpr (rank == 0) { return 1; }
else { return ParamExtent<slot, 0>(); }
}
}
template <typename fop_t, std::size_t slot>
static constexpr int StaticOpDim()
{
constexpr int rank = ParamRank<slot>();
if constexpr (is_gradient_fop_v<fop_t>)
{
if constexpr (rank == 0) { return 1; }
else if constexpr (rank == 1) { return ParamExtent<slot, 0>(); }
else { return ParamExtent<slot, 1>(); }
}
else
{
if constexpr (rank <= 1) { return 1; }
else { return ParamExtent<slot, 1>(); }
}
}
template <std::size_t input_slot>
static constexpr bool StaticInputDep()
{
using fop_t = tuple_element_t<input_slot, inputs_t>;
return fop_t::GetFieldId() == derivative_id;
}
template <std::size_t input_slot>
static constexpr int StaticInputVDim()
{
using fop_t = tuple_element_t<input_slot, inputs_t>;
return StaticVDim<fop_t, input_slot>();
}
template <std::size_t input_slot>
static constexpr int StaticInputOpDim()
{
using fop_t = tuple_element_t<input_slot, inputs_t>;
return StaticOpDim<fop_t, input_slot>();
}
template <std::size_t output_slot>
static constexpr int StaticOutputVDim()
{
using fop_t = tuple_element_t<output_slot, outputs_t>;
return StaticVDim<fop_t, n_inputs + output_slot>();
}
template <std::size_t output_slot>
static constexpr int StaticOutputOpDim()
{
using fop_t = tuple_element_t<output_slot, outputs_t>;
return StaticOpDim<fop_t, n_inputs + output_slot>();
}
template <std::size_t output_slot>
static constexpr int StaticOutputOffset()
{
int offset = 0;
for_constexpr<output_slot>([&](auto oc)
{
constexpr size_t o = oc.value;
offset += StaticOutputVDim<o>() * StaticOutputOpDim<o>();
});
return offset;
}
template <std::size_t input_slot>
static constexpr int StaticInputOpOffset()
{
int offset = 0;
for_constexpr<input_slot>([&](auto sc)
{
constexpr size_t s = sc.value;
if constexpr (StaticInputDep<s>()) { offset += StaticInputOpDim<s>(); }
});
return offset;
}
static constexpr int StaticTrialVDim()
{
int vdim = 1;
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if constexpr (StaticInputDep<s>()) { vdim = StaticInputVDim<s>(); }
});
return vdim;
}
static constexpr int StaticTotalTrialOpDim()
{
int op_dim = 0;
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if constexpr (StaticInputDep<s>()) { op_dim += StaticInputOpDim<s>(); }
});
return op_dim;
}
public:
DerivativeApply() = delete;
DerivativeApply(IntegratorContext ctx,
qfunc_t /*qfunc*/,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache_in):
inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache_in),
dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx, inputs),
ctx.unionfds,
ctx.ir)),
input_idx(create_input_vector_map(ctx, inputs)),
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
input_vdim(get_vdim(inputs)),
output_dtq(create_dtq_maps<Entity::Element>(
outputs,
dtqs,
create_union_field_map_for_dtq(ctx, outputs),
ctx.unionfds,
ctx.ir)),
output_idx(create_output_vector_map(ctx, outputs)),
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
output_vdim(get_vdim(outputs)),
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
input_size_on_qp(
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
out_op_dim(compute_out_op_dim(outputs)),
out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
output_size_on_qp(
[&]
{
int s = 0;
for_constexpr<n_outputs>([&](auto o)
{ s += get<o>(outputs).size_on_qp; });
return s;
}()),
trial_vdim(compute_trial_vdim(inputs, derivative_id)),
total_trial_op_dim(compute_total_trial_op_dim(
inputs, input_is_dependent, input_size_on_qp)),
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
q1d(tensor_1d_size(nq, dim))
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
int direction_field_idx = -1;
for (size_t uf = 0; uf < nfields; uf++)
{
if (static_cast<int>(ctx.unionfds[uf].id) == derivative_id)
{
direction_field_idx = static_cast<int>(uf);
break;
}
}
MFEM_ASSERT(direction_field_idx != -1,
"DerivativeApply: derivative direction field not found");
direction_fd = ctx.unionfds[static_cast<size_t>(direction_field_idx)];
}
//////////////////////////////////////////////////////////////////
template<typename Backend>
void run_kernels(std::vector<Vector *> &ye) const
{
Backend::Run(dim,
q1d,
ctx,
qp_cache,
// inputs
input_idx,
input_B,
input_G,
input_vdim,
input_d1d,
input_q1d,
input_size_on_qp,
input_is_dependent,
// outputs
output_idx,
output_B,
output_G,
output_vdim,
output_d1d,
output_q1d,
out_op_dim,
out_offsets,
trial_vdim,
total_trial_op_dim,
residual_size_on_qp,
output_size_on_qp,
direction_e,
ye,
// fallback arguments
dim,
q1d);
}
//////////////////////////////////////////////////////////////////
void operator()(const std::vector<Vector *> &,
const Vector *direction_l,
std::vector<Vector *> &ye) const
{
if (ctx.attr.Size() == 0) { return; }
MFEM_ASSERT(direction_l != nullptr,
"LocalQF DerivativeApply: direction vector is null");
restriction(direction_fd,
direction_rcache,
*direction_l,
direction_e,
ElementDofOrdering::LEXICOGRAPHIC);
if (q1d <= LocalQFLOBackendMQ1())
{
run_kernels<DerivativeApplyLO>(ye);
}
else if (q1d <= LocalQFHOBackendMQ1())
{
run_kernels<DerivativeApplyHO>(ye);
}
else
{
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
}
}
//////////////////////////////////////////////////////////////////
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
static void
derivative_apply_callback(const IntegratorContext &ctx,
const Vector &qp_cache,
// inputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_inputs> & /*in_idx*/,
const std::array<const real_t *, n_inputs> in_B,
const std::array<const real_t *, n_inputs> in_G,
const std::array<int, n_inputs> &in_vdim,
const std::array<int, n_inputs> &in_d1d,
const std::array<int, n_inputs> &in_q1d,
const std::array<int, n_inputs> &in_size_on_qp,
const std::array<bool, n_inputs> &input_dep,
// outputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_outputs> &out_idx,
const std::array<const real_t *, n_outputs> out_B,
const std::array<const real_t *, n_outputs> out_G,
const std::array<int, n_outputs> &out_vdim,
const std::array<int, n_outputs> &out_d1d,
const std::array<int, n_outputs> &out_q1d,
const std::array<int, n_outputs> &out_op_dim,
const std::array<int, n_outputs> &out_offsets,
const int trial_vdim,
const int total_trial_op_dim,
const int residual_size_on_qp,
const int output_size_on_qp,
const Vector &direction_e,
std::vector<Vector *> &ye,
// fallback arguments
const int dim,
const int q1d)
{
MFEM_CONTRACT_VAR(input_dep);
MFEM_CONTRACT_VAR(in_size_on_qp);
MFEM_CONTRACT_VAR(out_vdim);
MFEM_CONTRACT_VAR(out_op_dim);
MFEM_CONTRACT_VAR(out_offsets);
MFEM_CONTRACT_VAR(trial_vdim);
MFEM_CONTRACT_VAR(total_trial_op_dim);
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
if (ctx.attr.Size() == 0) { return; }
static constexpr auto B2D = backend_t::DIM == 2;
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
const int ne = ctx.nentities;
const int nq = ctx.ir.GetNPoints();
MFEM_CONTRACT_VAR(output_size_on_qp);
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
// --------------------------------------------------
// DIRECTION (trial): XE_dir for the dependent inputs
// --------------------------------------------------
const auto d_direction = direction_e.Read();
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE_dir;
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
using FOP = tuple_element_t<i, inputs_t>;
if (!input_dep[i]) { return; }
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
in_XE_dir[i] = Reshape(d_direction, d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP>)
{
in_XE_dir[i] = Reshape(d_direction, v, q, q, B2D ? 1 : q, ne);
}
else if constexpr (is_weight_fop_v<FOP>) { /* never a direction */ }
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------------------
// OUTPUTS: YE, 3(max DIM) + 1(VDIM) + 1(number of elements)
// --------------------------------------------------
std::array<DeviceTensor<3 + 1 + 1, real_t>, n_outputs> out_YE;
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = out_idx[i];
const int d = out_d1d[i], q = out_q1d[i], v = out_vdim[i];
using FOP = tuple_element_t<i, outputs_t>;
if constexpr (is_gradient_fop_v<FOP> || is_value_fop_v<FOP>)
{
MFEM_VERIFY(ye[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
out_YE[i] = Reshape(ye[k]->ReadWrite(), d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP>)
{
MFEM_VERIFY(ye[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
out_YE[i] = Reshape(ye[k]->ReadWrite(), v, q, q, B2D ? 1 : q, ne);
}
else
{
static_assert(false, "Unsupported FieldOperator");
}
});
auto cache_tensor = DeviceTensor<3, const real_t>(
qp_cache.Read(), nq, residual_size_on_qp, ne);
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
dfem::forall<MTPB>(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
// -----------------------------------------------
// Output integration registers, trial direction (shadow) registers
// and shared memory. `rargs` only ever holds test-function data, so
// it is an output-only bank: slot `o` is q-function parameter
// `n_inputs + o`.
// -----------------------------------------------
output_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> sargs;
MFEM_SHARED typename backend_t::Shared smem;
// -----------------------------------------------
// Load trial direction (sargs) for the dependent inputs
// -----------------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
if constexpr (!StaticInputDep<i>()) { return; }
const auto &XE = in_XE_dir[i];
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
const real_t *B = in_B[i], *G = in_G[i];
auto &sarg = get<i>(sargs);
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop<FOP>::value)
{
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, sarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
using FieldParamT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
backend_t::template LoadGradient<RNK,
decltype(sarg),
decltype(XE),
FieldParamT>(
smem, e, d, q, Q1D, B, G, XE, sarg);
}
else if constexpr (is_identity_fop_v<FOP> || is_weight_fop_v<FOP>)
{
// identity read at qp; weight is never a trial direction
}
else
{
static_assert(false, "Unsupported");
}
});
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Contract the cached Jacobian with the trial direction at each
// quadrature point and push the result into the test registers.
// -----------------------------------------------
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
// The trial direction at this quadrature point is the same
// for every test row (i, k), so pull each dependent input
// slot out of the register bank once, here, instead of once
// per row inside the contraction below.
args_tuple_t dvecs {};
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if constexpr (StaticInputDep<s>())
{
using SARG =
typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
get<s>(dvecs) = backend_t::template qp_pull<SARG>(
get<s>(sargs), qx, qy, qz);
}
});
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value, ao = n_inputs + o;
using FOP = tuple_element_t<o, outputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
constexpr int tv = StaticOutputVDim<o>();
constexpr int to = StaticOutputOpDim<o>();
constexpr int offset_o = StaticOutputOffset<o>();
constexpr int trial_vdim_ct = StaticTrialVDim();
constexpr int total_trial_op_dim_ct = StaticTotalTrialOpDim();
ARG fhat{};
MFEM_UNROLL(tv)
for (int i = 0; i < tv; i++)
{
MFEM_UNROLL(to)
for (int k = 0; k < to; k++)
{
const int row = offset_o + i * to + k;
const int cache_row =
row * trial_vdim_ct * total_trial_op_dim_ct;
real_t sum = 0.0;
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if constexpr (StaticInputDep<s>())
{
constexpr int op_dim_s = StaticInputOpDim<s>();
constexpr int m_offset = StaticInputOpOffset<s>();
const auto &dvec = get<s>(dvecs);
MFEM_UNROLL(trial_vdim_ct)
for (int j = 0; j < trial_vdim_ct; j++)
{
MFEM_UNROLL(op_dim_s)
for (int m = 0; m < op_dim_s; m++)
{
const int cache_idx =
cache_row + j * total_trial_op_dim_ct +
(m + m_offset);
sum += cache_tensor(q, cache_idx, e) *
qf_value_at(dvec, j, m);
}
}
}
});
qf_set_value_at(fhat, i, k, sum);
}
}
auto &YE = out_YE[o];
if constexpr (is_identity_fop_v<FOP>)
{
MFEM_UNROLL(tv)
for (int i = 0; i < tv; i++)
{
MFEM_UNROLL(to)
for (int k = 0; k < to; k++)
{
YE(i + tv * k, qx, qy, qz, e) =
qf_value_at(fhat, i, k);
}
}
}
else
{
backend_t::template qp_push<ARG>(
get<o>(rargs), qx, qy, qz, fhat);
}
});
}
}
}
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Integrate value / gradient outputs to the test dofs
// -----------------------------------------------
for_constexpr<n_outputs>([&](auto ic)
{
constexpr size_t i = ic.value, o = n_inputs + i;
const int d = out_d1d[i], q = out_q1d[i], Q1D = q1d;
const auto B = out_B[i], G = out_G[i];
auto &YE = out_YE[i];
auto &rarg = get<i>(rargs);
using FOP = tuple_element_t<i, outputs_t>;
if constexpr (is_value_fop_v<FOP>)
{
backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, rarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
using YE_t = decltype(YE);
using rarg_t = decltype(rarg);
using qf_param_t =
typename qf_param_slot<qfunc_t, o>::qf_decay_param_t;
constexpr auto RNK = qf_param_slot<qfunc_t, o>::extents.size();
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
smem, e, d, q, Q1D, B, G, YE, rarg);
}
else if constexpr (is_identity_fop_v<FOP>) { /* written at qp */ }
else
{
static_assert(false, "Unsupported");
}
});
},
ne,
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
q1d, in_d1d, out_d1d)),
0,
nullptr);
}
using ApplyKernelType =
decltype(&DerivativeApply::derivative_apply_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyLO,
ApplyKernelType,
(int, int) );
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyHO,
ApplyKernelType,
(int, int) );
};
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
ApplyKernelType
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyLO::Kernel()
{
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
return apply_t::template derivative_apply_callback<
LocalQFLOBackend<DIM, Q1D>>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
ApplyKernelType
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyLO::Fallback(int dim, int q1d)
{
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeApplyLO = typename apply_t::DerivativeApplyLO;
if (dim == 2)
{
return DispatchLOKernelByQ1D<DerivativeApplyLO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchLOKernelByQ1D<DerivativeApplyLO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
ApplyKernelType
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyHO::Kernel()
{
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
return apply_t::template derivative_apply_callback<LocalQFHOBackend<DIM>,
Q1D>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
ApplyKernelType
DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyHO::Fallback(int dim, int q1d)
{
using apply_t = DerivativeApply<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeApplyHO = typename apply_t::DerivativeApplyHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<DerivativeApplyHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<DerivativeApplyHO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,637 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "kernels.hpp"
#include "util.hpp"
#include <array>
namespace mfem::future::LocalQFImpl
{
// Cached transposed Jacobian apply: Jᵀ·w from the qp_cache
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
class DerivativeApplyTranspose
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
using qf_signature = typename get_function_signature<qfunc_t>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
using args_tuple_t = decay_tuple<qf_param_ts>;
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
"LocalQF: q-function arity must match inputs + outputs");
// Input tuple slot referencing the derivative field (compile-time)
static constexpr size_t deriv_input_idx_ct = []() constexpr
{
size_t idx = SIZE_MAX;
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
using FOP = tuple_element_t<i, inputs_t>;
if (FOP::GetFieldId() == derivative_id) { idx = i; }
});
return idx;
}();
static_assert(deriv_input_idx_ct < n_inputs,
"DerivativeApplyTranspose: derivative input slot not found");
const inputs_t inputs;
const outputs_t outputs;
const IntegratorContext ctx;
const Vector &qp_cache; // Jacobian cache from DerivativeSetup
const std::vector<const DofToQuad *> dtqs;
// inputs: dtq, B, G, d1d, q1d, vdim (trial / derivative fields)
const std::array<DofToQuadMap, n_inputs> input_dtq;
const std::array<const real_t *, n_inputs> input_B, input_G;
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
// outputs: dtq, idx, B, G, d1d, q1d, vdim (test / cotangent fields)
const std::array<DofToQuadMap, n_outputs> output_dtq;
const std::array<size_t, n_outputs> output_idx;
const std::array<const real_t *, n_outputs> output_B, output_G;
const std::array<int, n_outputs> output_d1d, output_q1d, output_vdim;
// Jacobian cache metadata
const std::array<bool, n_inputs> input_is_dependent;
const std::array<int, n_inputs> input_size_on_qp;
const std::array<int, n_outputs> out_op_dim;
const std::array<int, n_outputs> out_offsets;
const int output_size_on_qp;
const int trial_vdim;
const int total_trial_op_dim;
const int residual_size_on_qp;
// other constants
const int dim, ne, nq, q1d;
const size_t deriv_infd_idx; // index of the derivative field in ye
// output cotangent restriction workspace (blocked by element)
std::array<int, n_outputs> out_elem_dof_size;
mutable Vector dir_out_e;
/// One restriction cache per output field, resolved on first use.
mutable std::array<RestrictionCache<Entity::Element>, n_outputs>
out_rcaches;
public:
//////////////////////////////////////////////////////////////////
DerivativeApplyTranspose() = delete;
DerivativeApplyTranspose(IntegratorContext ctx,
qfunc_t /*qfunc*/,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache):
inputs(inputs), outputs(outputs), ctx(ctx), qp_cache(qp_cache),
dtqs(make_dtqs(ctx)), input_dtq(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx, inputs),
ctx.unionfds,
ctx.ir)),
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
input_vdim(get_vdim(inputs)),
output_dtq(create_dtq_maps<Entity::Element>(
outputs,
dtqs,
create_union_field_map_for_dtq(ctx, outputs),
ctx.unionfds,
ctx.ir)),
output_idx(create_output_vector_map(ctx, outputs)),
output_B(get_B(output_dtq)), output_G(get_G(output_dtq)),
output_d1d(get_D1D(output_dtq)), output_q1d(get_Q1D(output_dtq)),
output_vdim(get_vdim(outputs)),
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
input_size_on_qp(
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
out_op_dim(compute_out_op_dim(outputs)),
out_offsets(compute_out_offsets(output_vdim, out_op_dim)),
output_size_on_qp(
[&]
{
int s = 0;
for_constexpr<n_outputs>([&](auto o)
{ s += get<o>(outputs).size_on_qp; });
return s;
}()),
trial_vdim(compute_trial_vdim(inputs, derivative_id)),
total_trial_op_dim(compute_total_trial_op_dim(
inputs, input_is_dependent, input_size_on_qp)),
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
q1d(tensor_1d_size(nq, dim)),
deriv_infd_idx(find_infd_index(ctx, derivative_id)), out_elem_dof_size{}
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
MFEM_ASSERT(
deriv_infd_idx != SIZE_MAX,
"DerivativeApplyTranspose: derivative field not found in infds");
// Size the workspace that holds the output cotangent(s) in element
// layout.
int total_dir_e_size = 0;
for_constexpr<n_outputs>([&](auto o)
{
const int elem_sz = compute_element_dof_sz(
ctx.outfds[output_idx[o]], ne, ElementDofOrdering::LEXICOGRAPHIC);
out_elem_dof_size[o] = elem_sz;
total_dir_e_size += elem_sz;
});
dir_out_e.SetSize(total_dir_e_size * ne);
dir_out_e.UseDevice(true);
dir_out_e.Read();
}
//////////////////////////////////////////////////////////////////
template<typename Backend>
void run_kernels(std::vector<Vector *> &ye) const
{
Backend::Run(dim,
q1d,
ctx,
qp_cache,
dir_out_e,
// inputs (integration target metadata)
input_B,
input_G,
input_vdim,
input_d1d,
input_q1d,
input_size_on_qp,
input_is_dependent,
// outputs (direction interpolation metadata)
output_B,
output_G,
output_vdim,
output_d1d,
output_q1d,
out_op_dim,
out_offsets,
trial_vdim,
total_trial_op_dim,
residual_size_on_qp,
output_size_on_qp,
deriv_infd_idx,
ye,
// fallback arguments
dim,
q1d);
}
//////////////////////////////////////////////////////////////////
void operator()(const std::vector<Vector *> & /*xe*/,
const Vector *direction_l,
std::vector<Vector *> &ye) const
{
if (ctx.attr.Size() == 0) { return; }
MFEM_ASSERT(direction_l != nullptr,
"LocalQF DerivativeApplyTranspose: direction vector is null");
// Restrict output cotangent from L-vectors into element layout
// (dir_out_e).
int l_offset = 0;
int e_offset = 0;
for_constexpr<n_outputs>([&](auto o)
{
const size_t outfd = output_idx[o];
const auto &fd = ctx.outfds[outfd];
const int l_size = GetVSize(fd);
Vector dir_o_l(*const_cast<Vector *>(direction_l), l_offset, l_size);
dir_o_l.UseDevice(true);
const int elem_sz = out_elem_dof_size[o];
Vector dir_o_e(dir_out_e, e_offset, elem_sz * ne);
dir_o_e.UseDevice(true);
restriction(fd, out_rcaches[o], dir_o_l, dir_o_e,
ElementDofOrdering::LEXICOGRAPHIC);
l_offset += l_size;
e_offset += elem_sz * ne;
});
if (q1d <= LocalQFLOBackendMQ1())
{
run_kernels<DerivativeApplyTransposeLO>(ye);
}
else if (q1d <= LocalQFHOBackendMQ1())
{
run_kernels<DerivativeApplyTransposeHO>(ye);
}
else
{
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
}
}
//////////////////////////////////////////////////////////////////
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
static void derivative_apply_transpose_callback(
const IntegratorContext &ctx,
const Vector &qp_cache,
const Vector &dir_e, // restricted, concatenated output cotangents
// inputs (integration target metadata)
const std::array<const real_t *, n_inputs> in_B,
const std::array<const real_t *, n_inputs> in_G,
const std::array<int, n_inputs> &in_vdim,
const std::array<int, n_inputs> &in_d1d,
const std::array<int, n_inputs> &in_q1d,
const std::array<int, n_inputs> &in_size_on_qp,
const std::array<bool, n_inputs> &input_dep,
// outputs (direction interpolation metadata)
const std::array<const real_t *, n_outputs> out_B,
const std::array<const real_t *, n_outputs> out_G,
const std::array<int, n_outputs> &out_vdim,
const std::array<int, n_outputs> &out_d1d,
const std::array<int, n_outputs> &out_q1d,
const std::array<int, n_outputs> &out_op_dim,
const std::array<int, n_outputs> &out_offsets,
const int trial_vdim,
const int total_trial_op_dim,
const int residual_size_on_qp,
const int output_size_on_qp,
const size_t deriv_infd_idx,
std::vector<Vector *> &ye,
// fallback arguments
const int dim,
const int q1d)
{
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
if (ctx.attr.Size() == 0) { return; }
static constexpr auto B2D = backend_t::DIM == 2;
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
const int ne = ctx.nentities;
const int nq = ctx.ir.GetNPoints();
MFEM_CONTRACT_VAR(output_size_on_qp);
MFEM_CONTRACT_VAR(in_q1d);
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
// --------------------------------------------------
// DIRECTION (test cotangent): out_XE_dir, concatenated per output
// --------------------------------------------------
const auto d_dir = dir_e.Read();
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_outputs> out_XE_dir;
int e_offset = 0;
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value;
const int d = out_d1d[o], q = out_q1d[o], v = out_vdim[o];
using FOP = tuple_element_t<o, outputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
out_XE_dir[o] = Reshape(d_dir + e_offset, d, d, B2D ? 1 : d, v, ne);
e_offset += k_dim(d) * v * ne;
}
else if constexpr (is_identity_fop_v<FOP>)
{
out_XE_dir[o] = Reshape(d_dir + e_offset, v, q, q, B2D ? 1 : q, ne);
e_offset += k_dim(q) * v * ne;
}
else
{
static_assert(false, "Unsupported");
}
});
// --------------------------------------------------
// DERIVATIVE TRIAL FIELD: ye_XE (accumulates Jᵀ w)
// --------------------------------------------------
const int d_in = in_d1d[deriv_input_idx_ct];
const int v_in = in_vdim[deriv_input_idx_ct];
auto ye_XE = Reshape(
ye[deriv_infd_idx]->ReadWrite(), d_in, d_in, B2D ? 1 : d_in, v_in, ne);
auto cache_tensor = DeviceTensor<3, const real_t>(
qp_cache.Read(), nq, residual_size_on_qp, ne);
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
dfem::forall<MTPB>(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
// -----------------------------------------------
// Output cotangent (direction) registers live in the output slots;
// the trial integration data is pushed into the input slots.
// -----------------------------------------------
args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
MFEM_SHARED typename backend_t::Shared smem;
// -----------------------------------------------
// Interpolate the test cotangent to quadrature points (output slots)
// -----------------------------------------------
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value, ao = n_inputs + o;
using FOP = tuple_element_t<o, outputs_t>;
const auto &XE = out_XE_dir[o];
const int d = out_d1d[o], q = out_q1d[o], Q1D = q1d;
const real_t *B = out_B[o], *G = out_G[o];
auto &oarg = get<ao>(rargs);
if constexpr (is_value_fop_v<FOP>)
{
backend_t::LoadValue(smem, e, d, q, Q1D, B, XE, oarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
constexpr auto RNK = qf_param_slot<qfunc_t, ao>::extents.size();
using FieldParamT =
typename qf_param_slot<qfunc_t, ao>::qf_decay_param_t;
backend_t::template LoadGradient<RNK,
decltype(oarg),
decltype(XE),
FieldParamT>(
smem, e, d, q, Q1D, B, G, XE, oarg);
}
else if constexpr (is_identity_fop_v<FOP>)
{
// identity cotangent is read directly at qp from out_XE_dir
}
else
{
static_assert(false, "Unsupported");
}
});
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Contract the transposed cached Jacobian with the test cotangent at
// each quadrature point and push the trial result into the dependent
// input registers.
// -----------------------------------------------
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
// The test cotangent at this quadrature point is the same
// for every trial column (j, m), so pull each interpolated
// output slot out of the register bank once, here, instead
// of once per column inside the contraction below. Identity
// outputs have no register bank and are read from
// out_XE_dir at the point of use.
args_tuple_t wvecs {};
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value, ao = n_inputs + o;
using OFOP = tuple_element_t<o, outputs_t>;
if constexpr (is_value_fop_v<OFOP> ||
is_gradient_fop_v<OFOP>)
{
using OARG =
typename qf_param_slot<qfunc_t, ao>::qf_reg_param_t;
get<ao>(wvecs) = backend_t::template qp_pull<OARG>(
get<ao>(rargs), qx, qy, qz);
}
});
int m_offset = 0;
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if (!input_dep[s]) { return; }
using SARG =
typename qf_param_slot<qfunc_t, s>::qf_reg_param_t;
const int vdim_s = in_vdim[s];
const int op_dim_s = in_size_on_qp[s] / vdim_s;
SARG fhat{};
for (int j = 0; j < trial_vdim; j++)
{
for (int m = 0; m < op_dim_s; m++)
{
const int col =
j * total_trial_op_dim + (m + m_offset);
real_t sum = 0.0;
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value, ao = n_inputs + o;
using OFOP = tuple_element_t<o, outputs_t>;
const int tv = out_vdim[o], to = out_op_dim[o];
const auto offset_o = out_offsets[o];
const auto &cache = cache_tensor;
if constexpr (is_value_fop_v<OFOP> ||
is_gradient_fop_v<OFOP>)
{
const auto &wvec = get<ao>(wvecs);
for (int i = 0; i < tv; i++)
{
for (int k = 0; k < to; k++)
{
const int row = offset_o + i * to + k;
const int cache_idx =
row * trial_vdim *
total_trial_op_dim +
col;
sum += cache(q, cache_idx, e) *
qf_value_at(wvec, i, k);
}
}
}
else if constexpr (is_identity_fop_v<OFOP>)
{
const auto &XEo = out_XE_dir[o];
for (int i = 0; i < tv; i++)
{
for (int k = 0; k < to; k++)
{
const int row = offset_o + i * to + k;
const int cache_idx =
row * trial_vdim *
total_trial_op_dim +
col;
sum += cache(q, cache_idx, e) *
XEo(i + tv * k, qx, qy, qz, e);
}
}
}
});
qf_set_value_at(fhat, j, m, sum);
}
}
backend_t::template qp_push<SARG>(
get<s>(rargs), qx, qy, qz, fhat);
m_offset += op_dim_s;
});
}
}
}
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Integrate the trial result into the derivative field dofs. Multiple
// dependent input slots (e.g. value and gradient of the same field)
// accumulate into ye_XE via the writers' '+=' semantics.
// -----------------------------------------------
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if (!input_dep[s]) { return; }
using FOP = tuple_element_t<s, inputs_t>;
const int d = in_d1d[s], q = in_q1d[s], Q1D = q1d;
const real_t *B = in_B[s], *G = in_G[s];
auto &sarg = get<s>(rargs);
auto &YE = ye_XE;
if constexpr (is_value_fop_v<FOP>)
{
backend_t::WriteValue(smem, e, d, q, Q1D, B, YE, sarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
using YE_t = decltype(YE);
using rarg_t = decltype(sarg);
using qf_param_t =
typename qf_param_slot<qfunc_t, s>::qf_decay_param_t;
constexpr auto RNK = qf_param_slot<qfunc_t, s>::extents.size();
backend_t::template WriteGradient<RNK, rarg_t, YE_t, qf_param_t>(
smem, e, d, q, Q1D, B, G, YE, sarg);
}
else
{
// identity / weight derivative targets are not produced here
}
});
},
ne,
backend_t::thread_blocks(compute_kernel_thread_1d<inputs_t, outputs_t>(
q1d, in_d1d, out_d1d)),
0,
nullptr);
}
using TransposeKernelType =
decltype(&DerivativeApplyTranspose::
derivative_apply_transpose_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeLO,
TransposeKernelType,
(int, int) );
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeApplyTransposeHO,
TransposeKernelType,
(int, int) );
};
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeApplyTranspose<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::TransposeKernelType
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyTransposeLO::Kernel()
{
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
using transpose_t =
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
return transpose_t::template derivative_apply_transpose_callback<
LocalQFLOBackend<DIM, Q1D>>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeApplyTranspose<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::TransposeKernelType
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyTransposeLO::Fallback(int dim, int q1d)
{
using transpose_t =
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeApplyTransposeLO =
typename transpose_t::DerivativeApplyTransposeLO;
if (dim == 2)
{
return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchLOKernelByQ1D<DerivativeApplyTransposeLO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeApplyTranspose<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::TransposeKernelType
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyTransposeHO::Kernel()
{
using transpose_t =
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
return transpose_t::
template derivative_apply_transpose_callback<LocalQFHOBackend<DIM>, Q1D>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeApplyTranspose<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::TransposeKernelType
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeApplyTransposeHO::Fallback(int dim, int q1d)
{
using transpose_t =
DerivativeApplyTranspose<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeApplyTransposeHO =
typename transpose_t::DerivativeApplyTransposeHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<DerivativeApplyTransposeHO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,910 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "../../../kernels.hpp"
#include "kernels.hpp"
#include "util.hpp"
#include <array>
#include <type_traits>
namespace ker = mfem::kernels::internal;
namespace mfem::future::LocalQFImpl
{
namespace detail
{
template<int DIM>
MFEM_HOST_DEVICE inline int tensor_idx(int x, int y, int z, int N)
{
static_assert(DIM == 2 || DIM == 3);
if constexpr (DIM == 2) { assert(z == 0); }
return x + N * (y + N * z);
}
template<int DIM>
MFEM_HOST_DEVICE inline real_t
trial_basis_weight_value(const DeviceTensor<3, const real_t> &B,
const int qx,
const int qy,
const int qz,
const int Jx,
const int Jy,
const int Jz)
{
static_assert(DIM == 2 || DIM == 3);
return B(qx, 0, Jx) * B(qy, 0, Jy) * ((DIM == 3) ? B(qz, 0, Jz) : 1.0);
}
template<int DIM>
MFEM_HOST_DEVICE inline real_t
trial_basis_weight_gradient(const DeviceTensor<3, const real_t> &B,
const DeviceTensor<3, const real_t> &G,
const int m,
const int qx,
const int qy,
const int qz,
const int Jx,
const int Jy,
const int Jz)
{
const auto Gx = G(qx, 0, Jx), Gy = G(qy, 0, Jy);
const auto Bx = B(qx, 0, Jx), By = B(qy, 0, Jy);
if constexpr (DIM == 2)
{
MFEM_CONTRACT_VAR(qz & Jz);
return (m == 0) ? Gx * By : Bx * Gy;
}
else
{
const auto Bz = B(qz, 0, Jz), Gz = G(qz, 0, Jz);
return (m == 0) ? Gx * By * Bz
: (m == 1) ? Bx * Gy * Bz
: (m == 2) ? Bx * By * Gz
: (assert(false), 0.0);
}
}
template<int DIM, int MQ1, typename Shared, typename output_t>
MFEM_HOST_DEVICE void
map_quadrature_data_to_fields(DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const output_t &output,
const DofToQuadMap &dtq,
Shared &s,
const int tv_dof = -1)
{
using output_fop_t = std::decay_t<output_t>;
const auto B = dtq.B, G = dtq.G;
const bool f_slab = (tv_dof >= 0);
const int vdim = output.vdim;
const int vd_begin = f_slab ? tv_dof : 0;
const int vd_end = f_slab ? tv_dof + 1 : vdim;
if constexpr (is_value_fop_v<output_fop_t>)
{
const auto [q1d, unused, d1d] = B.GetShape();
MFEM_CONTRACT_VAR(unused);
const int test_dim = output.size_on_qp / vdim;
MFEM_CONTRACT_VAR(test_dim);
const int f_vdim = f_slab ? 1 : vdim;
if constexpr (DIM == 2)
{
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
ker::LoadMatrix(d1d, q1d, B, s.B);
ker::s_regs2d_t<MQ1> r_qp, Y;
for (int vd = vd_begin; vd < vd_end; vd++)
{
const int fi = f_slab ? 0 : vd;
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{ r_qp[qy][qx] = fqp(fi, 0, qx, qy); }
MFEM_SYNC_THREAD;
ker::Eval2d<MQ1, true>(d1d, q1d, s.M, s.B, r_qp, Y);
MFEM_FOREACH_THREAD(dy, y, d1d)
MFEM_FOREACH_THREAD(dx, x, d1d) { yd(dx, dy, vd) += Y[dy][dx]; }
MFEM_SYNC_THREAD;
}
}
else
{
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
ker::LoadMatrix(d1d, q1d, B, s.B);
ker::s_regs3d_t<MQ1> f_qp, Y;
for (int vd = vd_begin; vd < vd_end; vd++)
{
const int fi = f_slab ? 0 : vd;
for (int qz = 0; qz < q1d; qz++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{ f_qp[qz][qy][qx] = fqp(fi, 0, qx, qy, qz); }
}
MFEM_SYNC_THREAD;
ker::Eval3d<MQ1, true>(d1d, q1d, s.M, s.B, f_qp, Y);
for (int dz = 0; dz < d1d; dz++)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
MFEM_FOREACH_THREAD(dx, x, d1d)
{ yd(dx, dy, dz, vd) += Y[dz][dy][dx]; }
}
MFEM_SYNC_THREAD;
}
}
}
else if constexpr (is_gradient_fop_v<output_fop_t>)
{
const auto [q1d, unused, d1d] = G.GetShape();
MFEM_CONTRACT_VAR(unused);
const int test_dim = output.size_on_qp / vdim;
const int f_vdim = f_slab ? 1 : vdim;
if constexpr (DIM == 2)
{
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
ker::LoadMatrix(d1d, q1d, B, s.B);
ker::LoadMatrix(d1d, q1d, G, s.G);
ker::vd_regs2d_t<1, DIM, MQ1> X, Y;
for (int vd = vd_begin; vd < vd_end; vd++)
{
const int fi = f_slab ? 0 : vd;
MFEM_FOREACH_THREAD(qx, x, q1d)
MFEM_FOREACH_THREAD(qy, y, q1d)
for (int k = 0; k < DIM; k++)
{
X[0][k][qy][qx] = fqp(fi, k, qx, qy);
}
MFEM_SYNC_THREAD;
ker::Grad2d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
MFEM_FOREACH_THREAD(dy, y, d1d)
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t u = 0.0;
for (int k = 0; k < DIM; k++) { u += Y[0][k][dy][dx]; }
yd(dx, dy, vd) += u;
}
MFEM_SYNC_THREAD;
}
}
else
{
const auto fqp = Reshape(&f(0, 0, 0), f_vdim, test_dim, q1d, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
ker::LoadMatrix(d1d, q1d, B, s.B);
ker::LoadMatrix(d1d, q1d, G, s.G);
ker::vd_regs3d_t<1, DIM, MQ1> X, Y;
for (int vd = vd_begin; vd < vd_end; vd++)
{
const int fi = f_slab ? 0 : vd;
for (int qz = 0; qz < q1d; qz++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
for (int k = 0; k < DIM; k++)
{
X[0][k][qz][qy][qx] = fqp(fi, k, qx, qy, qz);
}
}
MFEM_SYNC_THREAD;
ker::Grad3d<1, DIM, MQ1, true>(d1d, q1d, s.M, s.B, s.G, X, Y);
for (int dz = 0; dz < d1d; dz++)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t u = 0.0;
for (int k = 0; k < DIM; k++) { u += Y[0][k][dz][dy][dx]; }
yd(dx, dy, dz, vd) += u;
}
}
MFEM_SYNC_THREAD;
}
}
}
else if constexpr (is_identity_fop_v<output_fop_t>)
{
const auto [q1d, unused, d1d] = B.GetShape();
MFEM_CONTRACT_VAR(unused);
MFEM_CONTRACT_VAR(d1d);
const int f_sq = f_slab ? 1 : output.size_on_qp;
const int sq_begin = f_slab ? tv_dof : 0;
const int sq_end = f_slab ? tv_dof + 1 : output.size_on_qp;
if constexpr (DIM == 2)
{
const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d);
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
for (int sq = sq_begin; sq < sq_end; sq++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{
int qz = 0;
MFEM_CONTRACT_VAR(qz);
yqp(sq, qx, qy) = fqp(0, qx, qy);
}
MFEM_SYNC_THREAD;
}
}
else
{
const auto fqp = Reshape(&f(0, 0, 0), f_sq, q1d, q1d, q1d);
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
for (int sq = sq_begin; sq < sq_end; sq++)
{
for (int qz = 0; qz < q1d; qz++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{ yqp(sq, qx, qy, qz) = fqp(0, qx, qy, qz); }
}
MFEM_SYNC_THREAD;
}
}
}
else
{
MFEM_ABORT_KERNEL("quadrature data mapping to field is not implemented"
" for this field descriptor with sum factorization on"
" tensor product elements");
}
}
template<int DIM,
int MQ1,
typename Shared,
typename input_fop_ts,
std::size_t n_inputs,
typename output_fop_t>
MFEM_HOST_DEVICE void assemble_element_mat_sumfact(
const DeviceTensor<5, real_t> &Ae,
const DeviceTensor<6, const real_t> &qpdc,
const int e,
const DeviceTensor<1, const real_t> &itod,
const input_fop_ts &inputs,
const output_fop_t &output,
const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
const DofToQuadMap &output_dtq,
const int q1d,
const int num_trial_dof_1d,
Shared &smem)
{
static constexpr int MQN = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
// Slab must hold full (test_vdim, test_op_dim, nq) fhat
static constexpr int FHAT_SLAB_MAX = MQN * 4;
static constexpr bool grad_out = is_gradient_fop_v<output_fop_t>;
static constexpr bool ident_out = is_identity_fop_v<output_fop_t>;
// qpdc shape: (nq, total_trial_op_dim, trial_vdim, test_op_dim, test_vdim, ne)
const int test_vdim = qpdc.GetShape()[4];
const int test_op_dim = qpdc.GetShape()[3];
const int trial_vdim = qpdc.GetShape()[2];
const int num_test_dof = Ae.GetShape()[0];
const int nq = qpdc.GetShape()[0];
const int size_on_qp = output.size_on_qp;
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
MFEM_VERIFY(test_op_dim <= DIM,
"DerivativeAssemble: test_op_dim exceeds spatial DIM");
MFEM_VERIFY(test_op_dim * nq <= FHAT_SLAB_MAX,
"DerivativeAssemble: fhat slab exceeds capacity");
#endif
MFEM_SHARED real_t fhat_storage[FHAT_SLAB_MAX];
const auto &inputs_ref = inputs;
// Iterate quadrature points using the thread-block mapping
const auto foreach_qp = [&](auto &&body)
{
if constexpr (DIM == 2)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
MFEM_FOREACH_THREAD(qy, y, q1d) { body(qx, qy, 0); }
}
else
{
for (int qz = 0; qz < q1d; qz++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d) { body(qx, qy, qz); }
}
}
};
const auto zero_slab = [&](const int n_comp)
{
foreach_qp([&](const int qx, const int qy, const int qz)
{
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
for (int k = 0; k < n_comp; k++) { fhat_storage[k * nq + q] = 0.0; }
});
MFEM_SYNC_THREAD;
};
const auto accumulate_tv = [&](const int Jx,
const int Jy,
const int Jz,
const int j,
const int tv,
const int tod_only = -1)
{
int m_offset = 0;
for_constexpr<n_inputs>([&](auto inp)
{
using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
const int trial_op_dim = static_cast<int>(itod(static_cast<int>(inp)));
if (trial_op_dim == 0) { return; }
const auto &B = input_dtq_maps[inp].B;
const auto &G = input_dtq_maps[inp].G;
if constexpr (is_value_fop<fop_t>::value)
{
foreach_qp([&](const int qx, const int qy, const int qz)
{
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
const real_t w =
trial_basis_weight_value<DIM>(B, qx, qy, qz, Jx, Jy, Jz);
for (int m = 0; m < trial_op_dim; m++)
{
for (int k = 0; k < test_op_dim; k++)
{
if (tod_only >= 0 && k != tod_only) { continue; }
const real_t f = qpdc(q, m + m_offset, j, k, tv, e);
if constexpr (grad_out && !ident_out)
{
fhat_storage[k * nq + q] += f * w;
}
else
{
fhat_storage[q] += f * w;
}
}
}
});
}
else if constexpr (is_gradient_fop<fop_t>::value)
{
foreach_qp([&](const int qx, const int qy, const int qz)
{
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
for (int m = 0; m < trial_op_dim; m++)
{
const real_t w = trial_basis_weight_gradient<DIM>(
B, G, m, qx, qy, qz, Jx, Jy, Jz);
for (int k = 0; k < test_op_dim; k++)
{
if (tod_only >= 0 && k != tod_only) { continue; }
const real_t f = qpdc(q, m + m_offset, j, k, tv, e);
if constexpr (grad_out && !ident_out)
{
fhat_storage[k * nq + q] += f * w;
}
else
{
fhat_storage[q] += f * w;
}
}
}
});
}
else
{
MFEM_ABORT_KERNEL("sum factorized sparse matrix assemble routine "
"not implemented for field operator");
}
MFEM_SYNC_THREAD;
m_offset += trial_op_dim;
});
};
for (int Jz = 0; Jz < ((DIM == 2) ? 1 : num_trial_dof_1d); Jz++)
{
for (int Jy = 0; Jy < num_trial_dof_1d; Jy++)
{
for (int Jx = 0; Jx < num_trial_dof_1d; Jx++)
{
const int J = tensor_idx<DIM>(Jx, Jy, Jz, num_trial_dof_1d);
for (int j = 0; j < trial_vdim; j++)
{
auto bvtfhat =
Reshape(&Ae(0, 0, J, j, e), num_test_dof, test_vdim);
const int fhat_size = test_vdim * test_op_dim * nq;
if (fhat_size <= FHAT_SLAB_MAX)
{
auto fhat =
Reshape(&fhat_storage[0], test_vdim, test_op_dim, nq);
for (int tv = 0; tv < test_vdim; tv++)
{
for (int tod = 0; tod < test_op_dim; tod++)
{
foreach_qp([&](const int qx, const int qy, const int qz)
{
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
fhat(tv, tod, q) = 0.0;
});
}
}
MFEM_SYNC_THREAD;
int m_offset = 0;
for_constexpr<n_inputs>([&](auto inp)
{
using fop_t = std::decay_t<decltype(get<inp>(inputs_ref))>;
const int trial_op_dim =
static_cast<int>(itod(static_cast<int>(inp)));
if (trial_op_dim == 0) { return; }
const auto &B = input_dtq_maps[inp].B;
const auto &G = input_dtq_maps[inp].G;
if constexpr (is_value_fop<fop_t>::value)
{
foreach_qp([&](const int qx, const int qy, const int qz)
{
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
const real_t w = trial_basis_weight_value<DIM>(
B, qx, qy, qz, Jx, Jy, Jz);
for (int m = 0; m < trial_op_dim; m++)
{
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t f =
qpdc(q, m + m_offset, j, k, i, e);
fhat(i, k, q) += f * w;
}
}
}
});
}
else if constexpr (is_gradient_fop<fop_t>::value)
{
foreach_qp([&](const int qx, const int qy, const int qz)
{
const int q = tensor_idx<DIM>(qx, qy, qz, q1d);
for (int m = 0; m < trial_op_dim; m++)
{
const real_t w = trial_basis_weight_gradient<DIM>(
B, G, m, qx, qy, qz, Jx, Jy, Jz);
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t f =
qpdc(q, m + m_offset, j, k, i, e);
fhat(i, k, q) += f * w;
}
}
}
});
}
else
{
MFEM_ABORT_KERNEL(
"sum factorized sparse matrix assemble routine "
"not implemented for field operator");
}
MFEM_SYNC_THREAD;
m_offset += trial_op_dim;
});
map_quadrature_data_to_fields<DIM, MQ1>(
bvtfhat, fhat, output, output_dtq, smem);
}
else if constexpr (ident_out)
{
for (int sq = 0; sq < size_on_qp; sq++)
{
const int tv = sq / test_op_dim;
const int tod = sq % test_op_dim;
zero_slab(1);
accumulate_tv(Jx, Jy, Jz, j, tv, tod);
auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
map_quadrature_data_to_fields<DIM, MQ1>(
bvtfhat, f_slab, output, output_dtq, smem, sq);
}
}
else if constexpr (grad_out)
{
for (int tv = 0; tv < test_vdim; tv++)
{
zero_slab(test_op_dim);
accumulate_tv(Jx, Jy, Jz, j, tv);
auto f_slab =
Reshape(&fhat_storage[0], 1, test_op_dim, nq);
map_quadrature_data_to_fields<DIM, MQ1>(
bvtfhat, f_slab, output, output_dtq, smem, tv);
}
}
else
{
for (int tv = 0; tv < test_vdim; tv++)
{
zero_slab(1);
accumulate_tv(Jx, Jy, Jz, j, tv);
auto f_slab = Reshape(&fhat_storage[0], 1, 1, nq);
map_quadrature_data_to_fields<DIM, MQ1>(
bvtfhat, f_slab, output, output_dtq, smem, tv);
}
}
}
}
}
}
}
} // namespace detail
// ────────────────────────────────────────────────────────────────────────────
// Assemble sparse Jacobian from cached quadrature derivatives (tensor 2D/3D)
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
class DerivativeAssemble
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
const IntegratorContext ctx;
const Vector &qp_cache;
inputs_t inputs;
outputs_t outputs;
const bool use_sum_factorization;
const std::vector<const DofToQuad *> dtqs;
const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
const std::array<bool, n_inputs> input_is_dependent;
const size_t trial_field_uf;
const size_t test_field_uf;
const ParFiniteElementSpace *test_fes;
const ParFiniteElementSpace *trial_fes;
const int test_vdim;
const int test_op_dim;
const int num_test_dof;
const int trial_vdim;
const int trial_op_dim;
const int num_trial_dof;
const int dim, ne, nq, q1d;
const int num_trial_dof_1d;
const int total_trial_op_dim;
mutable Vector inputs_trial_op_dim;
mutable Vector Ae_mem;
public:
DerivativeAssemble() = delete;
DerivativeAssemble(IntegratorContext ctx_in,
qfunc_t /*qfunc*/,
inputs_t inputs_in,
outputs_t outputs_in,
const Vector &qp_cache_in):
ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
outputs(outputs_in), use_sum_factorization(
[&]
{
const Element::Type etype =
Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
}()),
dtqs(
[&]
{
const DofToQuad::Mode dtq_mode = use_sum_factorization
? DofToQuad::Mode::TENSOR
: DofToQuad::Mode::FULL;
std::vector<const DofToQuad *> maps;
maps.reserve(ctx_in.unionfds.size());
for (const auto &field : ctx_in.unionfds)
{
maps.emplace_back(
GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
}
return maps;
}()),
input_dtq_maps(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx_in, inputs),
ctx_in.unionfds,
ctx_in.ir)),
output_dtq_maps(create_dtq_maps<Entity::Element>(
outputs,
dtqs,
create_union_field_map_for_dtq(ctx_in, outputs),
ctx_in.unionfds,
ctx_in.ir)),
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
test_field_uf(
find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
test_fes(
[&]
{
const auto *fes = std::get_if<const ParFiniteElementSpace *>(
&ctx_in.unionfds[test_field_uf].data);
MFEM_ASSERT(fes != nullptr && *fes != nullptr,
"LocalQFBackend: test space is not a ParFiniteElementSpace");
return *fes;
}()),
trial_fes(
[&]
{
const auto *fes = std::get_if<const ParFiniteElementSpace *>(
&ctx_in.unionfds[trial_field_uf].data);
MFEM_ASSERT(fes != nullptr && *fes != nullptr,
"LocalQFBackend: trial space is not a ParFiniteElementSpace");
return *fes;
}()),
test_vdim(get<0>(outputs).vdim),
test_op_dim(get<0>(outputs).size_on_qp / test_vdim),
num_test_dof(test_fes->GetFE(0)->GetDof()),
trial_vdim(compute_trial_vdim(inputs, derivative_id)), trial_op_dim(
[&]
{
int top = 0;
for_constexpr<n_inputs>([&](auto i)
{
if (get<i>(inputs).GetFieldId() == derivative_id)
{
top = get<i>(inputs).size_on_qp / get<i>(inputs).vdim;
}
});
return top;
}()),
num_trial_dof(trial_fes->GetFE(0)->GetDof()),
dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
num_trial_dof_1d(tensor_1d_size(num_trial_dof, dim)), total_trial_op_dim(
[&]
{
const auto in_qp_sizes =
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
return compute_total_trial_op_dim(
inputs, input_is_dependent, in_qp_sizes);
}()),
inputs_trial_op_dim(), Ae_mem()
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
MFEM_ASSERT(trial_field_uf != SIZE_MAX,
"DerivativeAssemble: trial field not found in unionfds");
MFEM_ASSERT(test_field_uf != SIZE_MAX,
"DerivativeAssemble: test field not found in unionfds");
MFEM_ASSERT(trial_vdim > 0,
"LocalQFBackend: could not determine trial vdim");
MFEM_ASSERT(total_trial_op_dim > 0,
"LocalQFBackend: no dependent inputs found");
inputs_trial_op_dim.UseDevice(true);
inputs_trial_op_dim.SetSize(n_inputs);
auto inputs_trial_op_dim_host = inputs_trial_op_dim.HostWrite();
for_constexpr<n_inputs>([&](auto i)
{
inputs_trial_op_dim_host[i] =
input_is_dependent[i]
? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
: 0;
});
const int elem_mat_size =
num_test_dof * test_vdim * num_trial_dof * trial_vdim;
Ae_mem.SetSize(elem_mat_size * ne, Device::GetDeviceMemoryType());
Ae_mem.UseDevice(true);
Ae_mem = 0.0;
}
void operator()(SparseMatrix *&A) const
{
if (ctx.attr.Size() == 0) { return; }
if (!(use_sum_factorization && (dim == 2 || dim == 3)))
{
MFEM_ABORT("DerivativeAssemble optimized path is implemented "
"for tensor-product 2D/3D elements only");
}
DerivativeAssembleHO::Run(dim,
q1d,
ctx,
qp_cache,
Ae_mem,
inputs,
outputs,
input_dtq_maps,
output_dtq_maps[0],
inputs_trial_op_dim,
test_vdim,
test_op_dim,
num_test_dof,
num_trial_dof,
num_trial_dof_1d,
trial_vdim,
total_trial_op_dim,
nq,
ne,
q1d,
dim);
A = new SparseMatrix;
A->OverrideSize(test_fes->GetVSize(), trial_fes->GetVSize());
const auto *test_restr = dynamic_cast<const ElementRestriction *>(
test_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
const auto *trial_restr = dynamic_cast<const ElementRestriction *>(
trial_fes->GetElementRestriction(ElementDofOrdering::LEXICOGRAPHIC));
MFEM_VERIFY(test_restr != nullptr && trial_restr != nullptr,
"DerivativeAssemble SparseMatrix assembly requires "
"H1/conforming ElementRestriction spaces");
test_restr->FillSparseMatrix(Ae_mem, *A, *trial_restr);
}
template<typename backend_t = LocalQFHOBackend<3>, int T_Q1D = 0>
static void derivative_assemble_callback(
const IntegratorContext &ctx,
const Vector &qp_cache,
Vector &Ae_mem,
const inputs_t &inputs,
const outputs_t &outputs,
const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
const DofToQuadMap &output_dtq,
const Vector &inputs_trial_op_dim,
const int test_vdim,
const int test_op_dim,
const int num_test_dof,
const int num_trial_dof,
const int num_trial_dof_1d,
const int trial_vdim,
const int total_trial_op_dim,
const int nq,
const int ne,
const int q1d,
const int dim)
{
static constexpr int DIM = backend_t::DIM;
static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
static constexpr int MNQ = (DIM == 2) ? MQ1 * MQ1 : MQ1 * MQ1 * MQ1;
MFEM_VERIFY(dim == DIM,
"DerivativeAssemble: mesh dim does not match backend");
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
MFEM_VERIFY(q1d <= MQ1, "q1d exceeds backend MQ1 limit");
MFEM_VERIFY(nq <= MNQ,
"DerivativeAssemble: nq exceeds backend quadrature capacity");
MFEM_VERIFY(test_op_dim <= DIM,
"DerivativeAssemble: test_op_dim exceeds spatial DIM");
if (ctx.attr.Size() == 0) { return; }
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
const auto qpdc = Reshape(qp_cache.Read(),
nq,
total_trial_op_dim,
trial_vdim,
test_op_dim,
test_vdim,
ne);
const auto itod = Reshape(inputs_trial_op_dim.Read(), n_inputs);
auto Ae = Reshape(Ae_mem.ReadWrite(),
num_test_dof,
test_vdim,
num_trial_dof,
trial_vdim,
ne);
dfem::forall(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
static constexpr int DIM = backend_t::DIM;
static constexpr int MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
MFEM_SHARED typename backend_t::Shared s;
detail::assemble_element_mat_sumfact<DIM, MQ1>(Ae,
qpdc,
e,
itod,
inputs,
get<0>(outputs),
input_dtq_maps,
output_dtq,
q1d,
num_trial_dof_1d,
s);
},
ne,
backend_t::thread_blocks(q1d),
0,
nullptr);
}
using AssembleKernelType =
decltype(&DerivativeAssemble::derivative_assemble_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleHO,
AssembleKernelType,
(int, int));
};
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline
typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
AssembleKernelType
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeAssembleHO::Kernel()
{
static_assert(DIM == 2 || DIM == 3);
using assemble_t =
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
return assemble_t::template derivative_assemble_callback<
LocalQFHOBackend<DIM, Q1D>>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline
typename DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
AssembleKernelType
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeAssembleHO::Fallback(int dim, int q1d)
{
using assemble_t =
DerivativeAssemble<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeAssembleHO = typename assemble_t::DerivativeAssembleHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<DerivativeAssembleHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<DerivativeAssembleHO, 3, 8>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,540 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "kernels.hpp"
#include "util.hpp"
#include <array>
namespace mfem::future::LocalQFImpl
{
// Assemble diagonal of cached Jacobian (square trial == test, tensor 2D/3D)
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
class DerivativeAssembleDiagonal
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
const IntegratorContext ctx;
const Vector &qp_cache;
inputs_t inputs;
outputs_t outputs;
const bool use_sum_factorization;
const std::vector<const DofToQuad *> dtqs;
const std::array<DofToQuadMap, n_inputs> input_dtq_maps;
const std::array<DofToQuadMap, n_outputs> output_dtq_maps;
const std::array<bool, n_inputs> input_is_dependent;
const size_t trial_field_uf;
const size_t test_field_uf;
const bool is_square;
const int test_vdim;
const std::array<int, n_outputs> out_vdim;
const std::array<int, n_outputs> out_op_dim;
const std::array<int, n_outputs> out_offsets;
const int output_size_on_qp;
const int num_test_dof;
const int num_test_dof_1d;
const int trial_vdim;
const int total_trial_op_dim;
const int num_trial_dof_1d;
const int residual_size_on_qp;
const int dim, ne, nq, q1d;
const std::array<int, n_inputs> inputs_trial_op_dim;
mutable Vector Ye_mem;
public:
DerivativeAssembleDiagonal() = delete;
DerivativeAssembleDiagonal(IntegratorContext ctx_in,
qfunc_t /*qfunc*/,
inputs_t inputs_in,
outputs_t outputs_in,
const Vector &qp_cache_in):
ctx(ctx_in), qp_cache(qp_cache_in), inputs(inputs_in),
outputs(outputs_in), use_sum_factorization(
[&]
{
const Element::Type etype =
Element::TypeFromGeometry(ctx_in.mesh.GetTypicalElementGeometry());
return (etype == Element::QUADRILATERAL || etype == Element::HEXAHEDRON);
}()),
dtqs(
[&]
{
const DofToQuad::Mode dtq_mode = use_sum_factorization
? DofToQuad::Mode::TENSOR
: DofToQuad::Mode::FULL;
std::vector<const DofToQuad *> maps;
maps.reserve(ctx_in.unionfds.size());
for (const auto &field : ctx_in.unionfds)
{
maps.emplace_back(
GetDofToQuad<Entity::Element>(field, ctx_in.ir, dtq_mode));
}
return maps;
}()),
input_dtq_maps(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx_in, inputs),
ctx_in.unionfds,
ctx_in.ir)),
output_dtq_maps(create_dtq_maps<Entity::Element>(
outputs,
dtqs,
create_union_field_map_for_dtq(ctx_in, outputs),
ctx_in.unionfds,
ctx_in.ir)),
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
trial_field_uf(find_union_field_index(ctx_in, derivative_id)),
test_field_uf(
find_union_field_index(ctx_in, get<0>(outputs).GetFieldId())),
is_square(
[&]
{
const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
&ctx_in.unionfds[test_field_uf].data);
const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
&ctx_in.unionfds[trial_field_uf].data);
return test_fes && trial_fes && *test_fes && *trial_fes &&
(*test_fes == *trial_fes);
}()),
test_vdim(get<0>(outputs).vdim),
out_vdim(get_vdim(outputs_in)),
out_op_dim(compute_out_op_dim(outputs_in)),
out_offsets(compute_out_offsets(out_vdim, out_op_dim)),
output_size_on_qp(
[&]
{
int s = 0;
for_constexpr<n_outputs>([&](auto o)
{ s += get<o>(outputs_in).size_on_qp; });
return s;
}()), num_test_dof(
[&]
{
const auto *test_fes = std::get_if<const ParFiniteElementSpace *>(
&ctx_in.unionfds[test_field_uf].data);
MFEM_ASSERT(test_fes != nullptr && *test_fes != nullptr,
"LocalQFBackend: test space is not a ParFiniteElementSpace");
return (*test_fes)->GetFE(0)->GetDof();
}()),
num_test_dof_1d(tensor_1d_size(num_test_dof, ctx_in.mesh.Dimension())),
trial_vdim(compute_trial_vdim(inputs, derivative_id)), total_trial_op_dim(
[&]
{
const auto input_size_on_qp =
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs>{});
return compute_total_trial_op_dim(
inputs, input_is_dependent, input_size_on_qp);
}()),
num_trial_dof_1d(
[&]
{
const auto *trial_fes = std::get_if<const ParFiniteElementSpace *>(
&ctx_in.unionfds[trial_field_uf].data);
MFEM_ASSERT(trial_fes != nullptr && *trial_fes != nullptr,
"LocalQFBackend: trial space is not a ParFiniteElementSpace");
const int num_trial_dof = (*trial_fes)->GetFE(0)->GetDof();
return tensor_1d_size(num_trial_dof, ctx_in.mesh.Dimension());
}()),
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
dim(ctx_in.mesh.Dimension()), ne(ctx_in.nentities),
nq(ctx_in.ir.GetNPoints()), q1d(tensor_1d_size(nq, dim)),
inputs_trial_op_dim(
[&]
{
std::array<int, n_inputs> itod{};
for_constexpr<n_inputs>([&](auto i)
{
itod[i] = input_is_dependent[i]
? get<i>(inputs).size_on_qp / get<i>(inputs).vdim
: 0;
});
return itod;
}()),
Ye_mem()
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
MFEM_ASSERT(
trial_field_uf != SIZE_MAX,
"DerivativeAssembleDiagonal: trial field not found in unionfds");
MFEM_ASSERT(
test_field_uf != SIZE_MAX,
"DerivativeAssembleDiagonal: test field not found in unionfds");
MFEM_ASSERT(trial_vdim > 0,
"LocalQFBackend: could not determine trial vdim");
MFEM_ASSERT(total_trial_op_dim > 0,
"LocalQFBackend: no dependent inputs found");
for_constexpr<n_outputs>([&](auto o)
{
MFEM_CONTRACT_VAR(o);
MFEM_ASSERT(out_vdim[o] == test_vdim,
"DerivativeAssembleDiagonal: all outputs must share the "
"test field vdim");
});
if (is_square)
{
Ye_mem.SetSize(num_test_dof * test_vdim * ne);
Ye_mem.UseDevice(true);
}
}
template<typename Backend>
void run_kernels() const
{
Backend::Run(dim,
q1d,
ctx,
qp_cache,
Ye_mem,
inputs,
outputs,
output_dtq_maps,
input_dtq_maps,
test_vdim,
out_op_dim,
out_offsets,
output_size_on_qp,
num_test_dof,
num_test_dof_1d,
trial_vdim,
total_trial_op_dim,
residual_size_on_qp,
inputs_trial_op_dim,
nq,
ne,
q1d,
dim);
}
void operator()(Vector &diag_e) const
{
if (!is_square) { return; }
if (ctx.attr.Size() == 0) { return; }
if (!(use_sum_factorization && (dim == 2 || dim == 3)))
{
MFEM_ABORT("DerivativeAssembleDiagonal optimized path is implemented "
"for tensor-product 2D/3D elements only");
}
MFEM_VERIFY(num_test_dof_1d == num_trial_dof_1d,
"DerivativeAssembleDiagonal requires matching tensor dofs");
MFEM_VERIFY(num_test_dof_1d <= DeviceDofQuadLimits::Get().MAX_D1D, "");
MFEM_VERIFY(q1d <= DeviceDofQuadLimits::Get().MAX_Q1D, "");
Ye_mem = 0.0;
if (q1d <= LocalQFLOBackendMQ1())
{
run_kernels<DerivativeAssembleDiagonalLO>();
}
else if (q1d <= LocalQFHOBackendMQ1())
{
run_kernels<DerivativeAssembleDiagonalHO>();
}
else
{
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
}
diag_e += Ye_mem;
}
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
static void derivative_assemble_diagonal_callback(
const IntegratorContext &ctx,
const Vector &qp_cache,
Vector &Ye_mem,
const inputs_t &inputs,
const outputs_t &outputs,
const std::array<DofToQuadMap, n_outputs> &output_dtq_maps,
const std::array<DofToQuadMap, n_inputs> &input_dtq_maps,
const int test_vdim,
const std::array<int, n_outputs> &out_op_dim,
const std::array<int, n_outputs> &out_offsets,
const int output_size_on_qp,
const int num_test_dof,
const int num_test_dof_1d,
const int trial_vdim,
const int total_trial_op_dim,
const int residual_size_on_qp,
const std::array<int, n_inputs> &inputs_trial_op_dim,
const int nq,
const int ne,
const int q1d,
const int dim)
{
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
if (ctx.attr.Size() == 0) { return; }
static constexpr bool B2D = backend_t::DIM == 2;
static constexpr int MTPB = backend_t::MAX_THREADS_PER_BLOCK();
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
auto cache_tensor = DeviceTensor<3, const real_t>(
qp_cache.Read(), nq, residual_size_on_qp, ne);
const int num_dofs_per_elem = num_test_dof * test_vdim;
auto Ye = Reshape(Ye_mem.ReadWrite(), num_dofs_per_elem, ne);
dfem::forall<MTPB>(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
// The cache is written with the quadrature index fastest, then the
// trial op index, then the (test vdim, test op) rows of all outputs
// stacked via out_offsets.
auto qpdc = Reshape(&cache_tensor(0, 0, e),
nq,
total_trial_op_dim,
trial_vdim,
output_size_on_qp);
// Backend-owned shared scratch for the sum-factorized contraction.
MFEM_SHARED typename backend_t::Shared s_diag;
const int nz_dof = B2D ? 1 : num_test_dof_1d;
for (int vd = 0; vd < test_vdim; vd++)
{
auto Y = Reshape(&Ye(vd * num_test_dof, e),
num_test_dof_1d,
num_test_dof_1d,
nz_dof);
MFEM_FOREACH_THREAD(dz_t, z, nz_dof)
{
MFEM_FOREACH_THREAD_DIRECT(dy_t, y, num_test_dof_1d)
{
MFEM_FOREACH_THREAD_DIRECT(dx_t, x, num_test_dof_1d)
{ Y(dx_t, dy_t, dz_t) = 0.0; }
}
}
MFEM_SYNC_THREAD;
// Accumulate every (output o, test op k, dependent input s,
// trial op m) block of the cached Jacobian into the diagonal via
// the backend driver.
for_constexpr<n_outputs>([&](auto o)
{
using test_fop_t = std::decay_t<decltype(get<o>(outputs))>;
const auto &out_dtq = output_dtq_maps[o];
const int test_op_dim = out_op_dim[static_cast<int>(o)];
// Test-basis factor along a spatial axis
const auto eval_test =
[&](const int k, const int axis, const int q, const int d)
{
const auto &B = out_dtq.B;
const auto &G = out_dtq.G;
if constexpr (is_value_fop<test_fop_t>::value)
{
return (k == 0) ? B(q, 0, d) : 0.0;
}
else if constexpr (is_gradient_fop<test_fop_t>::value)
{
return (k == axis) ? G(q, 0, d) : B(q, 0, d);
}
else
{
return 0.0;
}
};
for (int k = 0; k < test_op_dim; k++)
{
const int row =
out_offsets[static_cast<int>(o)] + vd * test_op_dim + k;
int m_offset = 0;
for_constexpr<n_inputs>([&](auto s)
{
using fop_t = std::decay_t<decltype(get<s>(inputs))>;
const int trial_op_dim =
inputs_trial_op_dim[static_cast<int>(s)];
if (trial_op_dim == 0) { return; }
const auto &in_dtq = input_dtq_maps[s];
const auto eval_input =
[&](const int m, const int axis, const int q,
const int d)
{
if constexpr (is_value_fop<fop_t>::value)
{
return (m == 0) ? in_dtq.B(q, 0, d) : 0.0;
}
else if constexpr (is_gradient_fop<fop_t>::value)
{
return (m == axis) ? in_dtq.G(q, 0, d)
: in_dtq.B(q, 0, d);
}
else
{
return 0.0;
}
};
for (int m = 0; m < trial_op_dim; m++)
{
const int col = m_offset + m;
backend_t::DiagContract(
s_diag,
num_test_dof_1d,
q1d,
nz_dof,
[&](int axis, int q, int d)
{ return eval_test(k, axis, q, d); },
[&](int axis, int q, int d)
{ return eval_input(m, axis, q, d); },
[&](int q) { return qpdc(q, col, vd, row); },
[&](int dx, int dy, int dz, real_t u)
{ Y(dx, dy, dz) += u; });
}
m_offset += trial_op_dim;
});
}
});
}
},
ne,
backend_t::thread_blocks(std::max(q1d, num_test_dof_1d)),
0,
nullptr);
}
using DiagonalKernelType =
decltype(&DerivativeAssembleDiagonal::
derivative_assemble_diagonal_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalLO,
DiagonalKernelType,
(int, int) );
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeAssembleDiagonalHO,
DiagonalKernelType,
(int, int) );
};
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeAssembleDiagonal<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::DiagonalKernelType
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeAssembleDiagonalLO::Kernel()
{
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
using diag_t =
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
return diag_t::template derivative_assemble_diagonal_callback<
LocalQFLOBackend<DIM, Q1D>>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeAssembleDiagonal<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::DiagonalKernelType
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeAssembleDiagonalLO::Fallback(int dim, int q1d)
{
using diag_t =
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeAssembleDiagonalLO =
typename diag_t::DerivativeAssembleDiagonalLO;
if (dim == 2)
{
return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchLOKernelByQ1D<DerivativeAssembleDiagonalLO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeAssembleDiagonal<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::DiagonalKernelType
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeAssembleDiagonalHO::Kernel()
{
using diag_t =
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
return diag_t::template derivative_assemble_diagonal_callback<
LocalQFHOBackend<DIM>,
Q1D>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeAssembleDiagonal<derivative_id,
qfunc_t,
inputs_t,
outputs_t>::DiagonalKernelType
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeAssembleDiagonalHO::Fallback(int dim, int q1d)
{
using diag_t =
DerivativeAssembleDiagonal<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeAssembleDiagonalHO =
typename diag_t::DerivativeAssembleDiagonalHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<DerivativeAssembleDiagonalHO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,600 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "kernels.hpp"
#include "util.hpp"
#include <array>
namespace mfem::future::LocalQFImpl
{
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
class DerivativeSetup
{
static constexpr auto inout_tuple =
merge_mfem_tuples_as_empty_std_tuple(inputs_t {}, outputs_t{});
static constexpr auto filtered_inout_tuple = filter_fields(inout_tuple);
static constexpr size_t nfields =
count_unique_field_ids(filtered_inout_tuple);
using qf_signature = typename get_function_signature<qfunc_t>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
using args_tuple_t = decay_tuple<qf_param_ts>;
static constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
static constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
static_assert(n_inputs + n_outputs == tuple_size<qf_param_ts>::value,
"LocalQF: q-function arity must match inputs + outputs");
qfunc_t qfunc;
const inputs_t inputs;
const outputs_t outputs;
const IntegratorContext ctx;
Vector &qp_cache;
const std::vector<const DofToQuad *> dtqs;
// inputs: dtq, idx, B, G, d1d, q1d, vdim
const std::array<DofToQuadMap, n_inputs> input_dtq;
const std::array<size_t, n_inputs> input_idx;
const std::array<const real_t *, n_inputs> input_B, input_G;
const std::array<int, n_inputs> input_d1d, input_q1d, input_vdim;
// Jacobian cache metadata
const std::array<bool, n_inputs> input_is_dependent;
const std::array<int, n_inputs> input_size_on_qp;
const std::array<int, n_outputs> out_vdim;
const std::array<int, n_outputs> out_op_dim;
const std::array<int, n_outputs> out_offsets;
const int output_size_on_qp;
const int trial_vdim;
const int total_trial_op_dim;
const int residual_size_on_qp;
// other constants
const int dim, ne, nq, q1d;
public:
//////////////////////////////////////////////////////////////////
DerivativeSetup() = delete;
DerivativeSetup(IntegratorContext ctx,
qfunc_t qfunc,
inputs_t inputs,
outputs_t outputs,
Vector &qp_cache):
qfunc(std::move(qfunc)), inputs(inputs), outputs(outputs), ctx(ctx),
qp_cache(qp_cache), dtqs(make_dtqs(ctx)),
input_dtq(create_dtq_maps<Entity::Element>(
inputs,
dtqs,
create_union_field_map_for_dtq(ctx, inputs),
ctx.unionfds,
ctx.ir)),
input_idx(create_input_vector_map(ctx, inputs)),
input_B(get_B(input_dtq)), input_G(get_G(input_dtq)),
input_d1d(get_D1D(input_dtq)), input_q1d(get_Q1D(input_dtq)),
input_vdim(get_vdim(inputs)),
input_is_dependent(compute_input_is_dependent(inputs, derivative_id)),
input_size_on_qp(
get_input_size_on_qp(inputs, std::make_index_sequence<n_inputs> {})),
out_vdim(get_vdim(outputs)), out_op_dim(compute_out_op_dim(outputs)),
out_offsets(compute_out_offsets(out_vdim, out_op_dim)), output_size_on_qp(
[&]
{
int s = 0;
for_constexpr<n_outputs>([&](auto o)
{ s += get<o>(outputs).size_on_qp; });
return s;
}()),
trial_vdim(compute_trial_vdim(inputs, derivative_id)),
total_trial_op_dim(compute_total_trial_op_dim(
inputs, input_is_dependent, input_size_on_qp)),
residual_size_on_qp(output_size_on_qp * trial_vdim * total_trial_op_dim),
dim(ctx.mesh.Dimension()), ne(ctx.nentities), nq(ctx.ir.GetNPoints()),
q1d(tensor_1d_size(nq, dim))
{
MFEM_ASSERT(ctx.unionfds.size() == nfields,
"LocalQFBackend: unionfds size mismatch");
qp_cache.SetSize(ne * nq * residual_size_on_qp);
qp_cache.UseDevice(true);
}
//////////////////////////////////////////////////////////////////
void operator()(const std::vector<Vector *> &xe)
{
if (ctx.attr.Size() == 0) { return; }
// Quadrature index is fastest-varying so that adjacent threads (one per
// quadrature point) touch adjacent addresses.
auto cache_tensor = DeviceTensor<3, real_t>(
qp_cache.ReadWrite(), nq, residual_size_on_qp, ne);
if (q1d <= LocalQFLOBackendMQ1())
{
run_kernels<DerivativeSetupLO>(xe, cache_tensor);
}
else if (q1d <= LocalQFHOBackendMQ1())
{
run_kernels<DerivativeSetupHO>(xe, cache_tensor);
}
else
{
MFEM_ABORT("Unsupported quadrature order for LocalQF backend");
}
}
//////////////////////////////////////////////////////////////////
template<typename Backend>
void run_kernels(const std::vector<Vector *> &xe,
DeviceTensor<3, real_t> &cache_tensor)
{
Backend::Run(dim,
q1d,
ctx,
qfunc,
// inputs
input_idx,
input_B,
input_G,
input_vdim,
input_d1d,
input_q1d,
input_size_on_qp,
input_is_dependent,
// outputs / cache metadata
out_vdim,
out_op_dim,
out_offsets,
trial_vdim,
total_trial_op_dim,
residual_size_on_qp,
// vectors
xe,
cache_tensor,
// fallback arguments
dim,
q1d);
}
//////////////////////////////////////////////////////////////////
/// Zeroes the q-function *output* slots of an argument tuple.
///
/// The argument tuple is built once per quadrature point and reused for
/// every trial seed. The q-function writes through its output parameters,
/// so those slots have to be restored before each call to give every seed
/// the same starting state a freshly value-initialized tuple would.
static MFEM_HOST_DEVICE inline void reset_output_args(args_tuple_t &args)
{
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t ao = n_inputs + oc.value;
auto &oarg = get<ao>(args);
oarg = std::remove_reference_t<decltype(oarg)> {};
});
}
//////////////////////////////////////////////////////////////////
template<typename backend_t = LocalQFLOBackend<3>, int T_Q1D = 0>
static void
derivative_setup_callback(const IntegratorContext &ctx,
qfunc_t &qfunc,
// inputs: idx, B, G, vdim, d1d, q1d
const std::array<size_t, n_inputs> &in_idx,
const std::array<const real_t *, n_inputs> in_B,
const std::array<const real_t *, n_inputs> in_G,
const std::array<int, n_inputs> &in_vdim,
const std::array<int, n_inputs> &in_d1d,
const std::array<int, n_inputs> &in_q1d,
const std::array<int, n_inputs> &in_size_on_qp,
const std::array<bool, n_inputs> &input_dep,
// outputs / cache metadata
const std::array<int, n_outputs> &out_vdim,
const std::array<int, n_outputs> &out_op_dim,
const std::array<int, n_outputs> &out_offsets,
const int trial_vdim,
const int total_trial_op_dim,
const int residual_size_on_qp,
const std::vector<Vector *> &xe,
DeviceTensor<3, real_t> &cache_tensor,
// fallback arguments
const int dim,
const int q1d)
{
MFEM_VERIFY(dim == ctx.mesh.Dimension(), "Dimension mismatch");
if (ctx.attr.Size() == 0) { return; }
static constexpr auto B2D = backend_t::DIM == 2;
static constexpr auto MQ1 = T_Q1D ? T_Q1D : backend_t::MQ1;
static constexpr auto MTPB = backend_t::MAX_THREADS_PER_BLOCK();
const int ne = ctx.nentities;
MFEM_CONTRACT_VAR(residual_size_on_qp);
constexpr auto k_dim = [](const int k) { return k * k * (B2D ? 1 : k); };
// --------------------------------------------------
// INPUTS: XE, 3(max DIM) + 1(VDIM) + 1(number of elements)
// --------------------------------------------------
std::array<DeviceTensor<3 + 1 + 1, const real_t>, n_inputs> in_XE;
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const size_t k = in_idx[i];
const int d = in_d1d[i], q = in_q1d[i], v = in_vdim[i];
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
MFEM_VERIFY(xe[k]->Size() == k_dim(d) * v * ne, "Size mismatch");
in_XE[i] = Reshape(xe[k]->Read(), d, d, B2D ? 1 : d, v, ne);
}
else if constexpr (is_identity_fop_v<FOP>)
{
MFEM_VERIFY(xe[k]->Size() == k_dim(q) * v * ne, "Size mismatch");
in_XE[i] = Reshape(xe[k]->Read(), v, q, q, B2D ? 1 : q, ne);
}
else if constexpr (is_weight_fop_v<FOP>)
{
MFEM_VERIFY(ctx.ir.GetNPoints() == k_dim(q1d),
"tensor-product IR expected");
in_XE[i] = Reshape(
ctx.ir.GetWeights().Read(), q1d, q1d, B2D ? 1 : q1d, 1, 1);
}
else
{
static_assert(false, "Unsupported");
}
});
const auto d_attr = ctx.attr.Read();
const bool has_attr = ctx.attr.Size() > 0;
const auto d_elem_attr = ctx.elem_attr->Read();
dfem::forall<MTPB>(
[=] MFEM_HOST_DEVICE(const int e, void *)
{
if (has_attr && !d_attr[d_elem_attr[e] - 1]) { return; }
// -----------------------------------------------
// Inputs argument registers + shared memory
// -----------------------------------------------
input_args_reg_t<backend_t, qfunc_t, inputs_t, outputs_t, MQ1> rargs;
MFEM_SHARED typename backend_t::Shared smem;
// -----------------------------------------------
// Load primal inputs (rargs) once for this element
// -----------------------------------------------
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
const auto &XE = in_XE[i];
const int d = in_d1d[i], q = in_q1d[i], Q1D = q1d;
const real_t *B = in_B[i], *G = in_G[i];
auto &rarg = get<i>(rargs);
using XE_t = decltype(XE);
using rarg_t = decltype(rarg);
using FOP = tuple_element_t<i, inputs_t>;
if constexpr (is_value_fop<FOP>::value)
{
backend_t::template LoadValue<rarg_t, XE_t>(
smem, e, d, q, Q1D, B, XE, rarg);
}
else if constexpr (is_gradient_fop_v<FOP>)
{
constexpr auto RNK = qf_param_slot<qfunc_t, i>::extents.size();
using FieldParamT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
backend_t::template LoadGradient<RNK, rarg_t, XE_t, FieldParamT>(
smem, e, d, q, q1d, B, G, XE, rarg);
}
else if constexpr (is_weight_fop_v<FOP> || is_identity_fop_v<FOP>)
{
// qp values are read directly from in_XE / IR
}
else
{
static_assert(false, "Unsupported");
}
});
MFEM_SYNC_THREAD;
// -----------------------------------------------
// Build the primal arguments once per quadrature point, then, for
// each trial seed (j, dependent input s, m), differentiate the
// q-function with a unit tangent and store the result row in the
// cache. Nothing in the primal pull depends on the seed, so the
// thread loop is the outermost one here. The seed loops only touch
// per-thread state, hence no barrier inside them.
// Warning: no 'DIRECT' on the 'Z' direction,
// as one backend may need to iterate over it.
// -----------------------------------------------
MFEM_FOREACH_THREAD(qz, z, (B2D ? 1 : q1d))
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
#ifdef MFEM_USE_ENZYME
args_tuple_t primal_args {};
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
auto &parg = get<i>(primal_args);
const auto &XE = in_XE[i];
using FOP = tuple_element_t<i, inputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
parg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
}
else if constexpr (is_weight_fop_v<FOP>)
{
parg = XE(qx, qy, qz, 0, 0);
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
parg = backend_t::template qp_pull<ARG>(
get<i>(rargs), qx, qy, qz);
}
else
{
static_assert(false, "Unsupported");
}
});
for (int j = 0; j < trial_vdim; j++)
{
int m_offset = 0;
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if (!input_dep[s]) { return; }
const int vdim_s = in_vdim[s];
const int op_dim_s = in_size_on_qp[s] / vdim_s;
for (int m = 0; m < op_dim_s; m++)
{
const int col_m = m + m_offset;
// Enzyme writes through the output slots of the
// primal tuple, so they are reset per seed.
reset_output_args(primal_args);
args_tuple_t shadow_args {};
qf_set_value_at(get<s>(shadow_args), j, m, 1.0);
call_enzyme_fwddiff(qfunc, primal_args, shadow_args);
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value, ao = n_inputs + o;
const auto &tangent = get<ao>(shadow_args);
const int tv = out_vdim[o], to = out_op_dim[o];
for (int i = 0; i < tv; i++)
{
for (int k = 0; k < to; k++)
{
const int row = out_offsets[o] + i * to + k;
const int cache_idx =
row * trial_vdim * total_trial_op_dim +
j * total_trial_op_dim + col_m;
cache_tensor(q, cache_idx, e) =
qf_value_at(tangent, i, k);
}
}
});
}
m_offset += op_dim_s;
});
}
#else // MFEM_USE_ENZYME
args_tuple_t qargs {};
for_constexpr<n_inputs>([&](auto ic)
{
constexpr size_t i = ic.value;
auto &qarg = get<i>(qargs);
const auto &XE = in_XE[i];
using FOP = tuple_element_t<i, inputs_t>;
using ARG =
typename qf_param_slot<qfunc_t, i>::qf_reg_param_t;
if constexpr (is_identity_fop_v<FOP>)
{
using DT =
typename qf_param_slot<qfunc_t, i>::qf_decay_param_t;
if constexpr (qf_param_uses_dual_v<DT>)
{
qarg = backend_t::template identity_qp_pull_dual<DT>(
false, XE, XE, qx, qy, qz, e);
}
else
{
qarg = as_tensor<ARG>(&XE(0, qx, qy, qz, e));
}
}
else if constexpr (is_weight_fop_v<FOP>)
{
qarg = XE(qx, qy, qz, 0, 0);
}
else if constexpr (is_value_fop_v<FOP> ||
is_gradient_fop_v<FOP>)
{
qarg = backend_t::template qp_pull<ARG>(
get<i>(rargs), qx, qy, qz);
}
else
{
static_assert(false, "Unsupported");
}
});
for (int j = 0; j < trial_vdim; j++)
{
int m_offset = 0;
for_constexpr<n_inputs>([&](auto sc)
{
constexpr size_t s = sc.value;
if (!input_dep[s]) { return; }
const int vdim_s = in_vdim[s];
const int op_dim_s = in_size_on_qp[s] / vdim_s;
for (int m = 0; m < op_dim_s; m++)
{
const int col_m = m + m_offset;
// The q-function writes through the output slots,
// so they are reset per seed.
reset_output_args(qargs);
qf_set_gradient_at(get<s>(qargs), j, m, 1.0);
call_qfunc_no_move(qfunc, qargs);
for_constexpr<n_outputs>([&](auto oc)
{
constexpr size_t o = oc.value, ao = n_inputs + o;
const auto &tangent = get<ao>(qargs);
const int tv = out_vdim[o], to = out_op_dim[o];
for (int i = 0; i < tv; i++)
{
for (int k = 0; k < to; k++)
{
const int row = out_offsets[o] + i * to + k;
const int cache_idx =
row * trial_vdim * total_trial_op_dim +
j * total_trial_op_dim + col_m;
cache_tensor(q, cache_idx, e) =
qf_gradient_at(tangent, i, k);
}
}
});
// Clear the seed so the next direction starts from
// the pristine (zero-tangent) primal state.
qf_set_gradient_at(get<s>(qargs), j, m, 0.0);
}
m_offset += op_dim_s;
});
}
#endif // MFEM_USE_ENZYME
}
}
}
},
ne,
backend_t::thread_blocks(
compute_kernel_thread_1d<inputs_t>(q1d, in_d1d)),
0,
nullptr);
}
using SetupKernelType =
decltype(&DerivativeSetup::derivative_setup_callback<>);
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupLO,
SetupKernelType,
(int, int) );
MFEM_REGISTER_KERNELS_HEADER_ONLY(DerivativeSetupHO,
SetupKernelType,
(int, int) );
};
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
SetupKernelType
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeSetupLO::Kernel()
{
static_assert((DIM == 2 || DIM == 3) && Q1D <= 8);
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
return setup_t::template derivative_setup_callback<
LocalQFLOBackend<DIM, Q1D>>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
SetupKernelType
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeSetupLO::Fallback(int dim, int q1d)
{
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeSetupLO = typename setup_t::DerivativeSetupLO;
if (dim == 2)
{
return DispatchLOKernelByQ1D<DerivativeSetupLO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchLOKernelByQ1D<DerivativeSetupLO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
template<int DIM, int Q1D>
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
SetupKernelType
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeSetupHO::Kernel()
{
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
return setup_t::template derivative_setup_callback<LocalQFHOBackend<DIM>,
Q1D>;
}
template<int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
inline typename DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
SetupKernelType
DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>::
DerivativeSetupHO::Fallback(int dim, int q1d)
{
using setup_t = DerivativeSetup<derivative_id, qfunc_t, inputs_t, outputs_t>;
using DerivativeSetupHO = typename setup_t::DerivativeSetupHO;
if (dim == 2)
{
return DispatchHOKernelByQ1D<DerivativeSetupHO, 2>(q1d);
}
else if (dim == 3)
{
return DispatchHOKernelByQ1D<DerivativeSetupHO, 3>(q1d);
}
else
{
MFEM_ABORT("Unsupported dimension");
return nullptr;
}
}
} // namespace mfem::future::LocalQFImpl
@@ -1,274 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../util.hpp"
#ifdef MFEM_USE_ENZYME
namespace mfem::future
{
template <typename T> struct function_traits;
template <typename R, typename C, typename... Args>
struct function_traits<R (C::*)(Args...) const>
{
using primal_return_type = R;
using args_tuple = std::tuple<Args...>;
static constexpr size_t arity = sizeof...(Args);
};
// Component count and writable counterpart of a per-point argument, which
// is either a tensor or a plain scalar.
template <typename Arg> struct qp_traits
{
static_assert(std::is_arithmetic_v<Arg>,
"per-point arguments must be tensors or scalars");
using view_type = Arg;
static constexpr int components = 1;
};
template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
{
using view_type = tensor<std::remove_const_t<T>, Sizes...>;
static constexpr int components = (Sizes * ... * 1);
};
// Generic FwdDiff: computes the full gradient of a pointwise qfunction at a
// single quadrature point.
//
// active_input is the index of the argument to differentiate with respect
// to; active_output is the index of the (scalar) output argument whose
// derivative is taken.
//
// operator()(args...) takes the qfunction's arguments, except that the
// active output position receives the *gradient*, shaped like the active
// input (d(output)/d(input component)).
//
// Per input component d, one enzyme fwddiff call with the one-hot seed e_d
// in the input tangent yields gradient entry d, i.e. grad_components enzyme
// calls per point. The output tangent is not pre-zeroed, so the qfunction
// must fully write its output (tangent stores overwrite).
//
// operator() is MFEM_HOST_DEVICE, stateless and allocation-free, so it is
// callable inside a GPU kernel: seed, primal scratch and tangent all live
// on the stack.
template <typename Func, size_t active_input, size_t active_output>
struct FwdDiff
{
using traits = function_traits<decltype(&Func::operator())>;
using args_tuple = typename traits::args_tuple;
static constexpr size_t arity = traits::arity;
static_assert(std::is_void_v<typename traits::primal_return_type>,
"FwdDiff only supports primal functions with void return type");
static_assert(active_input < arity && active_output < arity,
"active argument indices must be within the function arity");
static_assert(active_input != active_output,
"active input and output must be different arguments");
using input_type =
std::decay_t<std::tuple_element_t<active_input, args_tuple>>;
using output_type =
std::decay_t<std::tuple_element_t<active_output, args_tuple>>;
using grad_type = typename qp_traits<input_type>::view_type;
using output_view = typename qp_traits<output_type>::view_type;
static constexpr int grad_components = qp_traits<input_type>::components;
static_assert(qp_traits<output_type>::components == 1,
"gradient output requires a scalar output");
// Signature of the differentiated qfunction: the primal arguments, with
// the Active output slot receiving the (writable) gradient instead.
// Exposed through create_function_signature below so that
// DifferentiableOperator can deduce the parameter types, which it cannot
// do from the variadic operator().
template <size_t I>
using qf_arg_t = std::conditional_t<I == active_output, grad_type &,
std::tuple_element_t<I, args_tuple>>;
template <size_t... Is>
static FunctionSignature<void(qf_arg_t<Is>...)>
signature_impl(std::index_sequence<Is...>);
using signature =
decltype(signature_impl(std::make_index_sequence<arity> {}));
// d-th scalar of a per-point argument in flat row-major order, regardless
// of rank, built on the native operator[] (tensor has no flat-index
// accessor; flatten() returns a copy, so it cannot be written through).
MFEM_HOST_DEVICE static double &component(double &t, int) { return t; }
template <typename T, int n0, int... n>
MFEM_HOST_DEVICE static T &component(tensor<T, n0, n...> &t, int d)
{
if constexpr (sizeof...(n) == 0)
{
return t[d];
}
else
{
constexpr int stride = (n * ... * 1);
return component(t[d / stride], d % stride);
}
}
// Plain function with the qfunction's exact (reference) signature, so it
// can be handed to Enzyme as a function pointer; references are pointers
// to Enzyme, so primal arguments and shadows are passed by address below.
template <size_t... Is>
MFEM_HOST_DEVICE static void
static_call(std::tuple_element_t<Is, args_tuple>... args)
{
Func{}(args...);
}
template <size_t... Is>
static constexpr auto fn_ptr(std::index_sequence<Is...>)
{
return &static_call<Is...>;
}
static constexpr auto fn = fn_ptr(std::make_index_sequence<arity> {});
// Writable, zero-initialized scratch with the shape of argument I, used
// as its enzyme shadow.
template <size_t I>
using shadow_t = typename qp_traits<
std::decay_t<std::tuple_element_t<I, args_tuple>>>::view_type;
template <size_t... Is>
MFEM_HOST_DEVICE static auto make_shadows(std::index_sequence<Is...>)
{
return mfem::future::make_tuple(shadow_t<Is> {}...);
}
template <typename Shadows, size_t... Is>
MFEM_HOST_DEVICE static auto make_shadow_ptrs(Shadows &shadows,
std::index_sequence<Is...>)
{
return mfem::future::make_tuple(&mfem::future::get<int(Is)>(shadows)...);
}
// The caller's argument pointers, except the active output slot, which
// points to scalar scratch: the caller's slot holds the gradient, while
// the primal function writes its scalar output there.
template <size_t I, typename Ptrs>
MFEM_HOST_DEVICE static auto primal_ptr(Ptrs &ptrs, output_view &primal)
{
if constexpr (I == active_output) { return &primal; }
else { return mfem::future::get<int(I)>(ptrs); }
}
template <typename Ptrs, size_t... Is>
MFEM_HOST_DEVICE static auto make_primal_ptrs(Ptrs &ptrs,
output_view &primal,
std::index_sequence<Is...>)
{
return mfem::future::make_tuple(primal_ptr<Is>(ptrs, primal)...);
}
// Single flat enzyme call. The activity markers must appear directly in
// the __enzyme_fwddiff argument list — Enzyme cannot trace markers that
// were forwarded through function parameters (e.g. at -O0, where nothing
// is inlined). Every argument is therefore enzyme_dup'd in one sticky
// group; Const arguments simply carry a zero tangent, which is equivalent
// to marking them enzyme_const.
//
// always_inline is load-bearing: when FwdDiff is itself differentiated
// (second derivatives, forward-over-forward), Enzyme only recognizes this
// nested __enzyme_fwddiff call if it sits at most one call level below
// the function handed to the outer __enzyme_fwddiff. Without inlining
// (-O0) it sits two levels down (wrapper -> operator() -> call_enzyme)
// and the outer pass treats it as a regular call: the activity marker
// ints then receive undef shadows, which misaligns the argument pairing
// (observed as "cannot compute with global variable that doesn't have
// marked shadow global" at compile time or null-shadow segfaults at
// runtime). The always-inliner runs even at -O0, hoisting this call into
// operator() where the nested handling applies.
template <typename PrimalPtrs, typename ShadowPtrs, size_t... Is>
__attribute__((always_inline))
MFEM_HOST_DEVICE static void call_enzyme(PrimalPtrs &primal_ptrs,
ShadowPtrs &shadow_ptrs,
std::index_sequence<Is...>)
{
__enzyme_fwddiff<void>(fn, enzyme_dup,
mfem::future::get<int(Is)>(primal_ptrs)...,
enzyme_interleave,
mfem::future::get<int(Is)>(shadow_ptrs)...,
enzyme_runtime_activity);
}
template <typename... Args>
MFEM_HOST_DEVICE void operator()(Args &&...args) const
{
static_assert(sizeof...(Args) == arity, "Wrong number of arguments");
auto ptrs = mfem::future::make_tuple(&args...);
auto &grad = *mfem::future::get<int(active_output)>(ptrs);
static_assert(std::is_same_v<std::decay_t<decltype(grad)>, grad_type>,
"gradient argument must be shaped like the Active input "
"(with writable scalars)");
constexpr auto seq = std::make_index_sequence<arity> {};
output_view primal{};
auto primal_ptrs = make_primal_ptrs(ptrs, primal, seq);
auto shadows = make_shadows(seq);
auto shadow_ptrs = make_shadow_ptrs(shadows, seq);
auto &seed = mfem::future::get<int(active_input)>(shadows);
auto &tangent = mfem::future::get<int(active_output)>(shadows);
// One enzyme call per input component d: seed e_d in the input tangent
// and read gradient entry d off the output tangent.
for (int d = 0; d < grad_components; d++)
{
component(seed, d) = 1.0;
call_enzyme(primal_ptrs, shadow_ptrs, seq);
component(grad, d) = component(tangent, 0);
component(seed, d) = 0.0;
}
}
static void print() { print_impl(std::make_index_sequence<arity> {}); }
template <size_t... Is> static void print_impl(std::index_sequence<Is...>)
{
mfem::out << "for d in [0, " << grad_components
<< "): __enzyme_fwddiff<void>(fptr, enzyme_dup";
((mfem::out << ", "
<< get_type_name<std::tuple_element_t<Is, args_tuple>>()),
...);
mfem::out << ", enzyme_interleave";
(([&]
{
if constexpr (Is == active_input) { mfem::out << ", e_d seed"; }
else if constexpr (Is == active_output) { mfem::out << ", tangent out"; }
else { mfem::out << ", zero tangent"; }
}()),
...);
mfem::out << ")\n";
}
};
template <typename Func, size_t active_input, size_t active_output>
struct create_function_signature<FwdDiff<Func, active_input, active_output>>
{
using type =
typename FwdDiff<Func, active_input, active_output>::signature;
};
} // namespace mfem::future
#endif // MFEM_USE_ENZYME
-83
View File
@@ -1,83 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
// Explicit instantiation of the local Q-function backend device code
#include "../../../../config/config.hpp"
#ifdef MFEM_USE_MPI
#include "kernels.hpp"
namespace mfem::future
{
// ────────────────────────────────────────────────────────────────────────────
// Low-order backends instantiations for (DIM, Q1D)
// ────────────────────────────────────────────────────────────────────────────
template struct lo_ker_backend<2, 2>;
template struct lo_ker_backend<2, 3>;
template struct lo_ker_backend<2, 4>;
template struct lo_ker_backend<2, 5>;
template struct lo_ker_backend<2, 6>;
template struct lo_ker_backend<2, 7>;
template struct lo_ker_backend<2, 8>;
template struct lo_ker_backend<3, 2>;
template struct lo_ker_backend<3, 3>;
template struct lo_ker_backend<3, 4>;
template struct lo_ker_backend<3, 5>;
template struct lo_ker_backend<3, 6>;
template struct lo_ker_backend<3, 7>;
template struct lo_ker_backend<3, 8>;
template struct LocalQFLOBackend<2, 2>;
template struct LocalQFLOBackend<2, 3>;
template struct LocalQFLOBackend<2, 4>;
template struct LocalQFLOBackend<2, 5>;
template struct LocalQFLOBackend<2, 6>;
template struct LocalQFLOBackend<2, 7>;
template struct LocalQFLOBackend<2, 8>;
template struct LocalQFLOBackend<3, 2>;
template struct LocalQFLOBackend<3, 3>;
template struct LocalQFLOBackend<3, 4>;
template struct LocalQFLOBackend<3, 5>;
template struct LocalQFLOBackend<3, 6>;
template struct LocalQFLOBackend<3, 7>;
template struct LocalQFLOBackend<3, 8>;
// ────────────────────────────────────────────────────────────────────────────
// High-order backends instantiations for (DIM, Q1D)
// ────────────────────────────────────────────────────────────────────────────
template struct ho_ker_backend<2, 8>;
template struct ho_ker_backend<2, 10>;
template struct ho_ker_backend<2, 12>;
template struct ho_ker_backend<2, 16>;
template struct ho_ker_backend<3, 8>;
template struct ho_ker_backend<3, 10>;
template struct ho_ker_backend<3, 12>;
template struct ho_ker_backend<3, 16>;
template struct LocalQFHOBackend<2, 8>;
template struct LocalQFHOBackend<2, 10>;
template struct LocalQFHOBackend<2, 12>;
template struct LocalQFHOBackend<2, 16>;
template struct LocalQFHOBackend<3, 8>;
template struct LocalQFHOBackend<3, 10>;
template struct LocalQFHOBackend<3, 12>;
template struct LocalQFHOBackend<3, 16>;
} // namespace mfem::future
#endif // MFEM_USE_MPI
-77
View File
@@ -1,77 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "kernels_lo.hpp" // IWYU pragma: export
#include "kernels_ho.hpp" // IWYU pragma: export
namespace mfem::future
{
// ────────────────────────────────────────────────────────────────────────────
// Low-order backends instantiations for (DIM, Q1D)
// ────────────────────────────────────────────────────────────────────────────
extern template struct lo_ker_backend<2, 2>;
extern template struct lo_ker_backend<2, 3>;
extern template struct lo_ker_backend<2, 4>;
extern template struct lo_ker_backend<2, 5>;
extern template struct lo_ker_backend<2, 6>;
extern template struct lo_ker_backend<2, 7>;
extern template struct lo_ker_backend<2, 8>;
extern template struct lo_ker_backend<3, 2>;
extern template struct lo_ker_backend<3, 3>;
extern template struct lo_ker_backend<3, 4>;
extern template struct lo_ker_backend<3, 5>;
extern template struct lo_ker_backend<3, 6>;
extern template struct lo_ker_backend<3, 7>;
extern template struct lo_ker_backend<3, 8>;
extern template struct LocalQFLOBackend<2, 2>;
extern template struct LocalQFLOBackend<2, 3>;
extern template struct LocalQFLOBackend<2, 4>;
extern template struct LocalQFLOBackend<2, 5>;
extern template struct LocalQFLOBackend<2, 6>;
extern template struct LocalQFLOBackend<2, 7>;
extern template struct LocalQFLOBackend<2, 8>;
extern template struct LocalQFLOBackend<3, 2>;
extern template struct LocalQFLOBackend<3, 3>;
extern template struct LocalQFLOBackend<3, 4>;
extern template struct LocalQFLOBackend<3, 5>;
extern template struct LocalQFLOBackend<3, 6>;
extern template struct LocalQFLOBackend<3, 7>;
extern template struct LocalQFLOBackend<3, 8>;
// ────────────────────────────────────────────────────────────────────────────
// High-order backends instantiations for (DIM, Q1D)
// ────────────────────────────────────────────────────────────────────────────
extern template struct ho_ker_backend<2, 8>;
extern template struct ho_ker_backend<2, 10>;
extern template struct ho_ker_backend<2, 12>;
extern template struct ho_ker_backend<2, 16>;
extern template struct ho_ker_backend<3, 8>;
extern template struct ho_ker_backend<3, 10>;
extern template struct ho_ker_backend<3, 12>;
extern template struct ho_ker_backend<3, 16>;
extern template struct LocalQFHOBackend<2, 8>;
extern template struct LocalQFHOBackend<2, 10>;
extern template struct LocalQFHOBackend<2, 12>;
extern template struct LocalQFHOBackend<2, 16>;
extern template struct LocalQFHOBackend<3, 8>;
extern template struct LocalQFHOBackend<3, 10>;
extern template struct LocalQFHOBackend<3, 12>;
extern template struct LocalQFHOBackend<3, 16>;
} // namespace mfem::future
-803
View File
@@ -1,803 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../../kernels.hpp"
namespace ker = mfem::kernels::internal;
#include "../../util.hpp" // for ThreadBlocks
#include "util.hpp"
namespace mfem::future
{
// ────────────────────────────────────────────────────────────────────────────
inline constexpr int LocalQFHOBackendMQ1() { return 16; }
// ────────────────────────────────────────────────────────────────────────────
/// Register type for one HO q-function parameter
template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
struct ho_qreg;
template<typename KerOps, typename T>
struct ho_qreg<KerOps, T, 0>
{
using type = typename KerOps::template val_reg_t<1>;
};
template<typename KerOps, typename T>
struct ho_qreg<KerOps, T, 1>
{
static constexpr int e0 = qf_param_shape<T>::extents[0];
using type = typename KerOps::template val_reg_t<e0>;
};
template<typename KerOps, typename T>
struct ho_qreg<KerOps, T, 2>
{
static constexpr int VDIM = qf_param_shape<T>::extents[0];
static constexpr int SDIM = qf_param_shape<T>::extents[1];
using type = typename KerOps::template del_reg_t<VDIM, SDIM>;
};
template<typename KerOps, typename T>
using ho_qreg_t = typename ho_qreg<KerOps, T>::type;
// ────────────────────────────────────────────────────────────────────────────
namespace hok
{
/// Load one quadrature-point value
template<int DIM, typename T, typename Reg>
MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
{
static_assert(DIM == 2 || DIM == 3);
constexpr int RNK = qf_param_shape<T>::rank;
if constexpr (DIM == 2)
{
MFEM_CONTRACT_VAR(qz);
if constexpr (RNK == 0) { return T{ reg(0, qy, qx) }; }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
T t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qy, qx); }
return t;
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
T t;
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qy, qx); }
}
return t;
}
}
else
{
if constexpr (RNK == 0) { return T{ reg(0, qz, qy, qx) }; }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
T t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd) { t(dd) = reg(dd, qz, qy, qx); }
return t;
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
T t;
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j) { t(i, j) = reg(i, j, qz, qy, qx); }
}
return t;
}
}
}
template<bool tangent, typename U>
MFEM_HOST_DEVICE inline auto qp_store(const U &v)
{
if constexpr (tangent) { return qf_store_gradient(v); }
else
{
return qf_store_value(v);
}
}
// Store primal value or dual tangent at one quadrature point
template<int DIM, typename T, typename Reg, bool tangent>
MFEM_HOST_DEVICE inline void
store_at(Reg &reg, int qx, int qy, int qz, const T &out)
{
static_assert(DIM == 2 || DIM == 3);
constexpr int RNK = qf_param_shape<T>::rank;
if constexpr (DIM == 2)
{
MFEM_CONTRACT_VAR(qz);
if constexpr (RNK == 0) { reg(0, qy, qx) = qp_store<tangent>(out); }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
reg(dd, qy, qx) = qp_store<tangent>(out(dd));
}
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
reg(i, j, qy, qx) = qp_store<tangent>(out(i, j));
}
}
}
}
else
{
if constexpr (RNK == 0) { reg(0, qz, qy, qx) = qp_store<tangent>(out); }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
reg(dd, qz, qy, qx) = qp_store<tangent>(out(dd));
}
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
reg(i, j, qz, qy, qx) = qp_store<tangent>(out(i, j));
}
}
}
}
}
// Pull primal/tangent pair into a dual q-function argument
template<int DIM, typename T, typename Reg>
MFEM_HOST_DEVICE inline auto
pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
{
if constexpr (!qf_param_uses_dual_v<T>)
{
return load_at<DIM, T>(preg, qx, qy, qz);
}
else
{
if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
constexpr int RNK = qf_param_shape<T>::rank;
if constexpr (DIM == 2)
{
MFEM_CONTRACT_VAR(qz);
if constexpr (RNK == 0)
{
return T{ preg(0, qy, qx), sreg(0, qy, qx) };
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
T t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
t(dd) = { preg(dd, qy, qx), sreg(dd, qy, qx) };
}
return t;
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
T t;
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
t(i, j) = { preg(i, j, qy, qx), sreg(i, j, qy, qx) };
}
}
return t;
}
}
else
{
if constexpr (RNK == 0)
{
return T{ preg(0, qz, qy, qx), sreg(0, qz, qy, qx) };
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
T t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
t(dd) = { preg(dd, qz, qy, qx), sreg(dd, qz, qy, qx) };
}
return t;
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
T t;
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
t(i, j) = { preg(i, j, qz, qy, qx), sreg(i, j, qz, qy, qx) };
}
}
return t;
}
}
}
}
} // namespace hok
// ────────────────────────────────────────────────────────────────────────────
/// HO tensor-product kernels
template<int T_DIM, int MQ1>
struct ho_ker_backend
{
static constexpr int DIM = T_DIM;
static_assert(DIM == 2 || DIM == 3);
template<int VDIM>
using val_reg_t = std::conditional_t<(DIM == 2),
ker::v_regs2d_t<VDIM, MQ1>,
ker::v_regs3d_t<VDIM, MQ1>>;
template<int VDIM, int SDIM>
using del_reg_t = std::conditional_t<(DIM == 2),
ker::vd_regs2d_t<VDIM, SDIM, MQ1>,
ker::vd_regs3d_t<VDIM, SDIM, MQ1>>;
struct Shared
{
real_t M[MQ1][MQ1], B[MQ1][MQ1], G[MQ1][MQ1];
};
template<typename XE_t, typename Dofs>
static MFEM_HOST_DEVICE void
load_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
{
if constexpr (DIM == 2) { ker::LoadDofs2d(e, d, XE, dofs); }
else
{
ker::LoadDofs3d(e, d, XE, dofs);
}
}
template<int VDIM, int SDIM, typename XE_t, typename Dofs>
static MFEM_HOST_DEVICE void
load_grad_dofs(const int e, const int d, const XE_t &XE, Dofs &dofs)
{
static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
load_dofs(e, d, XE, dofs);
}
template<typename Smem, typename Dofs, typename ArgReg>
static MFEM_HOST_DEVICE void
eval_value(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
{
if constexpr (DIM == 2) { ker::Eval2d(d, q, s.M, s.B, dofs, rarg); }
else
{
ker::Eval3d(d, q, s.M, s.B, dofs, rarg);
}
}
template<int VDIM, int SDIM, typename Smem, typename Dofs, typename ArgReg>
static MFEM_HOST_DEVICE void
grad(const int d, const int q, Smem &s, Dofs &dofs, ArgReg &rarg)
{
static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
if constexpr (DIM == 2) { ker::Grad2d(d, q, s.M, s.B, s.G, dofs, rarg); }
else
{
ker::Grad3d(d, q, s.M, s.B, s.G, dofs, rarg);
}
}
template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
static MFEM_HOST_DEVICE void write_value(const int d,
const int q,
const int e,
Smem &s,
ArgReg &rarg,
Dofs &dofs,
YE_t &YE)
{
if constexpr (DIM == 2)
{
ker::EvalTranspose2d(d, q, s.M, s.B, rarg, dofs);
ker::WriteDofs2d(e, d, dofs, YE);
}
else
{
ker::EvalTranspose3d(d, q, s.M, s.B, rarg, dofs);
ker::WriteDofs3d(e, d, dofs, YE);
}
}
template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
static MFEM_HOST_DEVICE void write_gradient_2d(const int d,
const int q,
const int e,
Smem &s,
ArgReg &rarg,
Dofs &dofs,
YE_t &YE)
{
ker::GradTranspose2d(d, q, s.M, s.B, s.G, rarg, dofs);
ker::WriteDofs2d(e, d, dofs, YE);
}
template<typename Smem, typename Dofs, typename ArgReg, typename YE_t>
static MFEM_HOST_DEVICE void write_gradient_3d(const int d,
const int q,
const int e,
Smem &s,
ArgReg &rarg,
Dofs &dofs,
YE_t &YE)
{
ker::GradTranspose3d(d, q, s.M, s.B, s.G, rarg, dofs);
ker::WriteDofs3d(e, d, dofs, YE);
}
template<int VDIM,
int SDIM,
typename Smem,
typename Dofs,
typename ArgReg,
typename YE_t>
static MFEM_HOST_DEVICE void write_gradient(const int d,
const int q,
const int e,
Smem &s,
ArgReg &rarg,
Dofs &dofs,
YE_t &YE)
{
static_assert(SDIM == DIM, "gradient spatial dim must match kernel DIM");
if constexpr (DIM == 2) { write_gradient_2d(d, q, e, s, rarg, dofs, YE); }
else
{
write_gradient_3d(d, q, e, s, rarg, dofs, YE);
}
}
};
// ────────────────────────────────────────────────────────────────────────────
template<int T_DIM, int T_Q1D = LocalQFHOBackendMQ1()>
struct LocalQFHOBackend
{
// ─────────────────────────────────────────────────────
static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
static_assert(DIM == 2 || DIM == 3);
// ─────────────────────────────────────────────────────
static inline ThreadBlocks thread_blocks(const int q1d)
{
MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
return { q1d, q1d, 1 };
}
// ─────────────────────────────────────────────────────
static inline constexpr int MAX_THREADS_PER_BLOCK() { return Q1D * Q1D; }
// ─────────────────────────────────────────────────────
using backend_t = ho_ker_backend<DIM, Q1D>;
// ─────────────────────────────────────────────────────
using Shared = typename backend_t::Shared;
// ─────────────────────────────────────────────────────
template<typename WT, typename WI, typename Cache, typename AddY>
static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
const int num_dof_1d,
const int q1d,
const int nz_dof,
WT wt,
WI wi,
Cache cache,
AddY add_y)
{
MFEM_CONTRACT_VAR(nz_dof);
const int nqz = (DIM == 3) ? q1d : 1;
const int ndz = (DIM == 3) ? num_dof_1d : 1;
ker::s_regs3d_t<MQ1> rz, ry;
auto &smem = s.M;
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
for (int dz = 0; dz < ndz; dz++)
{
real_t u = 0.0;
for (int qz = 0; qz < nqz; qz++)
{
const int q = qx + (qy + qz * q1d) * q1d;
const real_t wz =
(DIM == 3) ? (wt(2, qz, dz) * wi(2, qz, dz)) : real_t(1);
u += wz * cache(q);
}
rz[dz][qy][qx] = u;
}
}
}
MFEM_SYNC_THREAD;
for (int dz = 0; dz < ndz; dz++)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{ smem[qy][qx] = rz[dz][qy][qx]; }
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
real_t u = 0.0;
for (int qy = 0; qy < q1d; qy++)
{
u += wt(1, qy, dy) * wi(1, qy, dy) * smem[qy][qx];
}
ry[dz][dy][qx] = u;
}
}
MFEM_SYNC_THREAD;
}
for (int dz = 0; dz < ndz; dz++)
{
MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{ smem[dy][qx] = ry[dz][dy][qx]; }
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD_DIRECT(dy, y, num_dof_1d)
{
MFEM_FOREACH_THREAD_DIRECT(dx, x, num_dof_1d)
{
real_t u = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
u += wt(0, qx, dx) * wi(0, qx, dx) * smem[dy][qx];
}
add_y(dx, dy, dz, u);
}
}
MFEM_SYNC_THREAD;
}
}
// ─────────────────────────────────────────────────────
template<typename T>
using QReg = ho_qreg_t<backend_t, T>;
// ─────────────────────────────────────────────────────
template<typename ArgRegT, typename XE_T>
static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const XE_T &XE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
std::remove_reference_t<ArgRegT> dofs;
backend_t::load_dofs(e, d, XE, dofs);
backend_t::eval_value(d, q, s, dofs, rarg);
}
// ─────────────────────────────────────────────────────
template<int RNK,
typename ArgRegT,
typename XE_T,
typename FieldParamT = ArgRegT>
static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const real_t *G,
const XE_T &XE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
ker::LoadMatrix(d, q, G, s.G);
static_assert(RNK == 1 || RNK == 2);
static constexpr int VDIM =
(RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
static constexpr int SDIM = (RNK == 1)
? qf_param_shape<FieldParamT>::extents[0]
: qf_param_shape<FieldParamT>::extents[1];
if constexpr (SDIM == DIM)
{
typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
if constexpr (RNK == 1) { backend_t::load_dofs(e, d, XE, dofs); }
else
{
backend_t::template load_grad_dofs<VDIM, SDIM>(e, d, XE, dofs);
}
backend_t::template grad<VDIM, SDIM>(d, q, s, dofs, rarg);
}
}
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline auto
qp_pull(QReg<T> &reg, int qx, int qy, int qz)
{ return hok::load_at<DIM, T>(reg, qx, qy, qz); }
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline auto qp_pull_directional(
QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
{ return hok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
// ─────────────────────────────────────────────────────
template<typename DT, typename XE_T>
static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
const XE_T &XP,
const XE_T &XD,
int qx,
int qy,
int qz,
int e)
{
constexpr int RNK = qf_param_shape<DT>::rank;
if constexpr (RNK == 0)
{
DT t{};
t.value = XP(0, qx, qy, qz, e);
t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
return t;
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
DT t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
t(dd).value = XP(dd, qx, qy, qz, e);
t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
}
return t;
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
constexpr int e1 = qf_param_shape<DT>::extents[1];
DT t{};
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
t(i, j).gradient =
dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
}
}
return t;
}
else
{
static_assert(false, "Unsupported");
}
}
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline void
qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
{ hok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out); }
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline void
qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
{
hok::store_at<DIM, T, decltype(reg), qf_param_uses_dual_v<T>>(
reg, qx, qy, qz, out);
}
// ─────────────────────────────────────────────────────
template<typename DT, typename YE_T>
static MFEM_HOST_DEVICE inline void identity_qp_write_value(
YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
{
constexpr int RNK = qf_param_shape<DT>::rank;
if constexpr (qf_param_uses_dual_v<DT>)
{
if constexpr (RNK == 0)
{
YE(0, qx, qy, qz, e) = qf_store_value(qout);
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
}
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
constexpr int e1 = qf_param_shape<DT>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
}
}
}
else
{
static_assert(false, "Unsupported");
}
}
}
// ─────────────────────────────────────────────────────
template<typename DT, typename YE_T>
static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
{
constexpr int RNK = qf_param_shape<DT>::rank;
if constexpr (qf_param_uses_dual_v<DT>)
{
if constexpr (RNK == 0)
{
YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
}
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
constexpr int e1 = qf_param_shape<DT>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
}
}
}
else
{
static_assert(false, "Unsupported");
}
}
}
// ─────────────────────────────────────────────────────
template<typename ArgRegT, typename YE_T>
static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
YE_T &YE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
std::remove_reference_t<ArgRegT> dofs;
backend_t::write_value(d, q, e, s, rarg, dofs, YE);
}
// ─────────────────────────────────────────────────────
template<int RNK,
typename ArgRegT,
typename YE_T,
typename FieldParamT = ArgRegT>
static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const real_t *G,
YE_T &YE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
ker::LoadMatrix(d, q, G, s.G);
static_assert(RNK == 1 || RNK == 2);
static constexpr int VDIM =
(RNK == 1) ? 1 : qf_param_shape<FieldParamT>::extents[0];
static constexpr int SDIM = (RNK == 1)
? qf_param_shape<FieldParamT>::extents[0]
: qf_param_shape<FieldParamT>::extents[1];
if constexpr (SDIM == DIM)
{
typename backend_t::template del_reg_t<VDIM, SDIM> dofs;
backend_t::template write_gradient<VDIM, SDIM>(
d, q, e, s, rarg, dofs, YE);
}
}
};
/// @brief Dispatch to a compile-time HO kernel with MQ1 >= runtime @a q1d.
template <typename HOKernelTable, int DIM, int MQ1 = LocalQFHOBackendMQ1()>
inline typename HOKernelTable::KernelSignature
DispatchHOKernelByQ1D(int q1d)
{
MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
"Unsupported HO quadrature order: " << q1d);
return HOKernelTable::template Kernel<DIM, MQ1>();
}
} // namespace mfem::future
-813
View File
@@ -1,813 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../../kernels.hpp"
namespace ker = mfem::kernels::internal;
#include "../../util.hpp" // for ThreadBlocks
#include "../util.hpp" // for as_tensor
#include "util.hpp"
namespace mfem::future
{
// ────────────────────────────────────────────────────────────────────────────
inline constexpr int LocalQFLOBackendMQ1() { return 8; }
// ────────────────────────────────────────────────────────────────────────────
/// Register type for one LO q-function parameter
template<typename KerOps, typename T, int rank = qf_param_shape<T>::rank>
struct lo_qreg;
template<typename KerOps, typename T>
struct lo_qreg<KerOps, T, 0>
{
using type = typename KerOps::template qreg_t<1>;
};
template<typename KerOps, typename T>
struct lo_qreg<KerOps, T, 1>
{
static constexpr int e0 = qf_param_shape<T>::extents[0];
using type = typename KerOps::template qreg_t<e0>;
};
template<typename KerOps, typename T>
struct lo_qreg<KerOps, T, 2>
{
static constexpr int e0 = qf_param_shape<T>::extents[0];
static constexpr int e1 = qf_param_shape<T>::extents[1];
using type = typename KerOps::template qreg_vd_t<e0, e1>;
};
template<typename KerOps, typename T>
using lo_qreg_t = typename lo_qreg<KerOps, T>::type;
// ────────────────────────────────────────────────────────────────────────────
namespace lok
{
template<int DIM, typename Reg>
MFEM_HOST_DEVICE inline auto &at(Reg &reg, int qx, int qy, int qz)
{
if constexpr (DIM == 2)
{
MFEM_CONTRACT_VAR(qz);
return reg[qy][qx];
}
else
{
return reg[qz][qy][qx];
}
}
template<int DIM, typename T, typename Reg>
MFEM_HOST_DEVICE inline auto load_at(Reg &reg, int qx, int qy, int qz)
{
constexpr int RNK = qf_param_shape<T>::rank;
auto &qp = at<DIM>(reg, qx, qy, qz);
if constexpr (RNK == 0) { return T{ qp[0] }; }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
T t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd) { t(dd) = qp[dd]; }
return t;
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
T t;
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j) { t(i, j) = qp[i][j]; }
}
return t;
}
}
template<bool tangent, typename U>
MFEM_HOST_DEVICE inline auto qp_store(const U &v)
{
if constexpr (tangent) { return qf_store_gradient(v); }
else
{
return qf_store_value(v);
}
}
// Store primal value or dual tangent at one quadrature point
template<int DIM, typename T, typename Reg, bool tangent>
MFEM_HOST_DEVICE inline void
store_at(Reg &reg, int qx, int qy, int qz, const T &out)
{
constexpr int RNK = qf_param_shape<T>::rank;
auto &qp = at<DIM>(reg, qx, qy, qz);
if constexpr (RNK == 0) { qp[0] = qp_store<tangent>(out); }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd) { qp[dd] = qp_store<tangent>(out(dd)); }
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
qp[i][j] = qp_store<tangent>(out(i, j));
}
}
}
}
// Pull primal/tangent pair into a dual q-function argument
template<int DIM, typename T, typename Reg>
MFEM_HOST_DEVICE inline auto
pull_directional(Reg &preg, Reg &sreg, int qx, int qy, int qz, bool dependent)
{
if constexpr (!qf_param_uses_dual_v<T>)
{
return load_at<DIM, T>(preg, qx, qy, qz);
}
else
{
if (!dependent) { return load_at<DIM, T>(preg, qx, qy, qz); }
constexpr int RNK = qf_param_shape<T>::rank;
auto &pqp = at<DIM>(preg, qx, qy, qz);
auto &sqp = at<DIM>(sreg, qx, qy, qz);
if constexpr (RNK == 0) { return T{ pqp[0], sqp[0] }; }
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
T t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd) { t(dd) = { pqp[dd], sqp[dd] }; }
return t;
}
else
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
T t;
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j) { t(i, j) = { pqp[i][j], sqp[i][j] }; }
}
return t;
}
}
}
} // namespace lok
// ────────────────────────────────────────────────────────────────────────────
/// LO tensor-product kernels
template<int T_DIM, int MQ1>
struct lo_ker_backend
{
static constexpr int DIM = T_DIM;
static_assert(DIM == 2 || DIM == 3);
template<int VDIM>
using qreg_t = std::conditional_t<(DIM == 2),
ker::regs2d_t<VDIM, MQ1>,
ker::regs3d_t<VDIM, MQ1>>;
template<int VDIM, int SDIM>
using qreg_vd_t = std::conditional_t<(DIM == 2),
ker::regs2d_vd_t<VDIM, SDIM, MQ1>,
ker::regs3d_vd_t<VDIM, SDIM, MQ1>>;
struct Shared2d
{
real_t M[2][MQ1][MQ1][DIM];
real_t B[MQ1][MQ1], G[MQ1][MQ1];
};
struct Shared3d
{
real_t M[2][MQ1][MQ1][MQ1][DIM];
real_t B[MQ1][MQ1], G[MQ1][MQ1];
};
using Shared = std::conditional_t<(DIM == 2), Shared2d, Shared3d>;
template<typename FieldParamT, typename ArgRegT, typename XE_T>
static MFEM_HOST_DEVICE void load_value(Shared &s,
const int e,
const int d,
const int q,
const real_t *B,
const XE_T &XE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
using field_t = std::remove_cv_t<std::remove_reference_t<FieldParamT>>;
constexpr int RNK = qf_param_shape<field_t>::rank;
constexpr int VDIM = []()
{
if constexpr (RNK == 0) { return 1; }
else { return qf_param_shape<field_t>::extents[RNK - 1]; }
}();
if constexpr (DIM == 2)
{
ker::LoadDofs2d<VDIM, DIM, MQ1>(e, d, XE, s.M[0]);
ker::Eval2d(d, q, s.B, s.M[0], s.M[1], rarg);
}
else
{
ker::LoadDofs3d<VDIM, DIM, MQ1>(e, d, XE, s.M[0]);
ker::Eval3d(d, q, s.B, s.M[0], s.M[1], rarg);
}
}
template<int RNK,
typename ArgRegT,
typename XE_T,
typename FieldParamT = ArgRegT>
static MFEM_HOST_DEVICE void load_gradient(Shared &s,
const int e,
const int d,
const int q,
const real_t *B,
const real_t *G,
const XE_T &XE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
ker::LoadMatrix(d, q, G, s.G);
static_assert(RNK == 1 || RNK == 2);
if constexpr (RNK == 1)
{
static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[0];
if constexpr (SDIM == DIM)
{
if constexpr (DIM == 2)
{
ker::LoadDofs2d(e, d, 0, XE, s.M[0]);
ker::Grad2d(d, q, s.B, s.G, s.M[0], s.M[1], rarg);
}
else
{
ker::LoadDofs3d(e, d, 0, XE, s.M[0]);
ker::Grad3d(d, q, s.B, s.G, s.M[0], s.M[1], rarg);
}
}
}
if constexpr (RNK == 2)
{
static constexpr int VDIM = qf_param_shape<FieldParamT>::extents[0];
static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[1];
if constexpr (SDIM == DIM)
{
for (int c = 0; c < VDIM; ++c)
{
if constexpr (DIM == 2)
{
ker::LoadDofs2d(e, d, c, XE, s.M[0]);
ker::VectorGrad2d(d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
}
else
{
ker::LoadDofs3d(e, d, c, XE, s.M[0]);
ker::VectorGrad3d(d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
}
}
}
}
}
template<typename ArgRegT, typename YE_T>
static MFEM_HOST_DEVICE void write_value(Shared &s,
const int e,
const int d,
const int q,
const real_t *B,
const YE_T &YE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
if constexpr (DIM == 2)
{
ker::EvalTranspose2d(d, q, s.B, rarg, s.M[1], s.M[0]);
ker::WriteEvalDofs2d(d, 0, e, rarg, YE);
}
else
{
ker::EvalTranspose3d(d, q, s.B, rarg, s.M[1], s.M[0]);
ker::WriteEvalDofs3d(d, 0, e, rarg, YE);
}
}
template<int RNK,
typename ArgRegT,
typename YE_T,
typename FieldParamT = ArgRegT>
static MFEM_HOST_DEVICE void write_gradient(Shared &s,
const int e,
const int d,
const int q,
const real_t *B,
const real_t *G,
YE_T &YE,
ArgRegT &rarg)
{
ker::LoadMatrix(d, q, B, s.B);
ker::LoadMatrix(d, q, G, s.G);
static_assert(RNK == 1 || RNK == 2);
if constexpr (RNK == 1)
{
static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[0];
if constexpr (SDIM == DIM)
{
if constexpr (DIM == 2)
{
ker::GradTranspose2d(d, q, s.B, s.G, rarg, s.M[1], s.M[0]);
ker::WriteGradDofs2d(d, 0, e, rarg, YE);
}
else
{
ker::GradTranspose3d(d, q, s.B, s.G, rarg, s.M[1], s.M[0]);
ker::WriteGradDofs3d(d, 0, e, rarg, YE);
}
}
}
else if constexpr (RNK == 2)
{
static constexpr int VDIM = qf_param_shape<FieldParamT>::extents[0];
static constexpr int SDIM = qf_param_shape<FieldParamT>::extents[1];
if constexpr (SDIM == DIM)
{
for (int c = 0; c < VDIM; ++c)
{
if constexpr (DIM == 2)
{
ker::VectorGradTranspose2d(
d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
ker::WriteGradDofs2d(d, c, e, rarg, YE);
}
else
{
ker::VectorGradTranspose3d(
d, q, c, s.B, s.G, s.M[0], s.M[1], rarg);
ker::WriteGradDofs3d(d, c, e, rarg, YE);
}
}
}
}
else
{
static_assert(false, "Unsupported");
}
}
};
// ────────────────────────────────────────────────────────────────────────────
template<int T_DIM, int T_Q1D = LocalQFLOBackendMQ1()>
struct LocalQFLOBackend
{
// ─────────────────────────────────────────────────────
static constexpr int DIM = T_DIM, MQ1 = T_Q1D, Q1D = T_Q1D;
static_assert(DIM == 2 || DIM == 3);
// ─────────────────────────────────────────────────────
static inline ThreadBlocks thread_blocks(const int q1d)
{
MFEM_ASSERT(q1d <= Q1D, "q1d must be <= " << Q1D);
return { q1d, q1d, (DIM == 2) ? 1 : q1d };
}
// ─────────────────────────────────────────────────────
static inline constexpr int MAX_THREADS_PER_BLOCK()
{ return Q1D * Q1D * ((DIM == 2) ? 1 : Q1D); }
// ─────────────────────────────────────────────────────
using backend_t = lo_ker_backend<DIM, Q1D>;
// ─────────────────────────────────────────────────────
using Shared = typename backend_t::Shared;
// ─────────────────────────────────────────────────────
template<typename WT, typename WI, typename Cache, typename AddY>
static MFEM_HOST_DEVICE inline void DiagContract(Shared &s,
const int num_dof_1d,
const int q1d,
const int nz_dof,
WT wt,
WI wi,
Cache cache,
AddY add_y)
{
MFEM_CONTRACT_VAR(nz_dof);
real_t *base = reinterpret_cast<real_t *>(&s.M[0]);
auto s0 = reinterpret_cast<real_t(*)[Q1D][Q1D]>(base);
if constexpr (DIM == 3)
{
auto s1 =
reinterpret_cast<real_t(*)[Q1D][Q1D]>(base + Q1D * Q1D * Q1D);
// reduce qz → dz : s0[dz][qy][qx]
MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
MFEM_FOREACH_THREAD(qy, y, q1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t u = 0.0;
for (int qz = 0; qz < q1d; qz++)
{
const int q = qx + (qy + qz * q1d) * q1d;
u += wt(2, qz, dz) * wi(2, qz, dz) * cache(q);
}
s0[dz][qy][qx] = u;
}
MFEM_SYNC_THREAD;
// reduce qy → dy : s1[dz][dy][qx]
MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t u = 0.0;
for (int qy = 0; qy < q1d; qy++)
{
u += wt(1, qy, dy) * wi(1, qy, dy) * s0[dz][qy][qx];
}
s1[dz][dy][qx] = u;
}
MFEM_SYNC_THREAD;
// reduce qx → dx : Y(dx,dy,dz)
MFEM_FOREACH_THREAD(dz, z, num_dof_1d)
MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
MFEM_FOREACH_THREAD(dx, x, num_dof_1d)
{
real_t u = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
u += wt(0, qx, dx) * wi(0, qx, dx) * s1[dz][dy][qx];
}
add_y(dx, dy, dz, u);
}
MFEM_SYNC_THREAD;
}
else
{
// reduce qy → dy : s0[0][dy][qx]
MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t u = 0.0;
for (int qy = 0; qy < q1d; qy++)
{
const int q = qx + qy * q1d;
u += wt(1, qy, dy) * wi(1, qy, dy) * cache(q);
}
s0[0][dy][qx] = u;
}
MFEM_SYNC_THREAD;
// reduce qx → dx : Y(dx,dy,0)
MFEM_FOREACH_THREAD(dy, y, num_dof_1d)
MFEM_FOREACH_THREAD(dx, x, num_dof_1d)
{
real_t u = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
u += wt(0, qx, dx) * wi(0, qx, dx) * s0[0][dy][qx];
}
add_y(dx, dy, 0, u);
}
MFEM_SYNC_THREAD;
}
}
// ─────────────────────────────────────────────────────
template<typename T>
using QReg = lo_qreg_t<backend_t, T>;
// ─────────────────────────────────────────────────────
template<typename ArgRegT, typename XE_T>
static inline MFEM_HOST_DEVICE void LoadValue(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const XE_T &XE,
ArgRegT &rarg)
{
backend_t::template load_value<ArgRegT>(s, e, d, q, B, XE, rarg);
}
// ─────────────────────────────────────────────────────
template<int RNK,
typename ArgRegT,
typename XE_T,
typename FieldParamT = ArgRegT>
static inline MFEM_HOST_DEVICE void LoadGradient(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const real_t *G,
const XE_T &XE,
ArgRegT &rarg)
{
backend_t::template load_gradient<RNK, ArgRegT, XE_T, FieldParamT>(
s, e, d, q, B, G, XE, rarg);
}
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline auto
qp_pull(QReg<T> &reg, int qx, int qy, int qz)
{
if constexpr (qf_param_uses_dual_v<T>)
{
return lok::load_at<DIM, T>(reg, qx, qy, qz);
}
else
{
constexpr int RNK = qf_param_shape<T>::rank;
if constexpr (RNK == 0)
{
return as_tensor<real_t>(&lok::at<DIM>(reg, qx, qy, qz)[0]);
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
return as_tensor<real_t, e0>(&lok::at<DIM>(reg, qx, qy, qz)[0]);
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
return as_tensor<real_t, e0, e1>(
&lok::at<DIM>(reg, qx, qy, qz)[0][0]);
}
else
{
static_assert(false, "Unsupported");
}
}
}
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline auto qp_pull_directional(
QReg<T> &preg, QReg<T> &sreg, int qx, int qy, int qz, bool dependent)
{ return lok::pull_directional<DIM, T>(preg, sreg, qx, qy, qz, dependent); }
// ─────────────────────────────────────────────────────
template<typename DT, typename XE_T>
static MFEM_HOST_DEVICE inline DT identity_qp_pull_dual(bool dependent,
const XE_T &XP,
const XE_T &XD,
int qx,
int qy,
int qz,
int e)
{
constexpr int RNK = qf_param_shape<DT>::rank;
if constexpr (RNK == 0)
{
DT t{};
t.value = XP(0, qx, qy, qz, e);
t.gradient = dependent ? XD(0, qx, qy, qz, e) : 0.0;
return t;
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
DT t{};
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
t(dd).value = XP(dd, qx, qy, qz, e);
t(dd).gradient = dependent ? XD(dd, qx, qy, qz, e) : 0.0;
}
return t;
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
constexpr int e1 = qf_param_shape<DT>::extents[1];
DT t{};
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
t(i, j).value = XP(i + e0 * j, qx, qy, qz, e);
t(i, j).gradient =
dependent ? XD(i + e0 * j, qx, qy, qz, e) : 0.0;
}
}
return t;
}
else
{
static_assert(false, "Unsupported");
}
}
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline void
qp_push(QReg<T> &reg, int qx, int qy, int qz, const T &out)
{
if constexpr (qf_param_uses_dual_v<T>)
{
lok::store_at<DIM, T, decltype(reg), false>(reg, qx, qy, qz, out);
}
else
{
constexpr int RNK = qf_param_shape<T>::rank;
if constexpr (RNK == 0)
{
as_tensor<real_t>(&lok::at<DIM>(reg, qx, qy, qz)[0]) = out;
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
as_tensor<real_t, e0>(&lok::at<DIM>(reg, qx, qy, qz)[0]) = out;
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<T>::extents[0];
constexpr int e1 = qf_param_shape<T>::extents[1];
as_tensor<real_t, e0, e1>(&lok::at<DIM>(reg, qx, qy, qz)[0][0]) =
out;
}
else
{
static_assert(false, "Unsupported");
}
}
}
// ─────────────────────────────────────────────────────
template<typename T>
static MFEM_HOST_DEVICE inline void
qp_push_tangent(QReg<T> &reg, int qx, int qy, int qz, const T &out)
{
if constexpr (!qf_param_uses_dual_v<T>)
{
qp_push<T>(reg, qx, qy, qz, out);
}
else
{
lok::store_at<DIM, T, decltype(reg), true>(reg, qx, qy, qz, out);
}
}
// ─────────────────────────────────────────────────────
template<typename DT, typename YE_T>
static MFEM_HOST_DEVICE inline void identity_qp_write_value(
YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
{
constexpr int RNK = qf_param_shape<DT>::rank;
if constexpr (qf_param_uses_dual_v<DT>)
{
if constexpr (RNK == 0)
{
YE(0, qx, qy, qz, e) = qf_store_value(qout);
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
YE(dd, qx, qy, qz, e) = qf_store_value(qout(dd));
}
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
constexpr int e1 = qf_param_shape<DT>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
YE(i + e0 * j, qx, qy, qz, e) = qf_store_value(qout(i, j));
}
}
}
else
{
static_assert(false, "Unsupported");
}
}
}
// ─────────────────────────────────────────────────────
template<typename DT, typename YE_T>
static MFEM_HOST_DEVICE inline void identity_qp_write_tangent(
YE_T &YE, int qx, int qy, int qz, int e, const DT &qout)
{
constexpr int RNK = qf_param_shape<DT>::rank;
if constexpr (qf_param_uses_dual_v<DT>)
{
if constexpr (RNK == 0)
{
YE(0, qx, qy, qz, e) = qf_store_gradient(qout);
}
else if constexpr (RNK == 1)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
MFEM_UNROLL(e0)
for (int dd = 0; dd < e0; ++dd)
{
YE(dd, qx, qy, qz, e) = qf_store_gradient(qout(dd));
}
}
else if constexpr (RNK == 2)
{
constexpr int e0 = qf_param_shape<DT>::extents[0];
constexpr int e1 = qf_param_shape<DT>::extents[1];
MFEM_UNROLL(e0)
for (int i = 0; i < e0; ++i)
{
MFEM_UNROLL(e1)
for (int j = 0; j < e1; ++j)
{
YE(i + e0 * j, qx, qy, qz, e) = qf_store_gradient(qout(i, j));
}
}
}
else
{
static_assert(false, "Unsupported");
}
}
}
// ─────────────────────────────────────────────────────
template<typename ArgRegT, typename YE_T>
static inline MFEM_HOST_DEVICE void WriteValue(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const YE_T &YE,
ArgRegT &rarg)
{ backend_t::write_value(s, e, d, q, B, YE, rarg); }
// ─────────────────────────────────────────────────────
template<int RNK,
typename ArgRegT,
typename YE_T,
typename FieldParamT = ArgRegT>
static inline MFEM_HOST_DEVICE void WriteGradient(Shared &s,
const int e,
const int d,
const int q,
const int,
const real_t *B,
const real_t *G,
YE_T &YE,
ArgRegT &rarg)
{
backend_t::template write_gradient<RNK, ArgRegT, YE_T, FieldParamT>(
s, e, d, q, B, G, YE, rarg);
}
};
/// @brief Dispatch to a compile-time LO kernel matching runtime @a q1d.
template <typename LOKernelTable, int DIM, int MQ1 = LocalQFLOBackendMQ1()>
inline typename LOKernelTable::KernelSignature
DispatchLOKernelByQ1D(int q1d)
{
MFEM_VERIFY(q1d >= 2 && q1d <= MQ1,
"Unsupported LO quadrature order: " << q1d);
return LOKernelTable::template Kernel<DIM, MQ1>();
}
} // namespace mfem::future
-242
View File
@@ -1,242 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../integrator_ctx.hpp"
#include "action.hpp"
#include "derivative_action.hpp"
#include "derivative_setup.hpp"
#include "derivative_apply.hpp"
#include "derivative_assemble.hpp"
#include "derivative_assemble_diagonal.hpp"
#include "derivative_apply_transpose.hpp"
namespace mfem::future
{
struct LocalQFBackend
{
/**
* @brief Make an action for a local Q-function backend.
*
* @param ctx The integrator context.
* @param args The arguments to the action.
* @return The action.
*/
template<typename qfunc_t, typename inputs_t, typename outputs_t>
static auto MakeAction(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs)
{
return LocalQFImpl::Action<qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs);
}
/**
* @brief Make a derivative action for a local Q-function backend.
*
* @tparam derivative_id The id of the derivative.
* @param ctx The integrator context.
* @param args The arguments to the derivative action.
* @return The derivative action.
*/
template<int id, typename qfunc_t, typename inputs_t, typename outputs_t>
static auto MakeDerivativeAction(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs)
{
return LocalQFImpl::DerivativeAction<id, qfunc_t, inputs_t, outputs_t>(
ctx, qfunc, inputs, outputs);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
static auto MakeDerivativeSetup(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
Vector &qp_cache)
{
return LocalQFImpl::DerivativeSetup<
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
static auto MakeDerivativeApply(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeApply<
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
static auto MakeDerivativeApplyTranspose(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeApplyTranspose<
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
static auto MakeDerivativeAssemble(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeAssemble<
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
outputs, qp_cache);
}
template<
int derivative_id,
typename qfunc_t,
typename inputs_t,
typename outputs_t>
static auto MakeDerivativeAssembleDiagonal(
const IntegratorContext &ctx,
const qfunc_t &qfunc,
inputs_t inputs,
outputs_t outputs,
const Vector &qp_cache)
{
return LocalQFImpl::DerivativeAssembleDiagonal<
derivative_id, qfunc_t, inputs_t, outputs_t>(ctx, qfunc, inputs,
outputs, qp_cache);
}
};
// ────────────────────────────────────────────────────────────────────────────
template<int DIM, int Q1D, typename QT, typename IT, typename OT>
inline void AddAction()
{
using ker = LocalQFImpl::Action<QT, IT, OT>;
if constexpr (Q1D <= 8)
{
ker::ActionLO::template Specialization<DIM, Q1D>::Add();
}
else
{
ker::ActionHO::template Specialization<DIM, Q1D>::Add();
}
}
// ────────────────────────────────────────────────────────────────────────────
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
inline void AddDerivativeAction()
{
using ker = LocalQFImpl::DerivativeAction<DID, QT, IT, OT>;
if constexpr (Q1D <= 8)
{
ker::DerivativeActionLO::template Specialization<DIM, Q1D>::Add();
}
else
{
ker::DerivativeActionHO::template Specialization<DIM, Q1D>::Add();
}
}
// ────────────────────────────────────────────────────────────────────────────
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
inline void AddDerivativeSetup()
{
using ker = LocalQFImpl::DerivativeSetup<DID, QT, IT, OT>;
if constexpr (Q1D <= 8)
{
ker::DerivativeSetupLO::template Specialization<DIM, Q1D>::Add();
}
else
{
ker::DerivativeSetupHO::template Specialization<DIM, Q1D>::Add();
}
}
// ────────────────────────────────────────────────────────────────────────────
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
inline void AddDerivativeApply()
{
using ker = LocalQFImpl::DerivativeApply<DID, QT, IT, OT>;
if constexpr (Q1D <= 8)
{
ker::DerivativeApplyLO::template Specialization<DIM, Q1D>::Add();
}
else
{
ker::DerivativeApplyHO::template Specialization<DIM, Q1D>::Add();
}
}
// ────────────────────────────────────────────────────────────────────────────
template<int DIM, int Q1D, int DID, typename QT, typename IT, typename OT>
inline void AddDerivativeApplyTranspose()
{
using ker = LocalQFImpl::DerivativeApplyTranspose<DID, QT, IT, OT>;
if constexpr (Q1D <= 8)
{
ker::DerivativeApplyTransposeLO::template Specialization<DIM, Q1D>::Add();
}
else
{
ker::DerivativeApplyTransposeHO::template Specialization<DIM, Q1D>::Add();
}
}
// ────────────────────────────────────────────────────────────────────────────
template<int DIM, int Q1D, typename QT, typename IT, typename OT,
typename derivative_ids_t = std::index_sequence<>>
inline void AddLocalSpecializations()
{
AddAction<DIM, Q1D, QT, IT, OT>();
for_constexpr([&](auto i)
{
using derivative_id = decltype(i);
AddDerivativeAction<DIM, Q1D, derivative_id::value, QT, IT, OT>();
AddDerivativeSetup<DIM, Q1D, derivative_id::value, QT, IT, OT>();
AddDerivativeApply<DIM, Q1D, derivative_id::value, QT, IT, OT>();
AddDerivativeApplyTranspose<DIM, Q1D, derivative_id::value, QT, IT, OT>();
}, derivative_ids_t{});
}
}
@@ -1,274 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../../util.hpp"
#ifdef MFEM_USE_ENZYME
namespace mfem::future
{
template <typename T> struct function_traits;
template <typename R, typename C, typename... Args>
struct function_traits<R (C::*)(Args...) const>
{
using primal_return_type = R;
using args_tuple = tuple<Args...>;
static constexpr size_t arity = sizeof...(Args);
};
// Component count and writable counterpart of a per-point argument, which
// is either a tensor or a plain scalar.
template <typename Arg> struct qp_traits
{
static_assert(std::is_arithmetic_v<Arg>,
"per-point arguments must be tensors or scalars");
using view_type = Arg;
static constexpr int components = 1;
};
template <typename T, int... Sizes> struct qp_traits<tensor<T, Sizes...>>
{
using view_type = tensor<std::remove_const_t<T>, Sizes...>;
static constexpr int components = (Sizes * ... * 1);
};
template <typename... T1s, typename... T2s>
constexpr tuple<T1s..., T2s...> concat_tuples(tuple<T1s...>, tuple<T2s...>);
// RevDiff: computes the full gradient of a pointwise qfunction at a single
// quadrature point using one Enzyme reverse-mode (autodiff) call.
//
// operator()(args...) takes all qfunction input primals followed by one
// writable gradient output per Active input (same shape as that input).
// The qfunction's own output is not passed; Enzyme writes it to stack
// scratch (enzyme_dupnoneed).
//
// A single __enzyme_autodiff call with the output adjoint seeded to 1
// yields all gradient blocks simultaneously — O(1) calls per point
// regardless of input size, vs O(ncomp) for forward mode.
//
// operator() is MFEM_HOST_DEVICE, stateless and allocation-free.
template <typename Func, typename InputActivityTuple,
typename OutputActivityTuple>
struct RevDiff
{
using traits = function_traits<decltype(&Func::operator())>;
using args_tuple = typename traits::args_tuple;
using activity =
decltype(concat_tuples(InputActivityTuple{}, OutputActivityTuple{}));
static constexpr size_t arity = traits::arity;
static constexpr size_t num_inputs = tuple_size<InputActivityTuple>::value;
static_assert(std::is_void_v<typename traits::primal_return_type>,
"RevDiff only supports primal functions with void return type");
static_assert(tuple_size<activity>::value == arity,
"Number of input and output activity tags must match function "
"arity");
template <size_t I>
static constexpr bool is_active =
std::is_same_v<tuple_element_t<I, activity>, Active>;
// Index of the single Active tag in [Lo, Hi), or arity if not exactly one.
template <size_t Lo, size_t Hi, size_t... Is>
static constexpr size_t find_single_active(std::index_sequence<Is...>)
{
size_t idx = arity, count = 0;
(((Is >= Lo && Is < Hi && is_active<Is>) ? (idx = Is, ++count) : size_t{0}),
...);
return count == 1 ? idx : arity;
}
// Number of Active inputs and their argument indices, in ascending order.
// A qfunction may have several Active inputs at once: e.g. a field's value
// u and its gradient dudx both feed the output and both must be
// differentiated (the chain-rule contraction with the value/gradient shape
// functions then happens at the FE-operator level). We produce one gradient
// block, d(output)/d(input), per Active input — each computed with the other
// Active inputs frozen, so they come out as isolated partials.
template <size_t... Is>
static constexpr size_t count_active_inputs(std::index_sequence<Is...>)
{
return ((Is < num_inputs && is_active<Is> ? size_t{1} : size_t{0}) + ...);
}
static constexpr size_t num_active_inputs =
count_active_inputs(std::make_index_sequence<arity> {});
template <size_t... Is>
static constexpr std::array<size_t, num_active_inputs>
collect_active_inputs(std::index_sequence<Is...>)
{
std::array<size_t, num_active_inputs> idx{};
size_t j = 0;
(((Is < num_inputs && is_active<Is>) ? (idx[j++] = Is) : size_t{0}), ...);
return idx;
}
static constexpr auto active_inputs =
collect_active_inputs(std::make_index_sequence<arity> {});
// Slot index of argument I in the active_inputs array (compile-time).
template <size_t I>
static constexpr size_t slot_of = []() constexpr -> size_t
{
for (size_t s = 0; s < num_active_inputs; s++)
if (active_inputs[s] == I) { return s; }
return num_active_inputs;
}();
static constexpr size_t active_output =
find_single_active<num_inputs, arity>(std::make_index_sequence<arity> {});
static_assert(active_output < arity,
"gradient mode requires exactly one Active output");
static_assert(num_active_inputs >= 1,
"gradient mode requires at least one Active input");
static_assert(tuple_size<OutputActivityTuple>::value == 1,
"gradient mode requires exactly one (scalar) output");
using output_type =
std::decay_t<tuple_element_t<active_output, args_tuple>>;
using output_view = typename qp_traits<output_type>::view_type;
static_assert(qp_traits<output_type>::components == 1,
"gradient output requires a scalar output");
template <size_t I>
using primal_arg_t = tuple_element_t<I, args_tuple>;
template <size_t S>
using grad_arg_t =
typename qp_traits<std::decay_t<tuple_element_t<active_inputs[S], args_tuple>>>::view_type
&;
template <size_t... Is, size_t... Ss>
static FunctionSignature<void(primal_arg_t<Is>..., grad_arg_t<Ss>...)>
signature_impl(std::index_sequence<Is...>, std::index_sequence<Ss...>);
using signature = decltype(signature_impl(std::make_index_sequence<num_inputs> {},
std::make_index_sequence<num_active_inputs> {}));
// Plain function with the qfunction's exact (reference) signature so it can
// be handed to Enzyme as a function pointer.
template <size_t... Is>
MFEM_HOST_DEVICE static __attribute__((always_inline)) void
static_call(tuple_element_t<Is, args_tuple>... args)
{
Func{}(args...);
}
template <size_t... Is>
static constexpr auto fn_ptr(std::index_sequence<Is...>)
{
return &static_call<Is...>;
}
static constexpr auto fn = fn_ptr(std::make_index_sequence<arity> {});
// Recursive builder of the per-argument reverse-mode enzyme call.
template <size_t I = 0, typename AllPtrs, typename... Built>
MFEM_HOST_DEVICE static __attribute__((always_inline)) void
call_enzyme_rev(AllPtrs &ptrs, output_view &scratch, output_view &adjoint,
Built... built)
{
if constexpr (I == arity)
{
__enzyme_autodiff<void>(fn, built...);
}
else if constexpr (I == active_output)
{
// Output: primal written to scratch (unused), adjoint seeded to 1.
call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
enzyme_dupnoneed, &scratch, &adjoint);
}
else if constexpr (is_active<I>)
{
// Active input: gradient accumulates into its grad-output slot.
call_enzyme_rev<I + 1>(
ptrs, scratch, adjoint, built..., enzyme_dup,
mfem::future::get<int(I)>(ptrs),
mfem::future::get<int(num_inputs + slot_of<I>)>(ptrs));
}
else
{
// Const input: primal only, no shadow.
call_enzyme_rev<I + 1>(ptrs, scratch, adjoint, built...,
enzyme_const, mfem::future::get<int(I)>(ptrs));
}
}
// Zero all gradient outputs before the enzyme call (Enzyme accumulates).
template <typename AllPtrs, size_t... Ss>
MFEM_HOST_DEVICE static
__attribute__((always_inline)) void zero_grads(
AllPtrs &ptrs,
std::index_sequence<Ss...>)
{
((*mfem::future::get<int(num_inputs + Ss)>(ptrs) =
std::decay_t<decltype(*mfem::future::get<int(num_inputs + Ss)>(ptrs))> {}),
...);
}
// Called once per quadrature point. Arguments are, in order:
// * the primal value of every qfunction input (active and const), then
// * one gradient output per Active input (ascending index order), each
// shaped like its Active input.
// The qfunction's own output slot is not passed; Enzyme writes it to stack
// scratch (enzyme_dupnoneed). A single __enzyme_autodiff call yields all
// gradient blocks simultaneously.
template <typename... Args>
MFEM_HOST_DEVICE __attribute__((always_inline)) void operator()(
Args &&...args) const
{
static_assert(sizeof...(Args) == num_inputs + num_active_inputs,
"expected one primal per input plus one gradient output per "
"Active input");
auto ptrs = mfem::future::make_tuple(&args...);
zero_grads(ptrs, std::make_index_sequence<num_active_inputs> {});
output_view out_scratch{};
output_view out_adjoint{1.0}; // seed: d(output)/d(output) = 1
call_enzyme_rev(ptrs, out_scratch, out_adjoint);
}
static __attribute__((always_inline)) void print() { print_impl(std::make_index_sequence<arity> {}); }
template <size_t... Is> static __attribute__((always_inline)) void print_impl(
std::index_sequence<Is...>)
{
mfem::out << "__enzyme_autodiff<void>(fptr";
(([&]
{
auto name = get_type_name<tuple_element_t<Is, args_tuple>>();
if constexpr (Is == active_output)
mfem::out << ", enzyme_dupnoneed, " << name << ", adjoint=1";
else if constexpr (is_active<Is>)
mfem::out << ", enzyme_dup, " << name << ", grad out";
else
{
mfem::out << ", enzyme_const, " << name;
}
}()),
...);
mfem::out << ")\n";
}
};
template <typename Func, typename InputActivityTuple, typename OutputActivityTuple>
struct create_function_signature<RevDiff<Func, InputActivityTuple, OutputActivityTuple>>
{
using type = typename
RevDiff<Func, InputActivityTuple, OutputActivityTuple>::signature;
};
} // namespace mfem::future
#endif // MFEM_USE_ENZYME
-622
View File
@@ -1,622 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
// Compile-time quadrature argument metadata for local q-functions
#include "../../../../linalg/tensor.hpp"
#include "../../integrator_ctx.hpp"
#include "../../util.hpp"
#include <algorithm>
#include <array>
#include <cstddef>
#include <type_traits>
namespace mfem::future
{
///////////////////////////////////////////////////////////////////////////////
template <typename T>
MFEM_HOST_DEVICE auto qf_store_value(const T &v)
{
if constexpr (is_dual_number<T>::value) { return v.value; }
else { return v; }
}
template <typename T>
MFEM_HOST_DEVICE auto qf_store_gradient(const T &v)
{
if constexpr (is_dual_number<T>::value) { return v.gradient; }
else { return v; }
}
///////////////////////////////////////////////////////////////////////////////
/// True when quadrature-point values of `T` carry dual-number derivatives
template <typename T>
struct qf_param_uses_dual : std::false_type {};
template <typename S, int... Is>
struct qf_param_uses_dual<tensor<S, Is...>> : is_dual_number<S> {};
template <typename V, typename G>
struct qf_param_uses_dual<dual<V, G>> : std::true_type {};
template <typename T>
constexpr bool qf_param_uses_dual_v = qf_param_uses_dual<T>::value;
///////////////////////////////////////////////////////////////////////////////
/// Static shape for one decayed q-function parameter type
template <typename T>
struct qf_param_shape
{
static constexpr int rank = 0;
static constexpr std::array<int, 0> extents {};
};
template <typename scalar_t, int... Is>
struct qf_param_shape<tensor<scalar_t, Is...>>
{
static constexpr int rank = sizeof...(Is);
static constexpr std::array<int, sizeof...(Is)> extents {{Is...}};
};
template <typename scalar_t>
struct qf_param_shape<tensor<scalar_t>>
{
static constexpr int rank = 0;
static constexpr std::array<int, 0> extents {};
};
template <>
struct qf_param_shape<real_t>
{
static constexpr int rank = 0;
static constexpr std::array<int, 0> extents {};
};
///////////////////////////////////////////////////////////////////////////////
/// Type used in quadrature registers for parameter
template <typename T>
struct qf_reg_t { using type = T; };
template <>
struct qf_reg_t<real_t> { using type = tensor<real_t>; };
///////////////////////////////////////////////////////////////////////////////
/// Per-parameter tensor info for slot `I` in the decayed q-function parameter tuple
template <typename qfunc_t, std::size_t I>
struct qf_param_slot
{
using qf_signature = typename get_function_signature<qfunc_t>::type;
using qf_param_ts = typename qf_signature::parameter_ts;
using qf_raw_param_t = typename tuple_element<I, qf_param_ts>::type;
using qf_decay_param_t =
std::remove_cv_t<std::remove_reference_t<qf_raw_param_t>>;
using qf_reg_param_t = typename qf_reg_t<qf_decay_param_t>::type;
static constexpr auto extents = qf_param_shape<qf_decay_param_t>::extents;
};
///////////////////////////////////////////////////////////////////////////////
/// Builds a register-bank tuple covering the q-function parameter slots
/// `[K0, N)`. `K` is the recursion cursor and starts at `K0`; the resulting
/// tuple is indexed from 0, so a bank starting at `K0 > 0` has its slot indices
/// rebased by `-K0` relative to the q-function parameter list.
template <
typename backend_t,
typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
std::size_t K0, std::size_t K, std::size_t N, typename... Acc>
struct build_args_reg_tuple_impl;
template <
typename backend_t,
typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
std::size_t K0, std::size_t N, typename... Acc>
struct build_args_reg_tuple_impl<backend_t, qfunc_t, inputs_t, outputs_t, MQ1, K0, N, N, Acc...>
{
using type = tuple<Acc...>;
static_assert(sizeof...(Acc) == N - K0);
static_assert(sizeof...(Acc) <= 9);
};
template <
typename backend_t,
typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1,
std::size_t K0, std::size_t K, std::size_t N, typename... Acc>
struct build_args_reg_tuple_impl
{
using qf_reg_param_t = typename qf_param_slot<qfunc_t, K>::qf_reg_param_t;
using R = typename backend_t::template QReg<qf_reg_param_t>;
using type = typename build_args_reg_tuple_impl<backend_t, qfunc_t, inputs_t,
outputs_t, MQ1, K0, K + 1, N, Acc..., R>::type;
};
template <
typename backend_t,
typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
using args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
inputs_t, outputs_t, MQ1, 0, 0,
tuple_size<inputs_t>::value + tuple_size<outputs_t>::value>::type;
/// Register bank covering q-function inputs only (same types as first
/// `n_inputs` slots of args_reg_t). Used where shadow / tangent paths never
/// touch output parameter registers.
template <
typename backend_t,
typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
using input_args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
inputs_t, outputs_t, MQ1, 0, 0,
tuple_size<inputs_t>::value>::type;
/// Register bank covering q-function outputs only (same types as the slots
/// from `n_inputs` onward in args_reg_t). Used where the primal / trial inputs
/// live in a separate bank and only the test registers are integrated.
/// Slot `o` of this bank is q-function parameter `n_inputs + o`.
template <
typename backend_t,
typename qfunc_t, typename inputs_t, typename outputs_t, int MQ1>
using output_args_reg_t = typename build_args_reg_tuple_impl<backend_t, qfunc_t,
inputs_t, outputs_t, MQ1,
tuple_size<inputs_t>::value,
tuple_size<inputs_t>::value,
tuple_size<inputs_t>::value + tuple_size<outputs_t>::value>::type;
template <typename ARG>
MFEM_HOST_DEVICE inline void qf_set_flat_value(ARG &a, int c, real_t v)
{
if constexpr (std::is_same_v<ARG, real_t>) { MFEM_CONTRACT_VAR(c); a = v; }
else if constexpr (is_dual_number<ARG>::value)
{
MFEM_CONTRACT_VAR(c);
a.value = v;
}
else
{
constexpr int RNK = qf_param_shape<ARG>::rank;
constexpr bool D = qf_param_uses_dual_v<ARG>;
if constexpr (RNK == 0)
{
if constexpr (D) { a(0).value = v; }
else { a(0) = v; }
}
else if constexpr (RNK == 1)
{
if constexpr (D) { a(c).value = v; }
else { a(c) = v; }
}
else
{
constexpr int e0 = qf_param_shape<ARG>::extents[0];
if constexpr (D) { a(c % e0, c / e0).value = v; }
else { a(c % e0, c / e0) = v; }
}
}
}
template <typename ARG>
MFEM_HOST_DEVICE inline void qf_set_flat_gradient(ARG &a, int c, real_t v)
{
if constexpr (is_dual_number<ARG>::value)
{
MFEM_CONTRACT_VAR(c);
a.gradient = v;
}
else if constexpr (qf_param_uses_dual_v<ARG>)
{
constexpr int RNK = qf_param_shape<ARG>::rank;
if constexpr (RNK == 0) { a(0).gradient = v; }
else if constexpr (RNK == 1) { a(c).gradient = v; }
else
{
constexpr int e0 = qf_param_shape<ARG>::extents[0];
a(c % e0, c / e0).gradient = v;
}
}
else
{
// Non-dual argument (e.g. Weight): never an active trial direction.
MFEM_CONTRACT_VAR(a);
MFEM_CONTRACT_VAR(c);
MFEM_CONTRACT_VAR(v);
}
}
///////////////////////////////////////////////////////////////////////////////
/// Two-index component access for a q-function argument.
///
/// `i` indexes vdim, `k` indexes the operator dimension, matching the
/// column-major packing `c = i + extents[0]*k` of the flat accessors above.
/// Prefer these wherever the caller already has both indices: the flat form
/// would have to undo the packing with an integer division, which is expensive
/// on device and pointless when `(i, k)` are right there.
///
/// The runtime extents of the callers agree with the static extents of `ARG`:
/// for rank 2, `vdim == extents[0]` and `op_dim == extents[1]`; for rank 1 one
/// of the two is 1 and the corresponding index is always 0, so `a(i + k)`
/// selects the right component; for rank 0 both are 1.
template <typename ARG>
MFEM_HOST_DEVICE inline real_t qf_value_at(const ARG &a, int i, int k)
{
if constexpr (std::is_same_v<ARG, real_t> || is_dual_number<ARG>::value)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
return qf_store_value(a);
}
else
{
constexpr int RNK = qf_param_shape<ARG>::rank;
if constexpr (RNK == 0)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
return qf_store_value(a(0));
}
else if constexpr (RNK == 1) { return qf_store_value(a(i + k)); }
else { return qf_store_value(a(i, k)); }
}
}
template <typename ARG>
MFEM_HOST_DEVICE inline real_t qf_gradient_at(const ARG &a, int i, int k)
{
if constexpr (is_dual_number<ARG>::value)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
return a.gradient;
}
else if constexpr (qf_param_uses_dual_v<ARG>)
{
constexpr int RNK = qf_param_shape<ARG>::rank;
if constexpr (RNK == 0)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
return a(0).gradient;
}
else if constexpr (RNK == 1) { return a(i + k).gradient; }
else { return a(i, k).gradient; }
}
else
{
// Non-dual argument carries no tangent: its derivative contribution is 0.
MFEM_CONTRACT_VAR(a);
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
return real_t(0);
}
}
template <typename ARG>
MFEM_HOST_DEVICE inline void qf_set_value_at(ARG &a, int i, int k, real_t v)
{
if constexpr (std::is_same_v<ARG, real_t>)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
a = v;
}
else if constexpr (is_dual_number<ARG>::value)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
a.value = v;
}
else
{
constexpr int RNK = qf_param_shape<ARG>::rank;
constexpr bool D = qf_param_uses_dual_v<ARG>;
if constexpr (RNK == 0)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
if constexpr (D) { a(0).value = v; }
else { a(0) = v; }
}
else if constexpr (RNK == 1)
{
if constexpr (D) { a(i + k).value = v; }
else { a(i + k) = v; }
}
else
{
if constexpr (D) { a(i, k).value = v; }
else { a(i, k) = v; }
}
}
}
template <typename ARG>
MFEM_HOST_DEVICE inline void qf_set_gradient_at(ARG &a, int i, int k, real_t v)
{
if constexpr (is_dual_number<ARG>::value)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
a.gradient = v;
}
else if constexpr (qf_param_uses_dual_v<ARG>)
{
constexpr int RNK = qf_param_shape<ARG>::rank;
if constexpr (RNK == 0)
{
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
a(0).gradient = v;
}
else if constexpr (RNK == 1) { a(i + k).gradient = v; }
else { a(i, k).gradient = v; }
}
else
{
// Non-dual argument (e.g. Weight): never an active trial direction.
MFEM_CONTRACT_VAR(a);
MFEM_CONTRACT_VAR(i);
MFEM_CONTRACT_VAR(k);
MFEM_CONTRACT_VAR(v);
}
}
///////////////////////////////////////////////////////////////////////////////
/// Maps each FOP slot to unionfds indices — used with dtqs / create_dtq_maps
template<typename C, typename T>
const auto create_union_field_map_for_dtq(C& ctx, T& io)
{
using FE = Entity::Element;
return create_descriptors_to_fields_map<FE>(ctx.unionfds, io);
}
/// **`xe[i]`** slot per input FOP — indices into **`ctx.infds`** (`SIZE_MAX` for Weight).
template<typename C, typename T>
const auto create_input_vector_map(C& ctx, T& io)
{
using FE = Entity::Element;
return create_descriptors_to_fields_map<FE>(ctx.infds, io);
}
/// **`ye[i]`** slot per output FOP — indices into **`ctx.outfds`**.
template<typename C, typename T>
const auto create_output_vector_map(C& ctx, T& io)
{
using FE = Entity::Element;
return create_descriptors_to_fields_map<FE>(ctx.outfds, io);
}
template<typename C>
const auto make_dtqs(C& ctx)
{
std::vector<const DofToQuad*> dtq_vec;
dtq_vec.reserve(ctx.unionfds.size());
constexpr auto dtq_mode = DofToQuad::Mode::TENSOR;
for (const auto &field: ctx.unionfds)
{
auto dtq = GetDofToQuad<Entity::Element>(field, ctx.ir, dtq_mode);
dtq_vec.emplace_back(dtq);
}
return dtq_vec;
}
///////////////////////////////////////////////////////////////////////////////
template<typename Tuple>
constexpr auto get_vdim(const Tuple& fields)
{
return future::apply([](const auto&... f)
{
return std::array<int, sizeof...(f)> {f.vdim...};
}, fields);
}
template<typename Tuple>
constexpr auto get_B(const Tuple& fields)
{
return future::apply([](const auto&... f)
{
return std::array<const real_t*, sizeof...(f)> {f.B...};
}, fields);
}
template<typename Tuple>
constexpr auto get_G(const Tuple& fields)
{
return future::apply([](const auto&... f)
{
return std::array<const real_t*, sizeof...(f)> {f.G...};
}, fields);
}
template<typename Tuple>
constexpr auto get_D1D(const Tuple& fields)
{
return future::apply([](const auto&... f)
{
return std::array<int, sizeof...(f)> {f.B.GetShape()[2]...};
}, fields);
}
template<typename Tuple>
constexpr auto get_Q1D(const Tuple& fields)
{
return future::apply([](const auto&... f)
{
return std::array<int, sizeof...(f)> {f.B.GetShape()[0]...};
}, fields);
}
///////////////////////////////////////////////////////////////////////////////
/// Per-output FOP layout metadata (shared by derivative setup / apply kernels).
template<typename outputs_t>
constexpr auto compute_out_qp_size(const outputs_t &outs)
{
constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
std::array<int, n_outputs> sizes{};
for_constexpr<n_outputs>([&](auto o) { sizes[o] = get<o>(outs).size_on_qp; });
return sizes;
}
template<typename outputs_t>
constexpr auto compute_out_op_dim(const outputs_t &outs)
{
constexpr std::size_t n_outputs = tuple_size<outputs_t>::value;
std::array<int, n_outputs> op{};
for_constexpr<n_outputs>([&](auto o)
{
op[o] = get<o>(outs).size_on_qp / get<o>(outs).vdim;
});
return op;
}
template<std::size_t N>
constexpr std::array<int, N> compute_out_offsets(
const std::array<int, N> &vdim,
const std::array<int, N> &op_dim)
{
std::array<int, N> offsets{};
offsets[0] = 0;
for (std::size_t o = 1; o < N; o++)
{
offsets[o] = offsets[o - 1] + vdim[o - 1] * op_dim[o - 1];
}
return offsets;
}
template<std::size_t N>
constexpr std::array<int, N> compute_out_flat_offsets(
const std::array<int, N> &vdim,
const std::array<int, N> &op_dim,
const int num_qp)
{
std::array<int, N> offsets{};
offsets[0] = 0;
for (std::size_t o = 1; o < N; o++)
{
offsets[o] = offsets[o - 1] + vdim[o - 1] * op_dim[o - 1] * num_qp;
}
return offsets;
}
template<typename inputs_t>
const auto compute_input_is_dependent(const inputs_t &ins, int deriv_id)
{
auto dependency_map = make_dependency_map(ins);
auto it = dependency_map.find(deriv_id);
MFEM_ASSERT(it != dependency_map.end(),
"Derivative ID not found in dependency map");
return it->second;
}
template<typename inputs_t>
constexpr int compute_trial_vdim(const inputs_t &ins, int deriv_id)
{
constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
int v = 1;
for_constexpr<n_inputs>([&](auto i)
{
if (get<i>(ins).GetFieldId() == deriv_id) { v = get<i>(ins).vdim; }
});
return v;
}
template<typename inputs_t>
constexpr int compute_total_trial_op_dim(
const inputs_t &ins,
const std::array<bool, tuple_size<inputs_t>::value> &dep,
const std::array<int, tuple_size<inputs_t>::value> &size_on_qp)
{
constexpr std::size_t n_inputs = tuple_size<inputs_t>::value;
int total = 0;
for_constexpr<n_inputs>([&](auto i)
{
if (dep[i]) { total += size_on_qp[i] / get<i>(ins).vdim; }
});
return total;
}
inline size_t find_union_field_index(const IntegratorContext &ctx, int field_id)
{
for (size_t uf = 0; uf < ctx.unionfds.size(); uf++)
{
if (static_cast<int>(ctx.unionfds[uf].id) == field_id) { return uf; }
}
return SIZE_MAX;
}
inline size_t find_infd_index(const IntegratorContext &ctx, int field_id)
{
for (size_t i = 0; i < ctx.infds.size(); i++)
{
if (static_cast<int>(ctx.infds[i].id) == field_id) { return i; }
}
return SIZE_MAX;
}
template<typename entity_t = Entity::Element>
inline int compute_element_dof_sz(
const FieldDescriptor &fd,
int num_entities,
ElementDofOrdering ordering)
{
auto R = get_restriction<entity_t>(fd, ordering);
MFEM_ASSERT(R != nullptr, "LocalQF: missing element restriction");
return num_entities ? (R->Height() / num_entities) : 0;
}
// ────────────────────────────────────────────────────────────────────────────
// Number of threads per 1D direction to launch the kernel with
template <typename inputs_t, typename outputs_t,
std::size_t N_in, std::size_t N_out>
inline int compute_kernel_thread_1d(
const int q1d,
const std::array<int, N_in> &in_d1d,
const std::array<int, N_out> &out_d1d)
{
int t1d = q1d;
for_constexpr<N_in>([&](auto ic)
{
using FOP = tuple_element_t<ic.value, inputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
t1d = std::max(t1d, in_d1d[ic.value]);
}
});
for_constexpr<N_out>([&](auto ic)
{
using FOP = tuple_element_t<ic.value, outputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
t1d = std::max(t1d, out_d1d[ic.value]);
}
});
return t1d;
}
// Inputs-only variant: used by kernels whose outputs are written at qp
template <typename inputs_t, std::size_t N_in>
inline int compute_kernel_thread_1d(
const int q1d,
const std::array<int, N_in> &in_d1d)
{
int t1d = q1d;
for_constexpr<N_in>([&](auto ic)
{
using FOP = tuple_element_t<ic.value, inputs_t>;
if constexpr (is_value_fop_v<FOP> || is_gradient_fop_v<FOP>)
{
t1d = std::max(t1d, in_d1d[ic.value]);
}
});
return t1d;
}
} // namespace mfem::future
-277
View File
@@ -1,277 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "mfem.hpp"
#include "../tuple.hpp"
#include <memory>
#include <type_traits>
#include <vector>
namespace mfem::future
{
// Scratch storage and q-function shadow helpers for dFEM backends. The bank
// supports two scratch kinds:
// - quadrature-point scratch: real_t buffers sized as NQ * components_per_qp,
// - global scratch: one tuple of qfunction-local temporaries, independent of
// NQ, used for values such as flags, scalars, or small Vector workspaces.
template <typename... GlobalScratchTypes>
struct ScratchBank
{
//=================================
///<--- Global scratch utilities.
//=================================
using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
template <typename T>
static T MakeGlobalScratchShadow(const T &)
{
return T {};
}
static Vector MakeGlobalScratchShadow(const Vector &primal)
{
Vector shadow(primal.Size());
shadow.UseDevice(true);
shadow = 0.0;
return shadow;
}
template <typename Tuple, size_t... Is>
static auto MakeGlobalScratchShadowTuple(const Tuple &primal,
std::index_sequence<Is...>)
{
return make_tuple(MakeGlobalScratchShadow(get<Is>(primal))...);
}
template <typename Tuple>
static auto MakeGlobalScratchShadowTuple(const Tuple &primal)
{
return MakeGlobalScratchShadowTuple(
primal, std::make_index_sequence<tuple_size<Tuple>::value> {});
}
//===========================
///<--- Scratch objects
//===========================
mutable GlobalScratchTuple global;
int nq = 0;
std::vector<int> components;
std::vector<int> sizes;
std::vector<std::shared_ptr<Vector>> owned;
std::vector<real_t *> ptrs;
//===========================
///<--- Setter methods
//===========================
void SetScratch(const int nq_,
std::initializer_list<int> components_per_qp = {1})
{
SetScratch(nq_, std::vector<int>(components_per_qp));
}
void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
{
nq = nq_;
components.clear();
sizes.clear();
owned.clear();
ptrs.clear();
for (int component_count : components_per_qp)
{
AddScratch(component_count);
}
}
void AddScratch(const int components_per_qp = 1)
{
MFEM_VERIFY(nq > 0, "SetScratch must be called before AddScratch");
MFEM_VERIFY(components_per_qp > 0,
"scratch components per quadrature point must be positive");
owned.push_back(std::make_shared<Vector>());
Vector &scratch = *owned.back();
const int size = components_per_qp * nq;
scratch.SetSize(size);
scratch.UseDevice(true);
scratch = 0.0;
components.push_back(components_per_qp);
sizes.push_back(scratch.Size());
ptrs.push_back(scratch.ReadWrite());
}
void SetGlobalScratch(const GlobalScratchTuple &global_)
{
global = global_;
}
//===========================
///<--- Getter methods
//===========================
real_t *GetScratchPointer(const int i) const { return ptrs[i]; }
real_t *operator[](const int i) const { return ptrs[i]; }
Vector &GetScratchVector(const int i) const { return *owned[i]; }
template <int I>
auto &GetGlobalScratch() const
{
return get<I>(global);
}
//===========================
///<--- Utils methods
//===========================
void CloneScratchLayoutTo(ScratchBank &shadow) const
{
shadow.SetScratch(nq, components);
shadow.SetGlobalScratch(MakeGlobalScratchShadowTuple(global));
}
int Size() const { return static_cast<int>(ptrs.size()); }
};
// Shared base for Q-functions that use ScratchBank and need a matching scratch
// shadow for forward differentiation.
template <typename... GlobalScratchTypes>
struct QFWithScratch
{
using GlobalScratchTuple = tuple<GlobalScratchTypes...>;
int nq = 0;
ScratchBank<GlobalScratchTypes...> scratch;
void SetScratch(const int nq_,
std::initializer_list<int> components_per_qp = {1})
{
nq = nq_;
scratch.SetScratch(nq, components_per_qp);
}
void SetScratch(const int nq_, const std::vector<int> &components_per_qp)
{
nq = nq_;
scratch.SetScratch(nq, components_per_qp);
}
void SetScratch(const int nq_, const int num_scratch_elem,
const int components_per_qp = 1)
{
nq = nq_;
scratch.SetScratch(nq,
std::vector<int>(num_scratch_elem, components_per_qp));
}
void SetGlobalScratch(const GlobalScratchTuple &global_scratch_)
{
scratch.SetGlobalScratch(global_scratch_);
}
Vector &GetScratchVector(const int i) const
{
return scratch.GetScratchVector(i);
}
real_t *GetScratchPointer(const int i) const
{
return scratch.GetScratchPointer(i);
}
template <int I>
auto &GetGlobalScratch() const
{
return scratch.template GetGlobalScratch<I>();
}
void CloneScratchLayoutTo(QFWithScratch &shadow) const
{
shadow.nq = nq;
scratch.CloneScratchLayoutTo(shadow.scratch);
}
QFWithScratch CreateShadow() const
{
QFWithScratch shadow;
CloneScratchLayoutTo(shadow);
return shadow;
}
};
using QFWithScratchType = QFWithScratch<>;
using QFWithGlobalScratchType = QFWithScratch<bool, real_t, Vector>;
namespace detail
{
template <typename T>
struct qfunc_uses_scratch
{
private:
template <typename... GlobalScratchTypes>
static std::true_type Test(const QFWithScratch<GlobalScratchTypes...> *);
static std::false_type Test(...);
public:
static constexpr bool value = decltype(Test(
static_cast<std::remove_cv_t<std::remove_reference_t<T>> *>(nullptr)))::value;
};
template <typename T>
inline constexpr bool qfunc_uses_scratch_v =
qfunc_uses_scratch<T>::value;
struct unused_qfunc_shadow { };
template <typename qfunc_t, bool uses_scratch>
struct qfunc_shadow_type
{
using type = unused_qfunc_shadow;
};
template <typename qfunc_t>
struct qfunc_shadow_type<qfunc_t, true>
{
using type = decltype(std::declval<const qfunc_t &>().CreateShadow());
};
template <typename qfunc_t>
using qfunc_shadow_t = typename qfunc_shadow_type<qfunc_t,
qfunc_uses_scratch_v<qfunc_t>>::type;
// Create a persistent q-function shadow if the q-function uses scratch, otherwise return an empty struct.
template <typename qfunc_t>
inline qfunc_shadow_t<qfunc_t> MakeQFunctionShadowStorage(
const qfunc_t &qfunc)
{
if constexpr (qfunc_uses_scratch_v<qfunc_t>)
{
return qfunc.CreateShadow();
}
else
{
MFEM_CONTRACT_VAR(qfunc);
return {};
}
}
} // namespace detail
}
File diff suppressed because it is too large Load Diff
+19 -362
View File
@@ -11,387 +11,44 @@
#include "doperator.hpp"
#include <algorithm>
#ifdef MFEM_USE_MPI
using namespace mfem;
using namespace mfem::future;
namespace
void DifferentiableOperator::SetParameters(std::vector<Vector *> p) const
{
int GetTotalTrueVSize(const std::vector<FieldDescriptor> &fds)
{
int size = 0;
for (const auto &fd : fds)
MFEM_ASSERT(parameters.size() == p.size(),
"number of parameters doesn't match descriptors");
for (size_t i = 0; i < parameters.size(); i++)
{
size += mfem::future::GetTrueVSize(fd);
p[i]->Read();
parameters_l[i] = *p[i];
}
return size;
}
using DerivativeActionMap =
std::map<size_t, std::vector<derivative_action_t>>;
using DerivativeSetupMap =
std::map<size_t, std::vector<derivative_setup_t>>;
using DerivativeFieldMap =
std::map<size_t, std::vector<FieldDescriptor>>;
using SparseAssemblyMap =
std::map<size_t,
std::vector<assemble_derivative_sparsematrix_callback_t>>;
using HypreAssemblyMap =
std::map<size_t,
std::vector<assemble_derivative_hypreparmatrix_callback_t>>;
using DiagonalAssemblyMap =
std::map<size_t, std::vector<assemble_diagonal_callback_t>>;
using SecondDerivativeActionMap =
std::map<second_derivative_key_t, std::vector<derivative_action_t>>;
using SecondDerivativeSetupMap =
std::map<second_derivative_key_t, std::vector<derivative_setup_t>>;
using SecondDerivativeFieldMap =
std::map<second_derivative_key_t, std::vector<FieldDescriptor>>;
using SecondSparseAssemblyMap =
std::map<second_derivative_key_t,
std::vector<assemble_derivative_sparsematrix_callback_t>>;
using SecondHypreAssemblyMap =
std::map<second_derivative_key_t,
std::vector<assemble_derivative_hypreparmatrix_callback_t>>;
using SecondDiagonalAssemblyMap =
std::map<second_derivative_key_t, std::vector<assemble_diagonal_callback_t>>;
template <typename map_t>
const typename map_t::mapped_type &FindOrDefault(
const map_t &map, const typename map_t::key_type &id,
const typename map_t::mapped_type &fallback)
{
const auto it = map.find(id);
return it == map.end() ? fallback : it->second;
}
template <typename map_t>
typename map_t::mapped_type FindOrEmpty(
const map_t &map, const typename map_t::key_type &id)
{
const auto it = map.find(id);
return it == map.end() ? typename map_t::mapped_type{} : it->second;
}
const std::vector<derivative_action_t> &SelectActionCallbacks(
const std::vector<derivative_action_t> &direct_actions,
const DerivativeActionMap &cached_actions,
size_t derivative_id,
bool use_cached_setup)
{
if (use_cached_setup)
{
const auto it_apply = cached_actions.find(derivative_id);
if (it_apply != cached_actions.end() && !it_apply->second.empty())
{
return it_apply->second;
}
}
return direct_actions;
}
struct DerivativeCallbackSet
{
const DerivativeActionMap &actions;
const DerivativeActionMap &cached_actions;
const DerivativeActionMap &transpose_actions;
const DerivativeFieldMap &outfds;
const SparseAssemblyMap &assemble_sparse;
const HypreAssemblyMap &assemble_hypre;
const DiagonalAssemblyMap &assemble_diagonal;
const DerivativeSetupMap &setup;
const char *missing_action_message;
};
struct SecondDerivativeCallbackSet
{
const SecondDerivativeActionMap &actions;
const SecondDerivativeActionMap &cached_actions;
const SecondDerivativeActionMap &transpose_actions;
const SecondDerivativeFieldMap &outfds;
const SecondSparseAssemblyMap &assemble_sparse;
const SecondHypreAssemblyMap &assemble_hypre;
const SecondDiagonalAssemblyMap &assemble_diagonal;
const SecondDerivativeSetupMap &setup;
const char *missing_action_message;
};
template <typename vector_t>
std::shared_ptr<DerivativeOperator> MakeStatefulDerivativeOperator(
size_t derivative_id,
const vector_t &x,
const std::vector<FieldDescriptor> &infds,
const std::vector<FieldDescriptor> &default_outfds,
const DerivativeCallbackSet &callbacks,
bool use_cached_setup,
bool lvector_mode,
bool functional_gradient = false)
{
const auto it_action = callbacks.actions.find(derivative_id);
MFEM_ASSERT(it_action != callbacks.actions.end(),
callbacks.missing_action_message << derivative_id);
const size_t dfidx = FindIdx(derivative_id, infds);
const auto &doutfds =
FindOrDefault(callbacks.outfds, derivative_id, default_outfds);
const auto &mult_callbacks =
SelectActionCallbacks(it_action->second, callbacks.cached_actions,
derivative_id, use_cached_setup);
return std::make_shared<DerivativeOperator>(
GetTotalTrueVSize(doutfds),
GetTrueVSize(infds[dfidx]),
mult_callbacks,
FindOrEmpty(callbacks.transpose_actions, derivative_id),
infds[dfidx],
x,
infds,
doutfds,
FindOrEmpty(callbacks.assemble_sparse, derivative_id),
FindOrEmpty(callbacks.assemble_hypre, derivative_id),
FindOrEmpty(callbacks.assemble_diagonal, derivative_id),
FindOrEmpty(callbacks.setup, derivative_id),
lvector_mode,
functional_gradient);
}
const std::vector<derivative_action_t> &SelectSecondDerivativeActionCallbacks(
const std::vector<derivative_action_t> &direct_actions,
const SecondDerivativeActionMap &cached_actions,
second_derivative_key_t derivative_key,
bool use_cached_setup)
{
if (use_cached_setup)
{
const auto it_apply = cached_actions.find(derivative_key);
if (it_apply != cached_actions.end() && !it_apply->second.empty())
{
return it_apply->second;
}
}
return direct_actions;
}
template <typename vector_t>
std::shared_ptr<DerivativeOperator> MakeStatefulSecondDerivativeOperator(
size_t gradient_id,
size_t direction_id,
const vector_t &x,
const std::vector<FieldDescriptor> &infds,
const std::vector<FieldDescriptor> &default_outfds,
const SecondDerivativeCallbackSet &callbacks,
bool use_cached_setup,
bool lvector_mode)
{
const second_derivative_key_t derivative_key{gradient_id, direction_id};
const auto it_action = callbacks.actions.find(derivative_key);
MFEM_ASSERT(it_action != callbacks.actions.end(),
callbacks.missing_action_message << "(" << gradient_id << ", "
<< direction_id << ")");
const size_t dfidx = FindIdx(direction_id, infds);
const auto &doutfds =
FindOrDefault(callbacks.outfds, derivative_key, default_outfds);
const auto &mult_callbacks =
SelectSecondDerivativeActionCallbacks(
it_action->second, callbacks.cached_actions, derivative_key,
use_cached_setup);
return std::make_shared<DerivativeOperator>(
GetTotalTrueVSize(doutfds),
GetTrueVSize(infds[dfidx]),
mult_callbacks,
FindOrEmpty(callbacks.transpose_actions, derivative_key),
infds[dfidx],
x,
infds,
doutfds,
FindOrEmpty(callbacks.assemble_sparse, derivative_key),
FindOrEmpty(callbacks.assemble_hypre, derivative_key),
FindOrEmpty(callbacks.assemble_diagonal, derivative_key),
FindOrEmpty(callbacks.setup, derivative_key),
lvector_mode);
}
}
DifferentiableOperator::DifferentiableOperator(
const std::vector<FieldDescriptor> &infds,
const std::vector<FieldDescriptor> &outfds,
const std::vector<FieldDescriptor> &solutions,
const std::vector<FieldDescriptor> &parameters,
const ParMesh &mesh) :
Operator(),
mesh(mesh),
infds(infds),
outfds(outfds)
solutions(solutions),
parameters(parameters)
{
unionfds.clear();
unionfds.insert(unionfds.end(), infds.begin(), infds.end());
unionfds.insert(unionfds.end(), outfds.begin(), outfds.end());
std::sort(unionfds.begin(), unionfds.end());
auto last = std::unique(unionfds.begin(), unionfds.end());
unionfds.erase(last, unionfds.end());
fields.resize(solutions.size() + parameters.size());
fields_e.resize(fields.size());
solutions_l.resize(solutions.size());
parameters_l.resize(parameters.size());
infields_l.resize(infds.size());
for (size_t i = 0; i < infds.size(); i++)
for (size_t i = 0; i < solutions.size(); i++)
{
infields_l[i] = new Vector(GetVSize(infds[i]));
fields[i] = solutions[i];
}
infields_e.resize(infds.size());
}
void DifferentiableOperator::SetMultLevel(MultLevel level)
{
mult_level = level;
}
void DifferentiableOperator::Mult(const Vector &x, Vector &y) const
{
MFEM_ASSERT(!action_callbacks.empty(),
"no integrators have been set");
MFEM_ASSERT(dynamic_cast<const BlockVector*>(&x),
"x needs to be a BlockVector");
MFEM_ASSERT(dynamic_cast<const BlockVector*>(&y),
"y needs to be a BlockVector");
const auto &bx = static_cast<const BlockVector &>(x);
auto &by = static_cast<BlockVector &>(y);
Mult(bx, by);
}
void DifferentiableOperator::DisableTensorProductStructure(bool disable)
{
use_tensor_product_structure = !disable;
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
size_t derivative_id, const Vector &x)
{
return MakeStatefulDerivativeOperator(
derivative_id, x, infds, outfds,
for (size_t i = 0; i < parameters.size(); i++)
{
derivative_action_callbacks,
derivative_apply_callbacks,
daction_transpose_callbacks,
derivative_outfds,
assemble_derivative_sparsematrix_callbacks,
assemble_derivative_hypreparmatrix_callbacks,
assemble_diagonal_callbacks,
derivative_setup_callbacks,
"no derivative action has been found for ID "
},
true,
mult_level == MultLevel::LVECTOR,
IsFunctionalDerivative(derivative_id));
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
{
return MakeStatefulDerivativeOperator(
derivative_id, x, infds, outfds,
{
derivative_action_callbacks,
derivative_apply_callbacks,
daction_transpose_callbacks,
derivative_outfds,
assemble_derivative_sparsematrix_callbacks,
assemble_derivative_hypreparmatrix_callbacks,
assemble_diagonal_callbacks,
derivative_setup_callbacks,
"no derivative action has been found for ID "
},
use_cached_setup,
mult_level == MultLevel::LVECTOR,
IsFunctionalDerivative(derivative_id));
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetDerivative(
size_t derivative_id)
{
MFEM_ASSERT(has_functional_integrator,
"stateless GetDerivative is available only for functionals");
const auto it_action = derivative_action_callbacks.find(derivative_id);
MFEM_ASSERT(it_action != derivative_action_callbacks.end(),
"no derivative action has been found for ID " << derivative_id);
const size_t dfidx = FindIdx(derivative_id, infds);
const auto &doutfds =
FindOrDefault(derivative_outfds, derivative_id, outfds);
return std::make_shared<DerivativeOperator>(
GetTotalTrueVSize(doutfds),
GetTrueVSize(infds[dfidx]),
it_action->second,
infds,
doutfds);
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
size_t derivative_id, const Vector &x)
{
return GetSecondDerivative(derivative_id, derivative_id, x);
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
size_t gradient_id, size_t direction_id, const Vector &x)
{
MFEM_ASSERT(has_functional_integrator,
"second derivatives are available only for functionals");
return MakeStatefulSecondDerivativeOperator(
gradient_id, direction_id, x, infds, outfds,
{
second_derivative_action_callbacks,
second_derivative_apply_callbacks,
second_daction_transpose_callbacks,
second_derivative_outfds,
assemble_second_derivative_sparsematrix_callbacks,
assemble_second_derivative_hypreparmatrix_callbacks,
assemble_second_derivative_diagonal_callbacks,
second_derivative_setup_callbacks,
"no second derivative action has been found for ID "
},
false,
mult_level == MultLevel::LVECTOR);
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
size_t derivative_id, const MultiVector &x, const bool use_cached_setup)
{
return GetSecondDerivative(derivative_id, derivative_id, x, use_cached_setup);
}
std::shared_ptr<DerivativeOperator> DifferentiableOperator::GetSecondDerivative(
size_t gradient_id, size_t direction_id, const MultiVector &x,
const bool use_cached_setup)
{
MFEM_ASSERT(has_functional_integrator,
"second derivatives are available only for functionals");
return MakeStatefulSecondDerivativeOperator(
gradient_id, direction_id, x, infds, outfds,
{
second_derivative_action_callbacks,
second_derivative_apply_callbacks,
second_daction_transpose_callbacks,
second_derivative_outfds,
assemble_second_derivative_sparsematrix_callbacks,
assemble_second_derivative_hypreparmatrix_callbacks,
assemble_second_derivative_diagonal_callbacks,
second_derivative_setup_callbacks,
"no second derivative action has been found for ID "
},
use_cached_setup,
mult_level == MultLevel::LVECTOR);
fields[i + solutions.size()] = parameters[i];
}
}
#endif // MFEM_USE_MPI
+1142 -1046
View File
File diff suppressed because it is too large Load Diff
-64
View File
@@ -1,64 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "../fespace.hpp"
#include "../qspace.hpp"
#include "parameterspace.hpp"
namespace mfem::future
{
/// @brief FieldDescriptor struct
///
/// This struct is used to store information about a field.
struct FieldDescriptor
{
using data_variant_t =
std::variant<const FiniteElementSpace *,
const ParFiniteElementSpace *,
const VectorQuadratureSpace *,
const ParameterSpace *>;
/// Field ID
std::size_t id;
/// Field variant
data_variant_t data;
/// Default constructor
FieldDescriptor() :
id(SIZE_MAX), data(data_variant_t{}) {}
/// Constructor
template <typename T>
FieldDescriptor(std::size_t field_id, const T* v) :
id(field_id), data(v) {}
bool operator==(const FieldDescriptor& other) const
{
return id == other.id;
}
bool operator<(const FieldDescriptor& other) const
{
return id < other.id;
}
friend void swap(FieldDescriptor& a, FieldDescriptor& b)
{
using std::swap;
swap(a.id, b.id);
swap(a.data, b.data);
}
};
}
+2 -73
View File
@@ -10,7 +10,6 @@
// CONTRIBUTING.md for details.
#pragma once
#include <ostream>
#include <type_traits>
namespace mfem::future
@@ -75,15 +74,6 @@ struct is_identity_fop : std::false_type {};
template <int FIELD_ID>
struct is_identity_fop<Identity<FIELD_ID>> : std::true_type {};
template <typename T>
constexpr bool is_identity_fop_v = is_identity_fop<T>::value;
template <int FIELD_ID>
inline std::ostream& operator<<(std::ostream& out, Identity<FIELD_ID>)
{
return out << "Identity<" << FIELD_ID << ">";
}
/// @brief Weight FieldOperator.
///
/// This FieldOperator is used to signal that this field contains the quadrature
@@ -100,14 +90,6 @@ struct is_weight_fop : std::false_type {};
template <>
struct is_weight_fop<Weight> : std::true_type {};
template <typename T>
constexpr bool is_weight_fop_v = is_weight_fop<T>::value;
inline std::ostream& operator<<(std::ostream& out, Weight)
{
return out << "Weight";
}
/// @brief Value FieldOperator.
///
/// This FieldOperator is used to signal that the field contains the
@@ -119,20 +101,11 @@ public:
constexpr Value() : FieldOperator<FIELD_ID>() {};
};
template <typename T>
template< typename T >
struct is_value_fop : std::false_type {};
template <int T>
struct is_value_fop<Value<T>> : std::true_type {};
template <typename T>
constexpr bool is_value_fop_v = is_value_fop<T>::value;
template <int FIELD_ID>
inline std::ostream& operator<<(std::ostream& out, Value<FIELD_ID>)
{
return out << "Value<" << FIELD_ID << ">";
}
struct is_value_fop<Value<FIELD_ID>> : std::true_type {};
/// @brief Gradient FieldOperator.
///
@@ -151,15 +124,6 @@ struct is_gradient_fop : std::false_type {};
template <int FIELD_ID>
struct is_gradient_fop<Gradient<FIELD_ID>> : std::true_type {};
template <typename T>
constexpr bool is_gradient_fop_v = is_gradient_fop<T>::value;
template <int FIELD_ID>
inline std::ostream& operator<<(std::ostream& out, Gradient<FIELD_ID>)
{
return out << "Gradient<" << FIELD_ID << ">";
}
/// @brief Sum FieldOperator.
///
/// This FieldOperator is commonly used to signal that an output of a quadrature
@@ -177,39 +141,4 @@ struct is_sum_fop : std::false_type {};
template <int FIELD_ID>
struct is_sum_fop<Sum<FIELD_ID>> : std::true_type {};
template <typename T>
constexpr bool is_sum_fop_v = is_sum_fop<T>::value;
template <int FIELD_ID>
inline std::ostream& operator<<(std::ostream& out, Sum<FIELD_ID>)
{
return out << "Sum<" << FIELD_ID << ">";
}
/// @brief FunctionalValue FieldOperator.
///
/// This FieldOperator is commonly used to signal that an output of a quadrature
/// function should be summed.
template <int FIELD_ID = -1>
class FunctionalValue : public FieldOperator<FIELD_ID>
{
public:
constexpr FunctionalValue() : FieldOperator<FIELD_ID>() {};
};
template< typename T >
struct is_functionalvalue_fop : std::false_type {};
template <int FIELD_ID>
struct is_functionalvalue_fop<FunctionalValue<FIELD_ID>> : std::true_type {};
template <typename T>
constexpr bool is_functionalvalue_fop_v = is_functionalvalue_fop<T>::value;
template <int FIELD_ID>
inline std::ostream& operator<<(std::ostream& out, FunctionalValue<FIELD_ID>)
{
return out << "FunctionalValue<" << FIELD_ID << ">";
}
} // namespace mfem::future
+594
View File
@@ -0,0 +1,594 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "util.hpp"
namespace mfem::future
{
template <typename output_t>
MFEM_HOST_DEVICE
void map_quadrature_data_to_fields_impl(
DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const output_t &output,
const DofToQuadMap &dtq)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
// assuming the quadrature point residual has to "play nice with
// the test function"
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
{
const auto [num_qp, cdim, num_dof] = B.GetShape();
const int vdim = output.vdim > 0 ? output.vdim : cdim ;
for (int dof = 0; dof < num_dof; dof++)
{
for (int vd = 0; vd < vdim; vd++)
{
real_t acc = 0.0;
for (int qp = 0; qp < num_qp; qp++)
{
acc += B(qp, 0, dof) * f(vd, 0, qp);
}
y(dof, vd) += acc;
}
}
}
else if constexpr (
is_gradient_fop<std::decay_t<output_t>>::value)
{
const auto [num_qp, dim, num_dof] = G.GetShape();
const int vdim = output.vdim;
for (int dof = 0; dof < num_dof; dof++)
{
for (int vd = 0; vd < vdim; vd++)
{
real_t acc = 0.0;
for (int d = 0; d < dim; d++)
{
for (int qp = 0; qp < num_qp; qp++)
{
acc += G(qp, d, dof) * f(vd, d, qp);
}
}
y(dof, vd) += acc;
}
}
}
else if constexpr (is_sum_fop<std::decay_t<output_t>>::value)
{
// This is the "integral over all quadrature points type" applying
// B = 1 s.t. B^T * C \in R^1.
const auto [num_qp, unused, unused1] = B.GetShape();
auto cc = Reshape(&f(0, 0, 0), num_qp);
for (int i = 0; i < num_qp; i++)
{
y(0, 0) += cc(i);
}
}
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
{
const auto [num_qp, unused, num_dof] = B.GetShape();
const auto vdim = output.vdim;
auto cc = Reshape(&f(0, 0, 0), num_qp * vdim);
auto yy = Reshape(&y(0, 0), num_qp * vdim);
for (int i = 0; i < num_qp * vdim; i++)
{
yy(i) = cc(i);
}
}
else
{
MFEM_ABORT("quadrature data mapping to field is not implemented for"
" this field descriptor");
}
}
template <typename output_t>
MFEM_HOST_DEVICE
void map_quadrature_data_to_fields_tensor_impl_1d(
DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const output_t &output,
const DofToQuadMap &dtq,
std::array<DeviceTensor<1>, 6> &scratch_mem)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
const int vdim = output.vdim;
const int test_dim = output.size_on_qp / vdim;
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
auto yd = Reshape(&y(0, 0), d1d, vdim);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t acc = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
acc += fqp(vd, 0, qx) * B(qx, 0, dx);
}
yd(dx, vd) = acc;
}
}
MFEM_SYNC_THREAD;
}
else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = G.GetShape();
const int vdim = output.vdim;
const int test_dim = output.size_on_qp / vdim;
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d);
auto yd = Reshape(&y(0, 0), d1d, vdim);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t acc = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
acc += fqp(vd, 0, qx) * G(qx, 0, dx);
}
yd(dx, vd) = acc;
}
}
MFEM_SYNC_THREAD;
}
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
for (int sq = 0; sq < output.size_on_qp; sq++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
yqp(sq, qx) = fqp(sq, qx);
}
MFEM_SYNC_THREAD;
}
}
else
{
MFEM_ABORT("quadrature data mapping to field is not implemented for"
" this field descriptor with sum factorization on tensor product elements");
}
}
template <typename output_t>
MFEM_HOST_DEVICE
void map_quadrature_data_to_fields_tensor_impl_2d(
DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const output_t &output,
const DofToQuadMap &dtq,
std::array<DeviceTensor<1>, 6> &scratch_mem)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
const int vdim = output.vdim;
const int test_dim = output.size_on_qp / vdim;
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t acc = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
acc += fqp(vd, 0, qx, qy) * B(qx, 0, dx);
}
s0(qy, dx) = acc;
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t acc = 0.0;
for (int qy = 0; qy < q1d; qy++)
{
acc += s0(qy, dx) * B(qy, 0, dy);
}
yd(dx, dy, vd) += acc;
}
}
MFEM_SYNC_THREAD;
}
}
else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = G.GetShape();
const int vdim = output.vdim;
const int test_dim = output.size_on_qp / vdim;
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, vdim);
auto s0 = Reshape(&scratch_mem[0](0), q1d, d1d);
auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t uv[2] = {0.0, 0.0};
for (int qx = 0; qx < q1d; qx++)
{
uv[0] += fqp(vd, 0, qx, qy) * G(qx, 0, dx);
uv[1] += fqp(vd, 1, qx, qy) * B(qx, 0, dx);
}
s0(qy, dx) = uv[0];
s1(qy, dx) = uv[1];
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t uv[2] = {0.0, 0.0};
for (int qy = 0; qy < q1d; qy++)
{
uv[0] += s0(qy, dx) * B(qy, 0, dy);
uv[1] += s1(qy, dx) * G(qy, 0, dy);
}
yd(dx, dy, vd) += uv[0] + uv[1];
}
}
MFEM_SYNC_THREAD;
}
}
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
// // TODO: Check if this is the right fix for all cases
// auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d);
// auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d);
// for (int sq = 0; sq < output.size_on_qp; sq++)
// {
// MFEM_FOREACH_THREAD(qx, x, q1d)
// {
// yqp(sq, qx) = fqp(sq, qx);
// }
// MFEM_SYNC_THREAD;
// }
auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d);
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d);
for (int sq = 0; sq < output.size_on_qp; sq++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
yqp(sq, qx, qy) = fqp(sq, qx, qy);
}
}
MFEM_SYNC_THREAD;
}
}
else
{
MFEM_ABORT("quadrature data mapping to field is not implemented for"
" this field descriptor with sum factorization on tensor product elements");
}
}
template <typename output_t>
MFEM_HOST_DEVICE
void map_quadrature_data_to_fields_tensor_impl_3d(
DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const output_t &output,
const DofToQuadMap &dtq,
std::array<DeviceTensor<1>, 6> &scratch_mem)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
const int vdim = output.vdim;
const int test_dim = output.size_on_qp / vdim;
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
auto s1 = Reshape(&scratch_mem[1](0), q1d, d1d, d1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
real_t acc = 0.0;
for (int qx = 0; qx < q1d; qx++)
{
acc += fqp(vd, 0, qx, qy, qz) * B(qx, 0, dx);
}
s0(qz, qy, dx) = acc;
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
real_t acc = 0.0;
for (int qy = 0; qy < q1d; qy++)
{
acc += s0(qz, qy, dx) * B(qy, 0, dy);
}
s1(qz, dy, dx) = acc;
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
MFEM_FOREACH_THREAD(dz, z, d1d)
{
real_t acc = 0.0;
for (int qz = 0; qz < q1d; qz++)
{
acc += s1(qz, dy, dx) * B(qz, 0, dz);
}
yd(dx, dy, dz, vd) += acc;
}
}
}
MFEM_SYNC_THREAD;
}
}
else if constexpr (is_gradient_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = G.GetShape();
const int vdim = output.vdim;
const int test_dim = output.size_on_qp / vdim;
auto fqp = Reshape(&f(0, 0, 0), vdim, test_dim, q1d, q1d, q1d);
auto yd = Reshape(&y(0, 0), d1d, d1d, d1d, vdim);
auto s0 = Reshape(&scratch_mem[0](0), q1d, q1d, d1d);
auto s1 = Reshape(&scratch_mem[1](0), q1d, q1d, d1d);
auto s2 = Reshape(&scratch_mem[2](0), q1d, q1d, d1d);
auto s3 = Reshape(&scratch_mem[3](0), q1d, d1d, d1d);
auto s4 = Reshape(&scratch_mem[4](0), q1d, d1d, d1d);
auto s5 = Reshape(&scratch_mem[5](0), q1d, d1d, d1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t uvw[3] = {0.0, 0.0, 0.0};
for (int qx = 0; qx < q1d; qx++)
{
uvw[0] += fqp(vd, 0, qx, qy, qz) * G(qx, 0, dx);
uvw[1] += fqp(vd, 1, qx, qy, qz) * B(qx, 0, dx);
uvw[2] += fqp(vd, 2, qx, qy, qz) * B(qx, 0, dx);
}
s0(qz, qy, dx) = uvw[0];
s1(qz, qy, dx) = uvw[1];
s2(qz, qy, dx) = uvw[2];
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(qz, z, q1d)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t uvw[3] = {0.0, 0.0, 0.0};
for (int qy = 0; qy < q1d; qy++)
{
uvw[0] += s0(qz, qy, dx) * B(qy, 0, dy);
uvw[1] += s1(qz, qy, dx) * G(qy, 0, dy);
uvw[2] += s2(qz, qy, dx) * B(qy, 0, dy);
}
s3(qz, dy, dx) = uvw[0];
s4(qz, dy, dx) = uvw[1];
s5(qz, dy, dx) = uvw[2];
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dz, z, d1d)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(dx, x, d1d)
{
real_t uvw[3] = {0.0, 0.0, 0.0};
for (int qz = 0; qz < q1d; qz++)
{
uvw[0] += s3(qz, dy, dx) * B(qz, 0, dz);
uvw[1] += s4(qz, dy, dx) * B(qz, 0, dz);
uvw[2] += s5(qz, dy, dx) * G(qz, 0, dz);
}
yd(dx, dy, dz, vd) += uvw[0] + uvw[1] + uvw[2];
}
}
}
MFEM_SYNC_THREAD;
}
}
else if constexpr (is_identity_fop<std::decay_t<output_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
auto fqp = Reshape(&f(0, 0, 0), output.size_on_qp, q1d, q1d, q1d);
auto yqp = Reshape(&y(0, 0), output.size_on_qp, q1d, q1d, q1d);
for (int sq = 0; sq < output.size_on_qp; sq++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
yqp(sq, qx, qy, qz) = fqp(sq, qx, qy, qz);
}
}
}
MFEM_SYNC_THREAD;
}
}
else
{
MFEM_ABORT("quadrature data mapping to field is not implemented for"
" this field descriptor with sum factorization on tensor product elements");
}
}
template <typename output_t>
MFEM_HOST_DEVICE
void map_quadrature_data_to_fields(
DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const output_t &output,
const DofToQuadMap &dtq,
std::array<DeviceTensor<1>, 6> &scratch_mem,
const int &dimension,
const bool &use_sum_factorization)
{
if (use_sum_factorization)
{
if (dimension == 1)
{
map_quadrature_data_to_fields_tensor_impl_1d(y, f, output, dtq, scratch_mem);
}
else if (dimension == 2)
{
map_quadrature_data_to_fields_tensor_impl_2d(y, f, output, dtq, scratch_mem);
}
else if (dimension == 3)
{
map_quadrature_data_to_fields_tensor_impl_3d(y, f, output, dtq, scratch_mem);
}
else { MFEM_ABORT_KERNEL("dimension not supported"); }
}
else
{
map_quadrature_data_to_fields_impl(y, f, output, dtq);
}
}
template <size_t N, typename field_operator_ts>
MFEM_HOST_DEVICE
void map_quadrature_data_to_fields_conditional(
DeviceTensor<2, real_t> &y,
const DeviceTensor<3, real_t> &f,
const field_operator_ts &fops,
const DeviceTensor<1, const real_t> &op_dims,
const std::array<DofToQuadMap, N> &dtqmaps,
std::array<DeviceTensor<1>, 6> &scratch_mem,
const DeviceTensor<3> &fi_shmem,
const std::array<bool, N> &conditions,
const int &dimension,
const bool &use_sum_factorization)
{
int offset = 0;
for_constexpr<N>([&](auto i)
{
if (conditions[i])
{
[[maybe_unused]] const auto [K, unused, M] = f.GetShape();
const int L = static_cast<int>(op_dims(static_cast<size_t>(i)));
auto fi = Reshape(&fi_shmem(0, 0, 0), K, L, M);
for (int k = 0; k < K; k++)
{
for (int l = 0; l < L; l++)
{
for (int m = 0; m < M; m++)
{
fi(k, l, m) = f(k, l + offset, m);
}
}
}
if (use_sum_factorization)
{
if (dimension == 1)
{
map_quadrature_data_to_fields_tensor_impl_1d(
y, fi, get<i>(fops), dtqmaps[i], scratch_mem);
}
else if (dimension == 2)
{
map_quadrature_data_to_fields_tensor_impl_2d(
y, fi, get<i>(fops), dtqmaps[i], scratch_mem);
}
else if (dimension == 3)
{
map_quadrature_data_to_fields_tensor_impl_3d(
y, fi, get<i>(fops), dtqmaps[i], scratch_mem);
}
else { MFEM_ABORT_KERNEL("dimension not supported"); }
}
else
{
map_quadrature_data_to_fields_impl(y, fi, get<i>(fops), dtqmaps[i]);
}
offset += L;
}
});
}
} // namespace mfem::future
-37
View File
@@ -1,37 +0,0 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include <typeindex>
#include <unordered_map>
#include <vector>
#include "../../general/array.hpp"
#include "fielddescriptor.hpp"
namespace mfem::future
{
struct IntegratorContext
{
const ParMesh &mesh;
const Array<int> *elem_attr;
Array<int> attr;
const int nentities;
const std::vector<FieldDescriptor> &infds;
const std::vector<FieldDescriptor> &outfds;
const std::vector<FieldDescriptor> &unionfds;
const IntegrationRule &ir;
std::unordered_map<std::type_index, std::vector<int>> &in_qlayouts;
std::unordered_map<std::type_index, std::vector<int>> &out_qlayouts;
};
}
+673
View File
@@ -0,0 +1,673 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "util.hpp"
namespace mfem::future
{
template <typename field_operator_t>
MFEM_HOST_DEVICE inline
void map_field_to_quadrature_data_tensor_product_3d(
DeviceTensor<2> &field_qp,
const DofToQuadMap &dtq,
const DeviceTensor<1> &field_e,
const field_operator_t &input,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
{
auto [q1d, unused, d1d] = B.GetShape();
const int vdim = input.vdim;
const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d, q1d);
auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d, q1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(dz, z, d1d)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t acc = 0.0;
for (int dx = 0; dx < d1d; dx++)
{
acc += B(qx, 0, dx) * field(dx, dy, dz, vd);
}
s0(dz, dy, qx) = acc;
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dz, z, d1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
real_t acc = 0.0;
for (int dy = 0; dy < d1d; dy++)
{
acc += s0(dz, dy, qx) * B(qy, 0, dy);
}
s1(dz, qy, qx) = acc;
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(qz, z, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t acc = 0.0;
for (int dz = 0; dz < d1d; dz++)
{
acc += s1(dz, qy, qx) * B(qz, 0, dz);
}
fqp(vd, qx, qy, qz) = acc;
}
}
}
MFEM_SYNC_THREAD;
}
}
else if constexpr (
is_gradient_fop<std::decay_t<field_operator_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
const int vdim = input.vdim;
const int dim = input.dim;
const auto field = Reshape(&field_e[0], d1d, d1d, d1d, vdim);
auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d, q1d);
auto s0 = Reshape(&scratch_mem[0](0), d1d, d1d, q1d);
auto s1 = Reshape(&scratch_mem[1](0), d1d, d1d, q1d);
auto s2 = Reshape(&scratch_mem[2](0), d1d, q1d, q1d);
auto s3 = Reshape(&scratch_mem[3](0), d1d, q1d, q1d);
auto s4 = Reshape(&scratch_mem[4](0), d1d, q1d, q1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(dz, z, d1d)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t uv[2] = {0.0, 0.0};
for (int dx = 0; dx < d1d; dx++)
{
const real_t f = field(dx, dy, dz, vd);
uv[0] += f * B(qx, 0, dx);
uv[1] += f * G(qx, 0, dx);
}
s0(dz, dy, qx) = uv[0];
s1(dz, dy, qx) = uv[1];
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(dz, z, d1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t uvw[3] = {0.0, 0.0, 0.0};
for (int dy = 0; dy < d1d; dy++)
{
const real_t s0i = s0(dz, dy, qx);
uvw[0] += s1(dz, dy, qx) * B(qy, 0, dy);
uvw[1] += s0i * G(qy, 0, dy);
uvw[2] += s0i * B(qy, 0, dy);
}
s2(dz, qy, qx) = uvw[0];
s3(dz, qy, qx) = uvw[1];
s4(dz, qy, qx) = uvw[2];
}
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(qz, z, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t uvw[3] = {0.0, 0.0, 0.0};
for (int dz = 0; dz < d1d; dz++)
{
uvw[0] += s2(dz, qy, qx) * B(qz, 0, dz);
uvw[1] += s3(dz, qy, qx) * B(qz, 0, dz);
uvw[2] += s4(dz, qy, qx) * G(qz, 0, dz);
}
fqp(vd, 0, qx, qy, qz) = uvw[0];
fqp(vd, 1, qx, qy, qz) = uvw[1];
fqp(vd, 2, qx, qy, qz) = uvw[2];
}
}
}
MFEM_SYNC_THREAD;
}
}
// TODO: Create separate function for clarity
else if constexpr (
std::is_same_v<std::decay_t<field_operator_t>, Weight>)
{
const int num_qp = integration_weights.GetShape()[0];
// TODO: eeek
const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
auto w = Reshape(&integration_weights[0], q1d, q1d, q1d);
auto f = Reshape(&field_qp[0], q1d, q1d, q1d);
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qz, z, q1d)
{
f(qx, qy, qz) = w(qx, qy, qz);
}
}
}
MFEM_SYNC_THREAD;
}
else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
{
const int q1d = B.GetShape()[0];
auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d * q1d);
field_qp = field;
}
else
{
static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
"can't map field to quadrature data");
}
}
template <typename field_operator_t>
MFEM_HOST_DEVICE inline
void map_field_to_quadrature_data_tensor_product_2d(
DeviceTensor<2> &field_qp,
const DofToQuadMap &dtq,
const DeviceTensor<1> &field_e,
const field_operator_t &input,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
{
auto [q1d, unused, d1d] = B.GetShape();
const int vdim = input.vdim;
const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
auto fqp = Reshape(&field_qp[0], vdim, q1d, q1d);
auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t acc = 0.0;
for (int dx = 0; dx < d1d; dx++)
{
acc += B(qx, 0, dx) * field(dx, dy, vd);
}
s0(dy, qx) = acc;
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
real_t acc = 0.0;
for (int dy = 0; dy < d1d; dy++)
{
acc += s0(dy, qx) * B(qy, 0, dy);
}
fqp(vd, qx, qy) = acc;
}
}
MFEM_SYNC_THREAD;
}
}
else if constexpr (
is_gradient_fop<std::decay_t<field_operator_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
const int vdim = input.vdim;
const int dim = input.dim;
const auto field = Reshape(&field_e[0], d1d, d1d, vdim);
auto fqp = Reshape(&field_qp[0], vdim, dim, q1d, q1d);
auto s0 = Reshape(&scratch_mem[0](0), d1d, q1d);
auto s1 = Reshape(&scratch_mem[1](0), d1d, q1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(dy, y, d1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t uv[2] = {0.0, 0.0};
for (int dx = 0; dx < d1d; dx++)
{
const real_t f = field(dx, dy, vd);
uv[0] += f * B(qx, 0, dx);
uv[1] += f * G(qx, 0, dx);
}
s0(dy, qx) = uv[0];
s1(dy, qx) = uv[1];
}
}
MFEM_SYNC_THREAD;
MFEM_FOREACH_THREAD(qy, y, q1d)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t uv[2] = {0.0, 0.0};
for (int dy = 0; dy < d1d; dy++)
{
const real_t s0i = s0(dy, qx);
uv[0] += s1(dy, qx) * B(qy, 0, dy);
uv[1] += s0i * G(qy, 0, dy);
}
fqp(vd, 0, qx, qy) = uv[0];
fqp(vd, 1, qx, qy) = uv[1];
}
}
MFEM_SYNC_THREAD;
}
}
// TODO: Create separate function for clarity
else if constexpr (
std::is_same_v<std::decay_t<field_operator_t>, Weight>)
{
const int num_qp = integration_weights.GetShape()[0];
// TODO: eeek
const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
auto w = Reshape(&integration_weights[0], q1d, q1d);
auto f = Reshape(&field_qp[0], q1d, q1d);
MFEM_FOREACH_THREAD(qx, x, q1d)
{
MFEM_FOREACH_THREAD(qy, y, q1d)
{
f(qx, qy) = w(qx, qy);
}
}
MFEM_SYNC_THREAD;
}
else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
{
const int q1d = B.GetShape()[0];
auto field = Reshape(&field_e[0], input.size_on_qp, q1d * q1d);
field_qp = field;
}
else
{
static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
"can't map field to quadrature data");
}
}
template <typename field_operator_t>
MFEM_HOST_DEVICE inline
void map_field_to_quadrature_data_tensor_product_1d(
DeviceTensor<2> &field_qp,
const DofToQuadMap &dtq,
const DeviceTensor<1> &field_e,
const field_operator_t &input,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<std::decay_t<field_operator_t>>::value)
{
auto [q1d, unused, d1d] = B.GetShape();
const int vdim = input.vdim;
const auto field = Reshape(&field_e[0], d1d, vdim);
auto fqp = Reshape(&field_qp[0], vdim, q1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t acc = 0.0;
for (int dx = 0; dx < d1d; dx++)
{
acc += B(qx, 0, dx) * field(dx, vd);
}
fqp(vd, qx) = acc;
}
}
MFEM_SYNC_THREAD;
}
else if constexpr (
is_gradient_fop<std::decay_t<field_operator_t>>::value)
{
const auto [q1d, unused, d1d] = B.GetShape();
const int vdim = input.vdim;
const int dim = input.dim;
const auto field = Reshape(&field_e[0], d1d, vdim);
auto fqp = Reshape(&field_qp[0], vdim, dim, q1d);
for (int vd = 0; vd < vdim; vd++)
{
MFEM_FOREACH_THREAD(qx, x, q1d)
{
real_t acc = 0.0;
for (int dx = 0; dx < d1d; dx++)
{
acc += G(qx, 0, dx) * field(dx, vd);
}
fqp(vd, 0, qx) = acc;
}
MFEM_SYNC_THREAD;
}
}
// TODO: Create separate function for clarity
else if constexpr (
std::is_same_v<std::decay_t<field_operator_t>, Weight>)
{
const int num_qp = integration_weights.GetShape()[0];
// TODO: eeek
const int q1d = (int)floor(std::pow(num_qp, 1.0/input.dim) + 0.5);
auto w = Reshape(&integration_weights[0], q1d);
auto f = Reshape(&field_qp[0], q1d);
MFEM_FOREACH_THREAD(qx, x, q1d)
{
f(qx) = w(qx);
}
MFEM_SYNC_THREAD;
}
else if constexpr (is_identity_fop<std::decay_t<field_operator_t>>::value)
{
const int q1d = B.GetShape()[0];
auto field = Reshape(&field_e[0], input.size_on_qp, q1d);
field_qp = field;
}
else
{
static_assert(dfem::always_false<std::decay_t<field_operator_t>>,
"can't map field to quadrature data");
}
}
template <typename field_operator_t>
MFEM_HOST_DEVICE
void map_field_to_quadrature_data(
DeviceTensor<2> field_qp,
const DofToQuadMap &dtq,
const DeviceTensor<1> &field_e,
const field_operator_t &input,
const DeviceTensor<1, const real_t> &integration_weights)
{
[[maybe_unused]] auto B = dtq.B;
[[maybe_unused]] auto G = dtq.G;
if constexpr (is_value_fop<field_operator_t>::value)
{
auto [num_qp, dim, num_dof] = B.GetShape();
const int vdim = input.vdim;
const auto field = Reshape(&field_e(0), num_dof, vdim);
for (int vd = 0; vd < vdim; vd++)
{
for (int qp = 0; qp < num_qp; qp++)
{
real_t acc = 0.0;
for (int dof = 0; dof < num_dof; dof++)
{
acc += B(qp, 0, dof) * field(dof, vd);
}
field_qp(vd, qp) = acc;
}
}
}
else if constexpr (is_gradient_fop<field_operator_t>::value)
{
const auto [num_qp, dim, num_dof] = G.GetShape();
const int vdim = input.vdim;
const auto field = Reshape(&field_e(0), num_dof, vdim);
auto f = Reshape(&field_qp[0], vdim, dim, num_qp);
for (int vd = 0; vd < vdim; vd++)
{
for (int qp = 0; qp < num_qp; qp++)
{
for (int d = 0; d < dim; d++)
{
real_t acc = 0.0;
for (int dof = 0; dof < num_dof; dof++)
{
acc += G(qp, d, dof) * field(dof, vd);
}
f(vd, d, qp) = acc;
}
}
}
}
else if constexpr (std::is_same_v<field_operator_t, Weight>)
{
const int num_qp = integration_weights.GetShape()[0];
auto f = Reshape(&field_qp[0], num_qp);
for (int qp = 0; qp < num_qp; qp++)
{
f(qp) = integration_weights(qp);
}
}
else if constexpr (is_identity_fop<field_operator_t>::value)
{
auto [num_qp, unused, num_dof] = B.GetShape();
const int size_on_qp = input.size_on_qp;
const auto field = Reshape(&field_e[0], size_on_qp * num_qp);
auto f = Reshape(&field_qp[0], size_on_qp * num_qp);
for (int i = 0; i < size_on_qp * num_qp; i++)
{
f(i) = field(i);
}
}
else
{
static_assert(dfem::always_false<field_operator_t>,
"can't map field to quadrature data");
}
}
template <typename field_operator_ts, size_t N, size_t M>
MFEM_HOST_DEVICE inline
void map_fields_to_quadrature_data(
std::array<DeviceTensor<2>, N> &fields_qp,
const std::array<DeviceTensor<1>, M> &fields_e,
const std::array<DofToQuadMap, N> &dtqmaps,
const std::array<size_t, N> &input_to_field,
const field_operator_ts &fops,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem,
const int &dimension,
const bool &use_sum_factorization = false)
{
// When the input_to_field map returns -1, this means the requested input
// is the integration weight. Weights don't have a user defined field
// attached to them and we create a dummy field which is not accessed
// inside the functions it is passed to.
const auto dummy_field_weight = DeviceTensor<1>(nullptr, 0);
for_constexpr<N>([&](auto i)
{
const DeviceTensor<1> &field_e =
(input_to_field[i] == SIZE_MAX) ? dummy_field_weight :
fields_e[input_to_field[i]];
if (use_sum_factorization)
{
if (dimension == 1)
{
map_field_to_quadrature_data_tensor_product_1d(
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
integration_weights, scratch_mem);
}
else if (dimension == 2)
{
map_field_to_quadrature_data_tensor_product_2d(
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
integration_weights, scratch_mem);
}
else if (dimension == 3)
{
map_field_to_quadrature_data_tensor_product_3d(
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
integration_weights, scratch_mem);
}
else { MFEM_ABORT_KERNEL("unsupported dimension"); }
}
else
{
map_field_to_quadrature_data(
fields_qp[i], dtqmaps[i], field_e, get<i>(fops),
integration_weights);
}
});
}
template <typename field_operator_t>
MFEM_HOST_DEVICE
void map_field_to_quadrature_data_conditional(
DeviceTensor<2> &field_qp,
const DeviceTensor<1> &field_e,
const DofToQuadMap &dtqmap,
field_operator_t &fop,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem,
const bool &condition,
const int &dimension,
const bool &use_sum_factorization = false)
{
if (condition)
{
if (use_sum_factorization)
{
if (dimension == 1)
{
map_field_to_quadrature_data_tensor_product_1d(
field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
}
else if (dimension == 2)
{
map_field_to_quadrature_data_tensor_product_2d(
field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
}
else if (dimension == 3)
{
map_field_to_quadrature_data_tensor_product_3d(
field_qp, dtqmap, field_e, fop, integration_weights, scratch_mem);
}
}
else
{
map_field_to_quadrature_data(
field_qp, dtqmap, field_e, fop, integration_weights);
}
}
}
template <size_t num_fields, size_t num_inputs, typename field_operator_ts>
MFEM_HOST_DEVICE
void map_fields_to_quadrature_data_conditional(
std::array<DeviceTensor<2>, num_inputs> &fields_qp,
const std::array<DeviceTensor<1, const real_t>, num_fields> &fields_e,
const std::array<DofToQuadMap, num_inputs> &dtqmaps,
field_operator_ts fops,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem,
const std::array<bool, num_inputs> &conditions,
const bool &use_sum_factorization = false)
{
for_constexpr<num_inputs>([&](auto i)
{
map_field_to_quadrature_data_conditional(
fields_qp[i], fields_e[i], dtqmaps[i], get<i>(fops), integration_weights,
scratch_mem, conditions[i], use_sum_factorization);
});
}
template <size_t N, typename field_operator_ts>
MFEM_HOST_DEVICE
void map_direction_to_quadrature_data_conditional(
std::array<DeviceTensor<2>, N> &directions_qp,
const DeviceTensor<1> &direction_e,
const std::array<DofToQuadMap, N> &dtqmaps,
field_operator_ts fops,
const DeviceTensor<1, const real_t> &integration_weights,
const std::array<DeviceTensor<1>, 6> &scratch_mem,
const std::array<bool, N> &conditions,
const int &dimension,
const bool &use_sum_factorization)
{
for_constexpr<N>([&](auto i)
{
if (conditions[i])
{
if (use_sum_factorization)
{
if (dimension == 1)
{
map_field_to_quadrature_data_tensor_product_1d(
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
integration_weights, scratch_mem);
}
else if (dimension == 2)
{
map_field_to_quadrature_data_tensor_product_2d(
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
integration_weights, scratch_mem);
}
else if (dimension == 3)
{
map_field_to_quadrature_data_tensor_product_3d(
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
integration_weights, scratch_mem);
}
}
else
{
map_field_to_quadrature_data(
directions_qp[i], dtqmaps[i], direction_e, get<i>(fops),
integration_weights);
}
}
});
}
}
+6 -9
View File
@@ -20,7 +20,7 @@ namespace mfem::future
class ParameterSpace
{
public:
ParameterSpace(int vdim = 1) : vdim(vdim) { dtq.FE = nullptr; dtq.IntRule = nullptr; }
ParameterSpace(int vdim = 1) : vdim(vdim) {}
/// @brief Get vector dimension at each point
///
@@ -43,7 +43,7 @@ public:
/// Get spatial dimension
///
/// returns always 1.
constexpr int Dimension() const
int Dimension() const
{
return 1;
}
@@ -65,7 +65,7 @@ public:
/// It should not be used by a user.
///
/// returns identity by default that is lazy evaluated.
virtual const Operator* GetElementRestriction(ElementDofOrdering) const
virtual const Operator* GetElementRestriction(ElementDofOrdering o) const
{
if (!elem_restr)
{
@@ -74,18 +74,15 @@ public:
return elem_restr.get();
}
virtual const Operator* GetB() const = 0;
virtual const Operator* GetBt() const = 0;
protected:
int vdim;
DofToQuad dtq;
mutable std::unique_ptr<Operator> prolongation, elem_restr, B, Bt;
mutable std::unique_ptr<Operator> prolongation;
mutable std::unique_ptr<Operator> elem_restr;
};
/// @brief Uniform parameter space
class UniformParameterSpace : public ParameterSpace
class UniformParameterSpace final : public ParameterSpace
{
public:
/// @brief Constructor for a uniform parameter space
+669
View File
@@ -0,0 +1,669 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "util.hpp"
#include "qfunction_transform.hpp"
namespace mfem::future
{
/// @brief Call a qfunction with the given parameters.
///
/// @param qfunc the qfunction to call.
/// @param input_shmem the input shared memory.
/// @param residual_shmem the residual shared memory.
/// @param rs_qp the size of the residual.
/// @param num_qp the number of quadrature points.
/// @param q1d the number of quadrature points in 1D.
/// @param dimension the spatial dimension.
/// @param use_sum_factorization whether to use sum factorization.
/// @tparam qf_param_ts the tuple type of the qfunction parameters.
template <
typename qf_param_ts,
typename qfunc_t,
std::size_t num_fields>
MFEM_HOST_DEVICE inline
void call_qfunction(
qfunc_t &qfunc,
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
DeviceTensor<2> &residual_shmem,
const int &rs_qp,
const int &num_qp,
const int &q1d,
const int &dimension,
const bool &use_sum_factorization)
{
if (use_sum_factorization)
{
if (dimension == 1)
{
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
{
auto qf_args = decay_tuple<qf_param_ts> {};
auto r = Reshape(&residual_shmem(0, q), rs_qp);
apply_kernel(r, qfunc, qf_args, input_shmem, q);
}
}
else if (dimension == 2)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
const int q = qx + q1d * qy;
auto qf_args = decay_tuple<qf_param_ts> {};
auto r = Reshape(&residual_shmem(0, q), rs_qp);
apply_kernel(r, qfunc, qf_args, input_shmem, q);
}
}
}
else if (dimension == 3)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
auto qf_args = decay_tuple<qf_param_ts> {};
auto r = Reshape(&residual_shmem(0, q), rs_qp);
apply_kernel(r, qfunc, qf_args, input_shmem, q);
}
}
}
}
else
{
#if !(defined(MFEM_USE_CUDA) || defined(MFEM_USE_HIP))
MFEM_ABORT("unsupported dimension for sum factorization");
#endif
}
MFEM_SYNC_THREAD;
}
else
{
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
{
auto qf_args = decay_tuple<qf_param_ts> {};
auto r = Reshape(&residual_shmem(0, q), rs_qp);
apply_kernel(r, qfunc, qf_args, input_shmem, q);
}
}
}
/// @brief Call a qfunction with the given parameters and
/// compute it's derivative action.
///
/// @param qfunc the qfunction to call.
/// @param input_shmem the input shared memory.
/// @param shadow_shmem the shadow shared memory.
/// @param residual_shmem the residual shared memory.
/// @param das_qp the size of the derivative action.
/// @param num_qp the number of quadrature points.
/// @param q1d the number of quadrature points in 1D.
/// @param dimension the spatial dimension.
/// @param use_sum_factorization whether to use sum factorization.
/// @tparam qf_param_ts the tuple type of the qfunction parameters.
template <
typename qf_param_ts,
typename qfunc_t,
std::size_t num_fields>
MFEM_HOST_DEVICE inline
void call_qfunction_derivative_action(
qfunc_t &qfunc,
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
DeviceTensor<2> &residual_shmem,
const int &das_qp,
const int &num_qp,
const int &q1d,
const int &dimension,
const bool &use_sum_factorization)
{
if (use_sum_factorization)
{
if (dimension == 1)
{
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
{
auto r = Reshape(&residual_shmem(0, q), das_qp);
auto qf_args = decay_tuple<qf_param_ts> {};
#ifdef MFEM_USE_ENZYME
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
shadow_shmem, q);
#else
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
#endif
}
}
else if (dimension == 2)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
const int q = qx + q1d * qy;
auto r = Reshape(&residual_shmem(0, q), das_qp);
auto qf_args = decay_tuple<qf_param_ts> {};
#ifdef MFEM_USE_ENZYME
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
shadow_shmem, q);
#else
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
#endif
}
}
}
else if (dimension == 3)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
auto r = Reshape(&residual_shmem(0, q), das_qp);
auto qf_args = decay_tuple<qf_param_ts> {};
#ifdef MFEM_USE_ENZYME
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
shadow_shmem, q);
#else
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
#endif
}
}
}
}
else
{
MFEM_ABORT_KERNEL("unsupported dimension");
}
}
else
{
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
{
auto r = Reshape(&residual_shmem(0, q), das_qp);
auto qf_args = decay_tuple<qf_param_ts> {};
#ifdef MFEM_USE_ENZYME
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
shadow_shmem, q);
#else
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
#endif
}
}
MFEM_SYNC_THREAD;
}
namespace detail
{
template <
typename qf_param_ts,
typename qfunc_t,
std::size_t num_fields>
MFEM_HOST_DEVICE inline
void call_qfunction_derivative(
qfunc_t &qfunc,
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
DeviceTensor<2> &residual_shmem,
DeviceTensor<5> &qpdc,
const DeviceTensor<1, const real_t> &itod,
const int &das_qp,
const int &q)
{
const int test_vdim = qpdc.GetShape()[0];
const int test_op_dim = qpdc.GetShape()[1];
const int trial_vdim = qpdc.GetShape()[2];
const int num_qp = qpdc.GetShape()[4];
const size_t num_inputs = itod.GetShape()[0];
for (int j = 0; j < trial_vdim; j++)
{
int m_offset = 0;
for (size_t s = 0; s < num_inputs; s++)
{
const int trial_op_dim = static_cast<int>(itod(s));
if (trial_op_dim == 0)
{
continue;
}
auto d_qp = Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
for (int m = 0; m < trial_op_dim; m++)
{
d_qp(j, m, q) = 1.0;
auto r = Reshape(&residual_shmem(0, q), das_qp);
auto qf_args = decay_tuple<qf_param_ts> {};
#ifdef MFEM_USE_ENZYME
auto qf_shadow_args = decay_tuple<qf_param_ts> {};
apply_kernel_fwddiff_enzyme(r, qfunc, qf_args, qf_shadow_args, input_shmem,
shadow_shmem, q);
#else
apply_kernel_native_dual(r, qfunc, qf_args, input_shmem, shadow_shmem, q);
#endif
d_qp(j, m, q) = 0.0;
auto f = Reshape(&r(0), test_vdim, test_op_dim);
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
qpdc(i, k, j, m + m_offset, q) = f(i, k);
}
}
}
m_offset += trial_op_dim;
}
}
}
}
/// @brief Call a qfunction with the given parameters and
/// compute it's derivative represented by the Jacobian on
/// each quadrature point.
///
/// @param qfunc the qfunction to call.
/// @param input_shmem the input shared memory.
/// @param shadow_shmem the shadow shared memory.
/// @param residual_shmem the residual shared memory.
/// @param qpdc the quadrature point data cache holding the resulting
/// Jacobians on each quadrature point.
/// @param itod inputs trial operator dimension.
/// If input is dependent the value corresponds to the spatial dimension, otherwise
/// a zero indicates non-dependence on the variable.
/// @param das_qp the size of the derivative action.
/// @param q1d the number of quadrature points in 1D.
/// @param dimension the spatial dimension.
/// @param use_sum_factorization whether to use sum factorization.
/// @tparam qf_param_ts the tuple type of the qfunction parameters.
template <
typename qf_param_ts,
typename qfunc_t,
std::size_t num_fields>
MFEM_HOST_DEVICE inline
void call_qfunction_derivative(
qfunc_t &qfunc,
const std::array<DeviceTensor<2>, num_fields> &input_shmem,
const std::array<DeviceTensor<2>, num_fields> &shadow_shmem,
DeviceTensor<2> &residual_shmem,
DeviceTensor<5> &qpdc,
const DeviceTensor<1, const real_t> &itod,
const int &das_qp,
const int &q1d,
const int &dimension,
const bool &use_sum_factorization)
{
if (use_sum_factorization)
{
if (dimension == 1)
{
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
{
detail::call_qfunction_derivative<qf_param_ts>(
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
}
}
else if (dimension == 2)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
const int q = qx + q1d * qy;
detail::call_qfunction_derivative<qf_param_ts>(
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
}
}
}
else if (dimension == 3)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
detail::call_qfunction_derivative<qf_param_ts>(
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
}
}
}
}
else
{
MFEM_ABORT_KERNEL("unsupported dimension");
}
}
else
{
const int num_qp = qpdc.GetShape()[4];
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
{
detail::call_qfunction_derivative<qf_param_ts>(
qfunc, input_shmem, shadow_shmem, residual_shmem, qpdc, itod, das_qp, q);
}
}
MFEM_SYNC_THREAD;
}
namespace detail
{
/// @brief Apply the quadrature point data cache (qpdc) to a vector
/// (usually a direction) on quadrature point q.
///
/// The qpdc consists of compatible data to be used for integration with a test
/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
/// function including integration weights and necessesary transformations.
///
/// @param fhat the qpdc applied to a vector in shadow_memory.
/// @param shadow_shmem the shadow shared memory.
/// @param qpdc the quadrature point data cache holding the resulting
/// Jacobians on each quadrature point.
/// @param op_dims operator dimensions.
/// If an operator is dependent, the value corresponds to the spatial dimension.
/// Otherwise a zero indicates indepence on the variable.
/// @param q the current quadrature point index.
/// @param transpose switch to use transpose action.
template <size_t N>
MFEM_HOST_DEVICE inline
void apply_qpdc(
DeviceTensor<3> &fhat,
const std::array<DeviceTensor<2>, N> &shadow_shmem,
const DeviceTensor<5, const real_t> &qpdc,
const DeviceTensor<1, const real_t> &op_dims,
const int &q,
bool transpose)
{
const size_t num_ops = op_dims.GetShape()[0];
const int test_vdim = qpdc.GetShape()[0];
const int test_op_dim = qpdc.GetShape()[1];
const int trial_vdim = qpdc.GetShape()[2];
const int total_trial_op_dim = qpdc.GetShape()[3];
const int num_qp = qpdc.GetShape()[4];
if (transpose)
{
for (int j = 0; j < trial_vdim; j++)
{
for (int m = 0; m < total_trial_op_dim; m++)
{
fhat(j, m, q) = 0.0;
}
}
// Since we don't support more than output space right now
// shadow_shmem will always be of size 1.
constexpr int shadow_idx_tr = 0;
auto d_qp = Reshape(&(shadow_shmem[shadow_idx_tr])[0], test_vdim, test_op_dim,
num_qp);
int m_offset = 0;
for (size_t s = 0; s < num_ops; s++)
{
const int trial_op_dim = static_cast<int>(op_dims(s));
if (trial_op_dim == 0) { continue; }
for (int j = 0; j < trial_vdim; j++)
{
for (int m = 0; m < trial_op_dim; m++)
{
real_t sum = 0.0;
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
const real_t contrib = qpdc(i, k, j, m + m_offset, q) * d_qp(i, k, q);
sum += contrib;
}
}
fhat(j, m + m_offset, q) += sum;
}
}
m_offset += trial_op_dim;
}
}
else
{
for (int i = 0; i < test_vdim; i++)
{
for (int k = 0; k < test_op_dim; k++)
{
real_t sum = 0.0;
int m_offset = 0;
for (size_t s = 0; s < num_ops; s++)
{
const int trial_op_dim = static_cast<int>(op_dims(s));
if (trial_op_dim == 0) { continue; }
const auto d_qp =
Reshape(&(shadow_shmem[s])[0], trial_vdim, trial_op_dim, num_qp);
for (int j = 0; j < trial_vdim; j++)
{
for (int m = 0; m < trial_op_dim; m++)
{
sum += qpdc(i, k, j, m + m_offset, q) * d_qp(j, m, q);
}
}
m_offset += trial_op_dim;
}
fhat(i, k, q) = sum;
}
}
}
}
} // namespace detail
/// @brief Apply the quadrature point data cache (qpdc) to a vector
/// (usually a direction).
///
/// The qpdc consists of compatible data to be used for integration with a test
/// operator, e.g. Jacobians of a linearization from a FE operation with a trial
/// function including integration weights and necessesary transformations.
///
/// @param fhat the qpdc applied to a vector in shadow_memory.
/// @param shadow_shmem the shadow shared memory.
/// @param qpdc the quadrature point data cache holding the resulting
/// Jacobians on each quadrature point.
/// @param itod inputs trial operator dimension.
/// If input is dependent the value corresponds to the spatial dimension, otherwise
/// a zero indicates non-dependence on the variable.
/// @param q1d number of quadrature points in 1D.
/// @param dimension spatial dimension.
/// @param use_sum_factorization whether to use sum factorization.
/// @param T switch to use transpose application.
template <size_t N>
MFEM_HOST_DEVICE inline
void apply_qpdc(
DeviceTensor<3> &fhat,
const std::array<DeviceTensor<2>, N> &shadow_shmem,
const DeviceTensor<5, const real_t> &qpdc,
const DeviceTensor<1, const real_t> &itod,
const int &q1d,
const int &dimension,
const bool &use_sum_factorization,
const bool T = false)
{
if (use_sum_factorization)
{
if (dimension == 1)
{
MFEM_FOREACH_THREAD_DIRECT(q, x, q1d)
{
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
}
}
else if (dimension == 2)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
const int q = qx + q1d * qy;
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
}
}
}
else if (dimension == 3)
{
MFEM_FOREACH_THREAD_DIRECT(qx, x, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qy, y, q1d)
{
MFEM_FOREACH_THREAD_DIRECT(qz, z, q1d)
{
const int q = qx + q1d * (qy + q1d * qz);
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
}
}
}
}
else
{
MFEM_ABORT_KERNEL("unsupported dimension");
}
}
else
{
const int num_qp = qpdc.GetShape()[4];
MFEM_FOREACH_THREAD_DIRECT(q, x, num_qp)
{
detail::apply_qpdc(fhat, shadow_shmem, qpdc, itod, q, T);
}
}
}
template <typename qfunc_t, typename args_ts, size_t num_args>
MFEM_HOST_DEVICE inline
void apply_kernel(
DeviceTensor<1, real_t> &f_qp,
const qfunc_t &qfunc,
args_ts &args,
const std::array<DeviceTensor<2>, num_args> &u,
int qp)
{
process_qf_args(u, args, qp);
process_qf_result(f_qp, get<0>(apply(qfunc, args)));
}
template <typename qfunc_t, typename arg_ts, size_t num_args>
MFEM_HOST_DEVICE inline
void apply_kernel_native_dual(
DeviceTensor<1, real_t> &f_qp,
const qfunc_t &qfunc,
arg_ts &args,
const std::array<DeviceTensor<2>, num_args> &u,
const std::array<DeviceTensor<2>, num_args> &v,
const int &qp_idx)
{
process_qf_args(u, v, args, qp_idx);
auto r = get<0>(apply(qfunc, args));
process_derivative_from_native_dual(f_qp, r);
}
#ifdef MFEM_USE_ENZYME
template <typename func_t, typename... arg_ts>
MFEM_HOST_DEVICE inline
auto qfunction_wrapper(const func_t &f, arg_ts &&...args)
{
return f(args...);
}
// Version for active function arguments only
//
// This is an Enzyme regression and can be removed in later versions.
template <typename qfunc_t, typename arg_ts, std::size_t... Is,
typename inactive_arg_ts>
MFEM_HOST_DEVICE inline
auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
arg_ts &&shadow_args,
std::index_sequence<Is...>,
inactive_arg_ts &&inactive_args,
std::index_sequence<>)
{
using qf_return_t = typename create_function_signature<
decltype(&qfunc_t::operator())>::type::return_t;
return __enzyme_fwddiff<qf_return_t>(
qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...>, enzyme_const,
(void *)&qfunc, enzyme_dup, &get<Is>(args)..., enzyme_interleave,
&get<Is>(shadow_args)...);
}
// Interleave function arguments for enzyme
template <typename qfunc_t, typename arg_ts, std::size_t... Is,
typename inactive_arg_ts, std::size_t... Js>
MFEM_HOST_DEVICE inline
auto fwddiff_apply_enzyme_indexed(qfunc_t &qfunc, arg_ts &&args,
arg_ts &&shadow_args,
std::index_sequence<Is...>,
inactive_arg_ts &&inactive_args,
std::index_sequence<Js...>)
{
using qf_return_t = typename create_function_signature<
decltype(&qfunc_t::operator())>::type::return_t;
return __enzyme_fwddiff<qf_return_t>(
qfunction_wrapper<qfunc_t, decltype(get<Is>(args))...,
decltype(get<Js>(inactive_args))...>,
enzyme_const, (void *)&qfunc, enzyme_dup, &get<Is>(args)...,
enzyme_const, &get<Js>(inactive_args)..., enzyme_interleave,
&get<Is>(shadow_args)...);
}
template <typename qfunc_t, typename arg_ts, typename inactive_arg_ts>
MFEM_HOST_DEVICE inline
auto fwddiff_apply_enzyme(qfunc_t &qfunc, arg_ts &&args,
arg_ts &&shadow_args,
inactive_arg_ts &&inactive_args)
{
auto arg_indices = std::make_index_sequence<
tuple_size<std::remove_reference_t<arg_ts>>::value> {};
auto inactive_arg_indices = std::make_index_sequence<
tuple_size<std::remove_reference_t<inactive_arg_ts>>::value> {};
return fwddiff_apply_enzyme_indexed(qfunc, args, shadow_args, arg_indices,
inactive_args, inactive_arg_indices);
}
template <typename qfunc_t, typename arg_ts, size_t num_args>
MFEM_HOST_DEVICE inline
void apply_kernel_fwddiff_enzyme(
DeviceTensor<1, real_t> &f_qp,
qfunc_t &qfunc,
arg_ts &args,
arg_ts &shadow_args,
const std::array<DeviceTensor<2>, num_args> &u,
const std::array<DeviceTensor<2>, num_args> &v,
int qp_idx)
{
process_qf_args(u, args, qp_idx);
process_qf_args(v, shadow_args, qp_idx);
process_qf_result(f_qp,
get<0>(fwddiff_apply_enzyme(qfunc, args, shadow_args, tuple<> {})));
}
#endif // MFEM_USE_ENZYME
} // namespace mfem::future
+359
View File
@@ -0,0 +1,359 @@
// Copyright (c) 2010-2025, Lawrence Livermore National Security, LLC. Produced
// at the Lawrence Livermore National Laboratory. All Rights reserved. See files
// LICENSE and NOTICE for details. LLNL-CODE-806117.
//
// This file is part of the MFEM library. For more information and source code
// availability visit https://mfem.org.
//
// MFEM is free software; you can redistribute it and/or modify it under the
// terms of the BSD-3 license. We welcome feedback and contributions, see file
// CONTRIBUTING.md for details.
#pragma once
#include "util.hpp"
#include "../../linalg/tensor.hpp"
namespace mfem::future
{
template <typename T0, typename T1, typename T2>
MFEM_HOST_DEVICE
void process_qf_arg(const T0 &, const T1 &, T2 &)
{
static_assert(dfem::always_false<T0, T1, T2>,
"process_qf_arg not implemented for arg type");
}
template <typename T>
MFEM_HOST_DEVICE
void process_qf_arg(
const DeviceTensor<1, T> &u,
const DeviceTensor<1, T> &v,
T &arg)
{
arg = u(0);
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
tensor<dual<T, T>, n, m> &arg)
{
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
arg(j, i).value = u((i * n) + j);
}
}
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
dual<T, T> &arg)
{
arg.value = u(0);
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
const DeviceTensor<1> &v,
dual<T, T> &arg)
{
arg.value = u(0);
arg.gradient = v(0);
}
template <typename T, int n>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
const DeviceTensor<1> &v,
tensor<dual<T, T>, n> &arg)
{
for (int i = 0; i < n; i++)
{
arg(i).value = u(i);
arg(i).gradient = v(i);
}
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
const DeviceTensor<1> &v,
tensor<dual<T, T>, n, m> &arg)
{
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
arg(j, i).value = u((i * n) + j);
arg(j, i).gradient = v((i * n) + j);
}
}
}
template <typename T, int n>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1, T> &r,
const tensor<dual<T, T>, n> &x)
{
for (size_t i = 0; i < n; i++)
{
r(i) = x(i).value;
}
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1, T> &r,
const tensor<dual<T, T>, n, m> &x)
{
for (size_t i = 0; i < n; i++)
{
for (size_t j = 0; j < m; j++)
{
r(i + n * j) = x(i, j).value;
}
}
}
template <typename arg_type>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<2> &u,
const DeviceTensor<2> &v,
arg_type &arg,
const int &qp)
{
const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
const auto v_qp = Reshape(&v(0, qp), v.GetShape()[0]);
process_qf_arg(u_qp, v_qp, arg);
}
template <size_t num_fields, typename qf_args>
MFEM_HOST_DEVICE inline
void process_qf_args(
const std::array<DeviceTensor<2>, num_fields> &u,
const std::array<DeviceTensor<2>, num_fields> &v,
qf_args &args,
const int &qp)
{
for_constexpr<tuple_size<qf_args>::value>([&](auto i)
{
process_qf_arg(u[i], v[i], get<i>(args), qp);
});
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_derivative_from_native_dual(
DeviceTensor<1, T> &r,
const tensor<dual<T, T>, n, m> &x)
{
for (size_t i = 0; i < n; i++)
{
for (size_t j = 0; j < m; j++)
{
r(i + n * j) = x(i, j).gradient;
}
}
}
template <typename T, int n>
MFEM_HOST_DEVICE inline
void process_derivative_from_native_dual(
DeviceTensor<1, T> &r,
const tensor<dual<T, T>, n> &x)
{
for (size_t i = 0; i < n; i++)
{
r(i) = x(i).gradient;
}
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_derivative_from_native_dual(
DeviceTensor<1, T> &r,
const dual<T, T> &x)
{
r(0) = x.gradient;
}
template <typename T0, typename T1>
MFEM_HOST_DEVICE inline
void process_qf_arg(const T0 &, T1 &)
{
static_assert(dfem::always_false<T0, T1>,
"process_qf_arg not implemented for arg type");
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1, T> &u,
T &arg)
{
arg = u(0);
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1, T> &u,
tensor<T> &arg)
{
arg(0) = u(0);
}
template <typename T, int n>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
tensor<T, n> &arg)
{
for (int i = 0; i < n; i++)
{
arg(i) = u(i);
}
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1> &u,
tensor<T, n, m> &arg)
{
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
arg(j, i) = u((i * n) + j);
}
}
}
template <typename T, int n>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1, T> &u,
const DeviceTensor<1, T> &v,
tensor<T, n> &arg)
{
for (int i = 0; i < n; i++)
{
arg(i) = u(i);
}
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_qf_arg(
const DeviceTensor<1, T> &u,
const DeviceTensor<1, T> &v,
tensor<T, n, m> &arg)
{
for (int i = 0; i < m; i++)
{
for (int j = 0; j < n; j++)
{
arg(j, i) = u((i * n) + j);
}
}
}
template <typename arg_type>
MFEM_HOST_DEVICE inline
void process_qf_arg(const DeviceTensor<2> &u, arg_type &arg, int qp)
{
const auto u_qp = Reshape(&u(0, qp), u.GetShape()[0]);
process_qf_arg(u_qp, arg);
}
template <size_t num_fields, typename qf_args>
MFEM_HOST_DEVICE inline
void process_qf_args(
const std::array<DeviceTensor<2>, num_fields> &u,
qf_args &args,
const int &qp)
{
for_constexpr<tuple_size<qf_args>::value>([&](auto i)
{
process_qf_arg(u[i], get<i>(args), qp);
});
}
template <typename T0, typename T1>
MFEM_HOST_DEVICE inline
Vector process_qf_result(T0, T1)
{
static_assert(dfem::always_false<T0, T1>,
"process_qf_result not implemented for result type");
return Vector{};
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1, T> &r,
const T &x)
{
r(0) = x;
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1> &r,
const dual<T, T> &x)
{
r(0) = x.value;
}
template <typename T>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1, T> &r,
const tensor<T> &x)
{
r(0) = x(0);
}
template <typename T, int n>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1, T> &r,
const tensor<T, n> &x)
{
for (size_t i = 0; i < n; i++)
{
r(i) = x(i);
}
}
template <typename T, int n, int m>
MFEM_HOST_DEVICE inline
void process_qf_result(
DeviceTensor<1, T> &r,
const tensor<T, n, m> &x)
{
for (size_t i = 0; i < n; i++)
{
for (size_t j = 0; j < m; j++)
{
r(i + n * j) = x(i, j);
}
}
}
} // namespace mfem::future

Some files were not shown because too many files have changed in this diff Show More